From 348ecd9446a439d57883b1ee47717172066cea35 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 23 Jun 2026 14:38:49 -0400 Subject: [PATCH 01/14] 24048: Compute deviations as ratio to case value --- module/analysis_weights.amlg | 1 + module/feature_residuals.amlg | 48 +++++++++++++++++++++++++++++++++++ module/hyperparameters.amlg | 6 +++++ module/react.amlg | 28 ++++++++++++++++++++ module/residuals.amlg | 23 +++++++++++++++++ 5 files changed, 106 insertions(+) diff --git a/module/analysis_weights.amlg b/module/analysis_weights.amlg index 15b70a2cc..83c076854 100644 --- a/module/analysis_weights.amlg +++ b/module/analysis_weights.amlg @@ -489,6 +489,7 @@ )) (call !UpdateHyperparameters (assoc + dev_percent_map (get residuals_map "dev_percent_map") feature_deviations (get residuals_map "residual_map") confusion_matrix_map (get residuals_map ["prediction_stats" "confusion_matrix"]) ordinal_feature_deviations (get residuals_map "ordinal_residual_map") diff --git a/module/feature_residuals.amlg b/module/feature_residuals.amlg index fb2de9f88..116e46a4a 100644 --- a/module/feature_residuals.amlg +++ b/module/feature_residuals.amlg @@ -797,7 +797,55 @@ )) ) + (declare (assoc + dev_percent_map + (if (= "deviations" robust_residuals) + (let + (assoc + continuous_features (indices (remove residuals_map (indices !nominalsMap))) + ) + (if (size continuous_features) + (let + (assoc cont_feature_map (zip continuous_features) ) + + (declare (assoc + f_values_maps + (map + (lambda (retrieve_from_entity (current_value) cont_feature_map) ) + case_ids + ) + )) + + (map + (lambda (let + (assoc feature (current_index 1)) + + (if (contains_index !nominalsMap feature) + (current_value) + + (/ + ;;;todo: handle null tuples and edit features + (current_value) + (generalized_mean + (map + (lambda (get (current_value) feature)) + f_values_maps + ) + ) + ) + + ) + )) + residuals_map + ) + ) + ) + ) + ) + )) + (assoc + "dev_percent_map" dev_percent_map "residual_map" residuals_map "ordinal_residual_map" ordinal_residuals_map "hyperparam_map" hyperparam_map diff --git a/module/hyperparameters.amlg b/module/hyperparameters.amlg index 3ee4507d5..8c1f1d4ce 100644 --- a/module/hyperparameters.amlg +++ b/module/hyperparameters.amlg @@ -28,6 +28,7 @@ ordinal_feature_deviations .null attribute_map (assoc) confusion_matrix_map .null + dev_percent_map .null ) (if (size confusion_matrix_map) @@ -194,6 +195,11 @@ (assoc) ) + (if (size dev_percent_map) + (assoc "featureDeviationRatios" dev_percent_map) + (assoc) + ) + ;null uncertainties store a tuple of [null, value<->null uncertainty, null<->null uncertainty] (if (size null_deviations) (assoc diff --git a/module/react.amlg b/module/react.amlg index 7f2562cd8..cb5d37b7a 100644 --- a/module/react.amlg +++ b/module/react.amlg @@ -1602,6 +1602,34 @@ ) )) + (if feature_deviations + (let + (assoc + ctx_value_map (remove (zip context_features context_values) (indices !nominalsMap)) + ) + + (assign (assoc + feature_deviations + (append + feature_deviations + (map + (lambda + ;;;TODO: handle nulls and edit distance features + (* + (max + (abs (current_value)) + (get !cachedFeatureHalfMinGapMap (current_index)) + ) + (get hyperparam_map ["featureDeviationRatios" (current_index 1)]) + ) + ) + ctx_value_map + ) + ) + )) + ) + ) + (declare (assoc ;reference the code in !ReactionQuery local_data_cases_tuple @(target .true "!ReactionQuery") diff --git a/module/residuals.amlg b/module/residuals.amlg index d8ff2f096..5f3c11d4d 100644 --- a/module/residuals.amlg +++ b/module/residuals.amlg @@ -961,6 +961,7 @@ feature_is_edit_distance (contains_index !editDistanceFeatureTypesMap (current_value 1)) feature_is_non_string_edit_distance .false feature_deviations (get hyperparam_map "featureDeviations") + feature_deviation_ratios_map (get hyperparam_map "featureDeviationRatios") feature_weights (if (= .null (get hyperparam_map "featureMdaMap")) (get hyperparam_map "featureWeights") @@ -1069,6 +1070,28 @@ )) ) + (if feature_deviations + (assign (assoc + feature_deviations + (append + feature_deviations + (map + (lambda + ;;;TODO: handle nulls and edit distance features + (* + (max + (abs (current_value)) + (get !cachedFeatureHalfMinGapMap (current_index)) + ) + (get feature_deviation_ratios_map (current_index)) + ) + ) + (remove case_values_map (indices !nominalsMap)) + ) + ) + )) + ) + (call !InterpolateAndComputeDiffToCaseFullResiduals (assoc output_categorical_action_probabilities .true case_feature_value (get case_values_map feature) From e26c3d54a19c10dd080e249d4e8dea3bf0738d0a Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 23 Jun 2026 14:43:32 -0400 Subject: [PATCH 02/14] dont allow negatives or 0s when computing average ratio --- module/feature_residuals.amlg | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/module/feature_residuals.amlg b/module/feature_residuals.amlg index 116e46a4a..b1f05c4b6 100644 --- a/module/feature_residuals.amlg +++ b/module/feature_residuals.amlg @@ -823,12 +823,21 @@ (if (contains_index !nominalsMap feature) (current_value) + ;divide average deviation by average feature value (/ ;;;todo: handle null tuples and edit features (current_value) + + ;compute average absolute feature value from the cases (generalized_mean (map - (lambda (get (current_value) feature)) + (lambda + ;don't allow 0s by replacing them with half min gaps + (max + (abs (get (current_value) feature)) + (get !cachedFeatureHalfMinGapMap feature) + ) + ) f_values_maps ) ) From 17a47d22cac6b08f13a1f90357aacf665ff6e6b0 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 23 Jun 2026 14:46:09 -0400 Subject: [PATCH 03/14] add comments --- module/react.amlg | 1 + module/residuals.amlg | 1 + 2 files changed, 2 insertions(+) diff --git a/module/react.amlg b/module/react.amlg index cb5d37b7a..47297ea0d 100644 --- a/module/react.amlg +++ b/module/react.amlg @@ -1608,6 +1608,7 @@ ctx_value_map (remove (zip context_features context_values) (indices !nominalsMap)) ) + ;overwrite continuous deviations with ratio from case value (assign (assoc feature_deviations (append diff --git a/module/residuals.amlg b/module/residuals.amlg index 5f3c11d4d..59f4a13fd 100644 --- a/module/residuals.amlg +++ b/module/residuals.amlg @@ -1071,6 +1071,7 @@ ) (if feature_deviations + ;overwrite continuous deviations with ratio from case value (assign (assoc feature_deviations (append From 1c90382cabf0e9d0a59c6ea03dac028b1ea15247 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 23 Jun 2026 15:21:27 -0400 Subject: [PATCH 04/14] changed to harmonic mean of ratios --- module/feature_residuals.amlg | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/module/feature_residuals.amlg b/module/feature_residuals.amlg index b1f05c4b6..0ab9ea495 100644 --- a/module/feature_residuals.amlg +++ b/module/feature_residuals.amlg @@ -808,6 +808,7 @@ (let (assoc cont_feature_map (zip continuous_features) ) + ;list of maps for all the cases of feature -> value for all continuous features (declare (assoc f_values_maps (map @@ -818,29 +819,34 @@ (map (lambda (let - (assoc feature (current_index 1)) + (assoc + feature (current_index 1) + deviation_value (current_value 1) + ) (if (contains_index !nominalsMap feature) (current_value) - ;divide average deviation by average feature value - (/ - ;;;todo: handle null tuples and edit features - (current_value) + (generalized_mean + (map + (lambda + ;divide global deviation by feature value + (/ + ;;;todo: handle null tuples and edit features + deviation_value - ;compute average absolute feature value from the cases - (generalized_mean - (map - (lambda ;don't allow 0s by replacing them with half min gaps (max (abs (get (current_value) feature)) (get !cachedFeatureHalfMinGapMap feature) ) ) - f_values_maps + ) + f_values_maps ) + ;type of mean 1=arithmetic, 0=geometric, -1=harmonic + -1 ) ) From 77e642e5ec7e8ab205b5945736890b60b1dc81df Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 24 Jun 2026 12:27:43 -0400 Subject: [PATCH 05/14] add ratio map to trainee and utilize ratio in deviation compute --- module/analysis_weights.amlg | 22 +++++++++++++++++++--- module/attributes.amlg | 5 +++++ module/feature_residuals.amlg | 12 ++++++------ module/hyperparameters.amlg | 6 +++--- module/mda_weight.amlg | 29 +++++++++++++++++++++++++++-- module/residuals.amlg | 2 +- module/trainee.amlg | 3 +++ 7 files changed, 64 insertions(+), 15 deletions(-) diff --git a/module/analysis_weights.amlg b/module/analysis_weights.amlg index 83c076854..7fb17fc48 100644 --- a/module/analysis_weights.amlg +++ b/module/analysis_weights.amlg @@ -417,6 +417,21 @@ initial_residual_values_map (call !ComputeInitialResiduals) )) ) + (declare (assoc + initial_ratios_map + (if !ratioMeasureMap + (map + (lambda + ;initial deviation / median + (/ + (get initial_residual_values_map (current_index)) + (compute_on_contained_entities (query_quantile (current_index) 0.5)) + ) + ) + !ratioMeasureMap + ) + ) + )) (declare (assoc in_analyze .true @@ -438,14 +453,15 @@ residuals_map (if (= 0 iteration) (assoc - "residual_map" (get_value initial_residual_values_map) + "residual_map" (modify initial_residual_values_map) "ordinal_residual_map" (if (size !ordinalFeaturesSet) (filter (lambda (contains_index !ordinalFeaturesRangesMap (current_index))) - (get_value initial_residual_values_map) + (modify initial_residual_values_map) ) ) + "ratios_map" initial_ratios_map ) (call !ExpandResidualValuesToUncertainty (assoc @@ -489,7 +505,7 @@ )) (call !UpdateHyperparameters (assoc - dev_percent_map (get residuals_map "dev_percent_map") + ratios_map (get residuals_map "ratios_map") feature_deviations (get residuals_map "residual_map") confusion_matrix_map (get residuals_map ["prediction_stats" "confusion_matrix"]) ordinal_feature_deviations (get residuals_map "ordinal_residual_map") diff --git a/module/attributes.amlg b/module/attributes.amlg index 3714523e2..5e3b0c387 100644 --- a/module/attributes.amlg +++ b/module/attributes.amlg @@ -684,6 +684,11 @@ (call !SetNominalFeatures (assoc nominal_features nominals)) (call !SetCyclicFeatures (assoc feature_attributes cyclics_map)) + ;;;TODO: add logic and attribute, for now set all non-ordinal and non-cyclic continuous as ratio + (assign_to_entities (assoc + !ratioMeasureMap (remove (zip !trainedFeatures) (append nominals ordinals)) + )) + ;overwrite computed deviations with user specified ones if user specified ones were larger (if (> (size !userSpecifiedFeatureErrorsMap) 0) (assign_to_entities (assoc diff --git a/module/feature_residuals.amlg b/module/feature_residuals.amlg index 0ab9ea495..cd8a0191a 100644 --- a/module/feature_residuals.amlg +++ b/module/feature_residuals.amlg @@ -798,15 +798,15 @@ ) (declare (assoc - dev_percent_map + ratios_map (if (= "deviations" robust_residuals) (let (assoc - continuous_features (indices (remove residuals_map (indices !nominalsMap))) + ratio_features (indices (keep residuals_map (indices !ratioMeasureMap))) ) - (if (size continuous_features) + (if (size ratio_features) (let - (assoc cont_feature_map (zip continuous_features) ) + (assoc cont_feature_map (zip ratio_features) ) ;list of maps for all the cases of feature -> value for all continuous features (declare (assoc @@ -851,7 +851,7 @@ ) )) - residuals_map + (keep residuals_map ratio_features) ) ) ) @@ -860,7 +860,7 @@ )) (assoc - "dev_percent_map" dev_percent_map + "ratios_map" ratios_map "residual_map" residuals_map "ordinal_residual_map" ordinal_residuals_map "hyperparam_map" hyperparam_map diff --git a/module/hyperparameters.amlg b/module/hyperparameters.amlg index 8c1f1d4ce..9ef75b251 100644 --- a/module/hyperparameters.amlg +++ b/module/hyperparameters.amlg @@ -28,7 +28,7 @@ ordinal_feature_deviations .null attribute_map (assoc) confusion_matrix_map .null - dev_percent_map .null + ratios_map .null ) (if (size confusion_matrix_map) @@ -195,8 +195,8 @@ (assoc) ) - (if (size dev_percent_map) - (assoc "featureDeviationRatios" dev_percent_map) + (if (size ratios_map) + (assoc "featureDeviationRatios" ratios_map) (assoc) ) diff --git a/module/mda_weight.amlg b/module/mda_weight.amlg index c52b01105..084e28ddf 100644 --- a/module/mda_weight.amlg +++ b/module/mda_weight.amlg @@ -86,8 +86,33 @@ time_series_filter_query (list) dependent_queries_list .null fanout_features_queries .null + feature_deviation_ratios_map (get hyperparam_map "featureDeviationRatios") ) + (declare (assoc + feature_deviations + (if !ratioMeasureMap + (append + (get hyperparam_map "featureDeviations") + (map + (lambda + ;;;TODO: handle nulls and edit distance features + (* + (max + (abs (current_value)) + (get !cachedFeatureHalfMinGapMap (current_index)) + ) + (get feature_deviation_ratios_map (current_index)) + ) + ) + (keep case_values_map (indices !ratioMeasureMap)) + ) + ) + + (get hyperparam_map "featureDeviations") + ) + )) + (if !tsTimeFeature (if (contains_index case_values_map !tsTimeFeature) (assign (assoc @@ -150,7 +175,7 @@ p_parameter (get hyperparam_map "featureWeights") !queryFeatureAttributeTypeMap - (get hyperparam_map "featureDeviations") + feature_deviations .null dt_parameter (if valid_weight_feature weight_feature .null) @@ -309,7 +334,7 @@ p_parameter feature_weights !queryFeatureAttributeTypeMap - (get hyperparam_map "featureDeviations") + feature_deviations .null dt_parameter (if valid_weight_feature weight_feature .null) diff --git a/module/residuals.amlg b/module/residuals.amlg index 59f4a13fd..70eccb782 100644 --- a/module/residuals.amlg +++ b/module/residuals.amlg @@ -1087,7 +1087,7 @@ (get feature_deviation_ratios_map (current_index)) ) ) - (remove case_values_map (indices !nominalsMap)) + (keep case_values_map (indices !ratioMeasureMap)) ) ) )) diff --git a/module/trainee.amlg b/module/trainee.amlg index 0ee24152c..33213b830 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -249,6 +249,9 @@ ; categorical features are taken from a single case instead of interpolated between training cases !categoricalFeaturesSet (assoc) + ;map of continuous features that use ratio (e.g., 1/10 of a case value) instead of interval (a fixed value) for deviations + !ratioMeasureMap .null + ;map of nominal features for fast lookup !nominalsMap (assoc) From 7ede3e6de721db29c60ed9259d303712346765bb Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 24 Jun 2026 13:45:59 -0400 Subject: [PATCH 06/14] add to robust residuals --- module/residuals.amlg | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/module/residuals.amlg b/module/residuals.amlg index 70eccb782..fa73efa1b 100644 --- a/module/residuals.amlg +++ b/module/residuals.amlg @@ -682,6 +682,7 @@ case_values_map (zip case_features (retrieve_from_entity (current_value 1) case_features) ) time_series_filter_query (list) feature_weights (get hyperparam_map "featureWeights") + feature_deviation_ratios_map (get hyperparam_map "featureDeviationRatios") context_features context_features dependent_queries_list .null fanout_features_queries .null @@ -693,6 +694,29 @@ ) ) + (if feature_deviations + ;overwrite continuous deviations with ratio from case value + (assign (assoc + feature_deviations + (append + feature_deviations + (map + (lambda + ;;;TODO: handle nulls and edit distance features + (* + (max + (abs (current_value)) + (get !cachedFeatureHalfMinGapMap (current_index)) + ) + (get feature_deviation_ratios_map (current_index)) + ) + ) + (keep case_values_map (indices !ratioMeasureMap)) + ) + ) + )) + ) + (if !tsTimeFeature (assign (assoc context_features From 02997babd7ce1190df894ba0ac8c4d1959fff7a3 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 25 Jun 2026 08:14:10 -0400 Subject: [PATCH 07/14] fix attribute code --- module/attributes.amlg | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/module/attributes.amlg b/module/attributes.amlg index 5e3b0c387..e069beeec 100644 --- a/module/attributes.amlg +++ b/module/attributes.amlg @@ -685,9 +685,12 @@ (call !SetCyclicFeatures (assoc feature_attributes cyclics_map)) ;;;TODO: add logic and attribute, for now set all non-ordinal and non-cyclic continuous as ratio - (assign_to_entities (assoc - !ratioMeasureMap (remove (zip !trainedFeatures) (append nominals ordinals)) + (declare (assoc + ratio_features_map (remove (zip !trainedFeatures) (append nominals ordinals)) )) + (if (size ratio_features_map) + (assign_to_entities (assoc !ratioMeasureMap ratio_features_map )) + ) ;overwrite computed deviations with user specified ones if user specified ones were larger (if (> (size !userSpecifiedFeatureErrorsMap) 0) From 8bbb4055822867c0591b6b76473dd02fb322372d Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 25 Jun 2026 10:14:40 -0400 Subject: [PATCH 08/14] improved comments --- module/attributes.amlg | 2 +- module/feature_residuals.amlg | 4 +++- module/hyperparameters.amlg | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/module/attributes.amlg b/module/attributes.amlg index e069beeec..6c3b2a8eb 100644 --- a/module/attributes.amlg +++ b/module/attributes.amlg @@ -684,7 +684,7 @@ (call !SetNominalFeatures (assoc nominal_features nominals)) (call !SetCyclicFeatures (assoc feature_attributes cyclics_map)) - ;;;TODO: add logic and attribute, for now set all non-ordinal and non-cyclic continuous as ratio + ;;;TODO: add logic to parse ratio feature attribute, for now set all non-ordinal and non-cyclic continuous as ratio (declare (assoc ratio_features_map (remove (zip !trainedFeatures) (append nominals ordinals)) )) diff --git a/module/feature_residuals.amlg b/module/feature_residuals.amlg index cd8a0191a..343635321 100644 --- a/module/feature_residuals.amlg +++ b/module/feature_residuals.amlg @@ -799,6 +799,7 @@ (declare (assoc ratios_map + ;only set/output ratios when computing deviations (if (= "deviations" robust_residuals) (let (assoc @@ -824,6 +825,7 @@ deviation_value (current_value 1) ) + ;does not affect nominal features (if (contains_index !nominalsMap feature) (current_value) @@ -832,7 +834,7 @@ (lambda ;divide global deviation by feature value (/ - ;;;todo: handle null tuples and edit features + ;;;TODO: handle null tuples and edit features deviation_value ;don't allow 0s by replacing them with half min gaps diff --git a/module/hyperparameters.amlg b/module/hyperparameters.amlg index 9ef75b251..af0e89201 100644 --- a/module/hyperparameters.amlg +++ b/module/hyperparameters.amlg @@ -196,6 +196,7 @@ ) (if (size ratios_map) + ;;;TODO: add featureDeviationRatios to return type for hyperparameters (assoc "featureDeviationRatios" ratios_map) (assoc) ) From d2bcd42b6f59eea571c37db62ae43805903267e1 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 25 Jun 2026 12:53:38 -0400 Subject: [PATCH 09/14] adds support for continuous_interval and continuous_ratio --- module/analysis.amlg | 2 +- module/analysis_weights.amlg | 2 +- module/attribute_maps.amlg | 17 +++---- module/attributes.amlg | 26 ++++++++++- module/conviction.amlg | 6 +-- module/derive_utilities.amlg | 2 +- module/editing.amlg | 2 +- module/feature_residuals.amlg | 4 +- module/generate_features.amlg | 20 ++++---- module/mda_weight.amlg | 45 ++++++++++-------- module/react.amlg | 28 ++++++++---- module/react_utilities.amlg | 2 +- module/residuals.amlg | 61 +++++++++++++++++-------- module/scale.amlg | 2 +- module/train_utilities.amlg | 4 +- module/trainee.amlg | 5 +- module/types.amlg | 6 ++- unit_tests/ut_h_analyze.amlg | 2 +- unit_tests/ut_h_edit_dist_features.amlg | 10 ++-- unit_tests/ut_h_migration.amlg | 2 +- 20 files changed, 157 insertions(+), 91 deletions(-) diff --git a/module/analysis.amlg b/module/analysis.amlg index cc7fc4239..d868fefde 100644 --- a/module/analysis.amlg +++ b/module/analysis.amlg @@ -720,7 +720,7 @@ features ;filter the context features to get just continuous (filter - (lambda (= "continuous" (get !featureAttributes [(current_value 1) "type"]))) + (lambda (contains_value [ "continuous_interval" "continuous_ratio"] (get !featureAttributes [(current_value 1) "type"]))) context_features ) k_parameter .null diff --git a/module/analysis_weights.amlg b/module/analysis_weights.amlg index 7fb17fc48..c10c581b6 100644 --- a/module/analysis_weights.amlg +++ b/module/analysis_weights.amlg @@ -422,7 +422,7 @@ (if !ratioMeasureMap (map (lambda - ;initial deviation / median + ;;;TODO: use generalized mean of initial_deviation / case value, for now just use initial deviation / median (/ (get initial_residual_values_map (current_index)) (compute_on_contained_entities (query_quantile (current_index) 0.5)) diff --git a/module/attribute_maps.amlg b/module/attribute_maps.amlg index e034df7da..37ee981a1 100644 --- a/module/attribute_maps.amlg +++ b/module/attribute_maps.amlg @@ -117,14 +117,14 @@ (lambda (if (and - (= "continuous" (get (current_value) "type")) + (contains_value ["continuous_interval" "continuous_ratio"] (get (current_value) "type")) (= "formatted_date_time" (get (current_value) "data_type")) (= .null (get (current_value) "date_time_format")) ) (modify (current_value) "date_time_format" "%Y-%m-%dT%H:%M:%S") (and - (= "continuous" (get (current_value) "type")) + (contains_value ["continuous_interval" "continuous_ratio"] (get (current_value) "type")) (= .null (get (current_value) "date_time_format")) (= "formatted_time" (get (current_value) "data_type")) ) @@ -133,7 +133,7 @@ ;force 'data_type' to 'formatted_date_time' for all features with a date_time_format ;that are not already set to 'formatted_time' (and - (= "continuous" (get (current_value) "type")) + (contains_value ["continuous_interval" "continuous_ratio"] (get (current_value) "type")) (!= .null (get (current_value) "date_time_format")) (!= "formatted_time" (get (current_value) "data_type")) ) @@ -611,8 +611,8 @@ (lambda ;only allowed data_type values are string, json and amalgam, all others are ignored (if (and - (= "continuous" (get (last (current_value)) "type")) - (contains_value (list "string" "string_mixable" "json" "yaml" "amalgam") (get (last (current_value)) "data_type") ) + (contains_value ["continuous_interval" "continuous_ratio"] (get (last (current_value)) "type")) + (contains_value ["string" "string_mixable" "json" "yaml" "amalgam"] (get (last (current_value)) "data_type") ) ) (list (modify (first (current_value 1))) (modify (get (last (current_value 1)) "data_type"))) ;else don't output anything, map-filtering out any values that don't have datype defined @@ -851,7 +851,7 @@ (contains_index cyclics_map (current_index)) { - "difference_type" "continuous" + "difference_type" "continuous_interval" "data_type" "number" "cycle_range" (modify (get cyclics_map (current_index 1))) } @@ -872,17 +872,18 @@ "number" ) + difference_type (get (current_value 1) "type") ) (if (!= "code" data_type) { - "difference_type" "continuous" + "difference_type" difference_type "data_type" data_type } ;continuous code specific datatypes are provided { - "difference_type" "continuous" + "difference_type" difference_type "data_type" data_type "types_must_match" ;default to true diff --git a/module/attributes.amlg b/module/attributes.amlg index 6c3b2a8eb..698736033 100644 --- a/module/attributes.amlg +++ b/module/attributes.amlg @@ -8,7 +8,8 @@ (assoc ;assoc of feature -> attributes, attributes defined below: ; - ; 'type': string, one of 'continuous', 'ordinal' or 'nominal'. Default is 'continuous'. + ; 'type': string, one of 'continuous_interval', 'continuous_ratio', 'ordinal' or 'nominal'. Default is 'continuous_interval'. + ; note: 'continuous' is supported but will be overwritten with 'continuous_interval'. ; ; 'bounds' : assoc of of bounds for feature value generation, keys of assoc defined below as: ; 'min': number or date string, the minimum value to be output @@ -232,6 +233,20 @@ ) )) + ;backwards compatibility with type='continuous', overwrite with 'continuous_interval' + (assign (assoc + feature_attributes + (map + (lambda + (if (= "continuous" (get (current_value) "type")) + (modify (current_value) "type" "continuous_interval") + (current_value) + ) + ) + feature_attributes + ) + )) + (declare (assoc ordinals (list) nominals (list) @@ -239,6 +254,7 @@ cyclics_map (assoc) nominals_map (assoc) ordinals_map (assoc) + continuous_map (assoc) boundaries_map (assoc) source_to_derived_feature_map (assoc) @@ -294,6 +310,11 @@ (lambda (= "nominal" (get (current_value) "type"))) feature_attributes ) + continuous_map + (filter + (lambda (contains_value ["continuous_interval" "continuous_ratio"] (get (current_value) "type"))) + feature_attributes + ) boundaries_map (filter (lambda (or (!= .null (get (current_value) "bounds")) (!= .null (get (current_value) "allow_null")))) @@ -677,6 +698,7 @@ !sharedDeviationsPrimaryFeatures (get shared_deviations_map "shared_deviations_primary_features") !sharedDeviationsNonPrimaryFeatures (get shared_deviations_map "shared_deviations_non_primary_features") + !continuousMap (zip (indices continuous_map)) )) (call !UpdateDefinedFeatures) @@ -846,7 +868,7 @@ "ordinal" ;else - "continuous" + "continuous_interval" ) decimal_places (get !featureRoundingMap (list (current_index 2) 1)) significant_digits (get !featureRoundingMap (list (current_index 2) 0)) diff --git a/module/conviction.amlg b/module/conviction.amlg index 42e4ffad7..735d6824a 100644 --- a/module/conviction.amlg +++ b/module/conviction.amlg @@ -264,10 +264,10 @@ (map (lambda (if (or (= "p_value_of_addition" (current_value)) (= "p_value_of_removal" (current_value))) - {"type" "continuous" "data_type" "number" "bounds" {"min" 0 "max" 1} } + {"type" "continuous_interval" "data_type" "number" "bounds" {"min" 0 "max" 1} } ;else convictions, entropy and distance contribution ranges are 0 to infinity - {"type" "continuous" "data_type" "number" "bounds" {"min" 0 "max" .infinity} } + {"type" "continuous_interval" "data_type" "number" "bounds" {"min" 0 "max" .infinity} } ) ) (indices computed_map) @@ -303,7 +303,7 @@ "weight_feature" (if use_case_weights weight_feature) ) !autoAnalyzeEnabled .true - !queryFeatureAttributeTypeMap (append !queryFeatureAttributeTypeMap (zip (values computed_map) {"difference_type" "continuous" "data_type" "number"}) ) + !queryFeatureAttributeTypeMap (append !queryFeatureAttributeTypeMap (zip (values computed_map) {"difference_type" "continuous_interval" "data_type" "number"}) ) )) ;need to cache computed values to be able to approximate any of these listed computations during future train calls diff --git a/module/derive_utilities.amlg b/module/derive_utilities.amlg index 81436febe..ef7fb4557 100644 --- a/module/derive_utilities.amlg +++ b/module/derive_utilities.amlg @@ -807,7 +807,7 @@ (lambda (or ;categorical features are in the "delta_features" list... - (!= "continuous" (get !featureAttributes [(current_value 1) "type"])) + (contains_value ["ordinal" "nominal"] (get !featureAttributes [(current_value 1) "type"])) (contains_value derived_action_features (get !derivedFeaturesMap (current_value))) ) ) diff --git a/module/editing.amlg b/module/editing.amlg index 7fa41df96..d069666d3 100644 --- a/module/editing.amlg +++ b/module/editing.amlg @@ -435,7 +435,7 @@ (assign (assoc feature_attributes (assoc - "type" "continuous" + "type" "continuous_interval" "bounds" (assoc "allow_null" .true) ) )) diff --git a/module/feature_residuals.amlg b/module/feature_residuals.amlg index 343635321..ffeebd34a 100644 --- a/module/feature_residuals.amlg +++ b/module/feature_residuals.amlg @@ -1985,7 +1985,7 @@ (lambda (or ;categorical features are in the "delta_features" list... - (!= "continuous" (get !featureAttributes [(current_value 1) "type"])) + (contains_value ["ordinal" "nominal"] (get !featureAttributes [(current_value 1) "type"])) (contains_value derived_action_features (get !derivedFeaturesMap (current_value))) ) ) @@ -2125,7 +2125,7 @@ feat_index (get feature_index_map (current_value 1)) ) - (if (= "continuous" (get !featureAttributes [feat_to_diff "type"])) + (if (contains_index !continuousMap feat_to_diff) (call !ComputeDiffTuple (assoc feature feat_to_diff feature_has_continuous_nulls (contains_index continuous_with_nulls_map feat_to_diff) diff --git a/module/generate_features.amlg b/module/generate_features.amlg index 3ad3b6042..1eaa12ac0 100644 --- a/module/generate_features.amlg +++ b/module/generate_features.amlg @@ -353,7 +353,7 @@ (if ;time-series feature that's continuous and doesn't have defined derived_feature_code and is not an id_feature (and (contains_index (current_value) "time_series") - (= "continuous" (get (current_value) "type")) + (contains_value ["continuous_interval" "continuous_ratio"] (get (current_value) "type")) (not (contains_index (current_value) "derived_feature_code")) (not (get (current_value) "id_feature")) ;covariate ts-features are not derived @@ -666,7 +666,7 @@ ;Cannot compute rates/deltas for categorical features or edit distance features, only for numeric continuous (if (and - (= "continuous" (get attributes "type")) + (contains_index !continuousMap feature) (not (contains_index !editDistanceFeatureTypesMap feature)) ) (let @@ -780,7 +780,7 @@ ;counter for following cases with the same time value within the same series ".synchronous_counter" (assoc - "type" "continuous" + "type" "continuous_interval" ;if previous case's counter is greater than 0 decrement, otherwise generate a new value "bounds" (assoc "min" 0.0 "allow_null" .false) "decimal_places" 0 @@ -788,7 +788,7 @@ ) ".synchronous_counter_lag_1" (assoc - "type" "continuous" + "type" "continuous_interval" "bounds" (assoc "min" 0.0 "allow_null" .true) "decimal_places" 0 "original_type" {"data_type" "integer" "size" 8} @@ -798,7 +798,7 @@ ) ".series_index" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "original_type" {"data_type" "integer" "size" 8} "derived_feature_code" "(+ 1 (call value {feature \".series_index\" lag 1}))" @@ -806,7 +806,7 @@ ) ".reverse_series_index" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "original_type" {"data_type" "integer" "size" 8} "bounds" (assoc "min" 0.0 "allow_null" .false) @@ -820,7 +820,7 @@ (assoc ".time_to_horizon" (assoc - "type" "continuous" + "type" "continuous_interval" "derived_feature_code" "(call value {feature \".time_to_horizon\" lag 1})" "bounds" (assoc "min" 0.0 "allow_null" .false) "original_type" {"data_type" "numeric" "size" 8} @@ -887,7 +887,7 @@ (append lag_sub_feature_names feature) ) ) - "type" (if (contains_index attributes "type") (get attributes "type") "continuous") + "type" (if (contains_index attributes "type") (get attributes "type") "continuous_interval") "derived_feature_code" (call !GenerateFeatureAttributeDerivation (assoc derivation_type "lag" @@ -940,7 +940,7 @@ (range (lambda (append (assoc - "type" (if (contains_index attributes "type") (get attributes "type") "continuous") + "type" (if (contains_index attributes "type") (get attributes "type") "continuous_interval") "bounds" (modify (append @@ -1001,7 +1001,7 @@ (range (lambda (append (assoc - "type" (if (contains_index attributes "type") (get attributes "type") "continuous") + "type" (if (contains_index attributes "type") (get attributes "type") "continuous_interval") "bounds" (append (assoc "allow_null" .true) diff --git a/module/mda_weight.amlg b/module/mda_weight.amlg index 084e28ddf..022ba2d08 100644 --- a/module/mda_weight.amlg +++ b/module/mda_weight.amlg @@ -87,31 +87,40 @@ dependent_queries_list .null fanout_features_queries .null feature_deviation_ratios_map (get hyperparam_map "featureDeviationRatios") + feature_deviations (get hyperparam_map "featureDeviations") ) - (declare (assoc - feature_deviations - (if !ratioMeasureMap + + (if !ratioMeasureMap + (assign (assoc + feature_deviations (append - (get hyperparam_map "featureDeviations") + feature_deviations (map - (lambda - ;;;TODO: handle nulls and edit distance features - (* - (max - (abs (current_value)) - (get !cachedFeatureHalfMinGapMap (current_index)) - ) - (get feature_deviation_ratios_map (current_index)) + (lambda (let + ;;;TODO: edit distance features + (assoc + updated_deviation + (* + (max + (abs (current_value 1)) + (get !cachedFeatureHalfMinGapMap (current_index 1)) + ) + (get feature_deviation_ratios_map (current_index 1)) + ) ) - ) - (keep case_values_map (indices !ratioMeasureMap)) + ;if the deviation is a tuple, overwrite the first value with this updated_deviation + (if (~ [] (get feature_deviations (current_index))) + (modify (get feature_deviations (current_index)) 0 updated_deviation) + updated_deviation + ) + + )) + (keep (filter case_values_map) (indices !ratioMeasureMap)) ) ) - - (get hyperparam_map "featureDeviations") - ) - )) + )) + ) (if !tsTimeFeature (if (contains_index case_values_map !tsTimeFeature) diff --git a/module/react.amlg b/module/react.amlg index 47297ea0d..0cf70d9e6 100644 --- a/module/react.amlg +++ b/module/react.amlg @@ -1614,17 +1614,25 @@ (append feature_deviations (map - (lambda - ;;;TODO: handle nulls and edit distance features - (* - (max - (abs (current_value)) - (get !cachedFeatureHalfMinGapMap (current_index)) - ) - (get hyperparam_map ["featureDeviationRatios" (current_index 1)]) + (lambda (let + ;;;TODO: edit distance features + (assoc + updated_deviation + (* + (max + (abs (current_value 1)) + (get !cachedFeatureHalfMinGapMap (current_index 1)) + ) + (get hyperparam_map ["featureDeviationRatios" (current_index 2)]) + ) ) - ) - ctx_value_map + ;if the deviation is a tuple, overwrite the first value with this updated_deviation + (if (~ [] (get feature_deviations (current_index))) + (modify (get feature_deviations (current_index)) 0 updated_deviation) + updated_deviation + ) + )) + (keep (filter ctx_value_map) (indices !ratioMeasureMap)) ) ) )) diff --git a/module/react_utilities.amlg b/module/react_utilities.amlg index f521465ef..d28b55898 100644 --- a/module/react_utilities.amlg +++ b/module/react_utilities.amlg @@ -881,7 +881,7 @@ continuous_context_features ;filter the context features to get just continuous (filter - (lambda (= "continuous" (get !featureAttributes [(current_value 1) "type"]))) + (lambda (contains_index !continuousMap (current_value)) context_features ) )) diff --git a/module/residuals.amlg b/module/residuals.amlg index 884eec24b..1661be79e 100644 --- a/module/residuals.amlg +++ b/module/residuals.amlg @@ -671,6 +671,8 @@ (declare (assoc total_samples 0)) ) + (declare (assoc original_feature_deviations (modify feature_deviations))) + ;iterate over each case and accumulate residuals for all the feature(s) (assign (assoc case_residuals_lists @@ -699,18 +701,27 @@ (assign (assoc feature_deviations (append - feature_deviations + original_feature_deviations (map - (lambda + (lambda (let ;;;TODO: handle nulls and edit distance features - (* - (max - (abs (current_value)) - (get !cachedFeatureHalfMinGapMap (current_index)) - ) - (get feature_deviation_ratios_map (current_index)) + (assoc + updated_deviation + (* + (max + (abs (current_value 1)) + (get !cachedFeatureHalfMinGapMap (current_index 1)) + ) + (get feature_deviation_ratios_map (current_index 1)) + ) ) - ) + ;if the deviation is a tuple, overwrite the first value with this updated_deviation + (if (~ [] (get original_feature_deviations (current_index))) + (modify (get original_feature_deviations (current_index)) 0 updated_deviation) + + updated_deviation + ) + )) (keep case_values_map (indices !ratioMeasureMap)) ) ) @@ -972,6 +983,7 @@ ;helper method for CalculateFeatureResiduals to calculate full residuals !RunFullResiduals (seq + (declare (assoc original_feature_deviations (get hyperparam_map "featureDeviations") )) (assign (assoc feature_residuals_lists @@ -1099,19 +1111,28 @@ (assign (assoc feature_deviations (append - feature_deviations + original_feature_deviations (map - (lambda - ;;;TODO: handle nulls and edit distance features - (* - (max - (abs (current_value)) - (get !cachedFeatureHalfMinGapMap (current_index)) - ) - (get feature_deviation_ratios_map (current_index)) + (lambda (let + ;;;TODO: edit distance features + (assoc + updated_deviation + (* + (max + (abs (current_value 1)) + (get !cachedFeatureHalfMinGapMap (current_index 1)) + ) + (get feature_deviation_ratios_map (current_index 1)) + ) ) - ) - (keep case_values_map (indices !ratioMeasureMap)) + ;if the deviation is a tuple, overwrite the first value with this updated_deviation + (if (~ [] (get original_feature_deviations (current_index))) + (modify (get original_feature_deviations (current_index)) 0 updated_deviation) + + updated_deviation + ) + )) + (keep (filter case_values_map) (indices !ratioMeasureMap)) ) ) )) diff --git a/module/scale.amlg b/module/scale.amlg index a8266ab33..212ef5c84 100644 --- a/module/scale.amlg +++ b/module/scale.amlg @@ -696,7 +696,7 @@ ;If there are not many (20, for now) unique values, this may be ok (and (= "nominal" (get old_attributes "type")) - (= "continuous" (get new_attributes "type")) + (contains_value ["continuous_interval" "continuous_ratio"] (get new_attributes "type")) (<= (size (compute_on_contained_entities (query_exists !internalLabelSession) diff --git a/module/train_utilities.amlg b/module/train_utilities.amlg index ac1227fcf..def9a525d 100644 --- a/module/train_utilities.amlg +++ b/module/train_utilities.amlg @@ -841,10 +841,10 @@ (accum_to_entities (assoc !featureAttributes (map - (lambda (assoc "type" "continuous" "bounds" (assoc "allow_null" .true) )) + (lambda (assoc "type" "continuous_interval" "bounds" (assoc "allow_null" .true) )) new_features ) - !queryFeatureAttributeTypeMap (map (lambda {"difference_type" "continuous" "data_type" "number"}) new_features) + !queryFeatureAttributeTypeMap (map (lambda {"difference_type" "continuous_interval" "data_type" "number"}) new_features) )) (assign_to_entities (assoc !trainedFeatures (sort (values (append !trainedFeatures (indices !featureAttributes)) .true) ) diff --git a/module/trainee.amlg b/module/trainee.amlg index 33cb44632..572498969 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -255,6 +255,9 @@ ;map of nominal features for fast lookup !nominalsMap (assoc) + ;map of continuous (non ordinal) features for fast lookup + !continuousMap (assoc) + ;set of features that will use novel nominal substition once output !novelSubstitionFeatureSet (assoc) @@ -278,7 +281,7 @@ ;store class probabilities for each nominal feature; weight feature -> feature -> probability !nominalClassProbabilitiesMap (assoc) - ;map of feature -> query attribute types for use in queries. i.e., "nominal", "continuous", specifying 'data_type', etc + ;map of feature -> query attribute types for use in queries. i.e., "nominal", "continuous_intetrval", specifying 'data_type', etc !queryFeatureAttributeTypeMap (assoc) ;set of ordinal features diff --git a/module/types.amlg b/module/types.amlg index a1b567833..8806b5c65 100644 --- a/module/types.amlg +++ b/module/types.amlg @@ -309,11 +309,13 @@ FeatureType (assoc type "string" - enum ["continuous" "ordinal" "nominal"] + enum ["continuous" "ordinal" "nominal" "continuous_interval" "continuous_ratio"] description (concat "The type of the feature.\n\n" - "- continuous: A continuous numeric value. (e.g. Temperature or humidity)\n" + "- continuous: A continuous numeric value. (e.g. Temperature or humidity), same as continuous_interval\n" + "- continuous_interval: A continuous numeric value. (e.g. number of hits), applicable to more even numerical distrubitions\n" + "- continuous_ratio: A continuous numeric value. (e.g. bank balance), applicable to long tail distributions with extreme differences in values\n" "- nominal: A numeric or string value with no ordering. (e.g. The name of a fruit)\n" "- ordinal: A nominal numeric value with ordering. (e.g. Rating scale, 1-5 stars)\n" ) diff --git a/unit_tests/ut_h_analyze.amlg b/unit_tests/ut_h_analyze.amlg index a9217a64a..40a995469 100644 --- a/unit_tests/ut_h_analyze.amlg +++ b/unit_tests/ut_h_analyze.amlg @@ -228,7 +228,7 @@ obs (keep (call_entity "howso" "debug_label" (assoc label "!queryFeatureAttributeTypeMap")) (list "TEST" "TEST_NOM")) exp { - TEST {data_type "number" difference_type "continuous"} + TEST {data_type "number" difference_type "continuous_interval"} TEST_NOM {data_type "string" difference_type "nominal"} } )) diff --git a/unit_tests/ut_h_edit_dist_features.amlg b/unit_tests/ut_h_edit_dist_features.amlg index 2852ceb22..c87c449fe 100644 --- a/unit_tests/ut_h_edit_dist_features.amlg +++ b/unit_tests/ut_h_edit_dist_features.amlg @@ -86,7 +86,7 @@ { amalgam { data_type "code" - difference_type "continuous" + difference_type "continuous_interval" nominal_numbers .false nominal_strings .true recursive_matching .true @@ -94,17 +94,17 @@ } json { data_type "code" - difference_type "continuous" + difference_type "continuous_interval" nominal_numbers .false nominal_strings .true recursive_matching .false types_must_match .true } - string {data_type "string" difference_type "continuous"} - x {data_type "number" difference_type "continuous"} + string {data_type "string" difference_type "continuous_interval"} + x {data_type "number" difference_type "continuous_interval"} yaml { data_type "code" - difference_type "continuous" + difference_type "continuous_interval" nominal_numbers .false nominal_strings .true recursive_matching .false diff --git a/unit_tests/ut_h_migration.amlg b/unit_tests/ut_h_migration.amlg index 134698aa3..528c8d141 100644 --- a/unit_tests/ut_h_migration.amlg +++ b/unit_tests/ut_h_migration.amlg @@ -87,7 +87,7 @@ )) (print "Continuous query feature attribute type: ") (call assert_same (assoc - exp {data_type "number" difference_type "continuous"} + exp {data_type "number" difference_type "continuous_interval"} obs (get query_feature_types_map "f1") )) (print "Nominal query feature attribute type: ") From dd8019ae218f5cc11b7da9df9f93580143a5d3e8 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 25 Jun 2026 15:22:49 -0400 Subject: [PATCH 10/14] overwrite parameter featureDeviations with ratio as appropriate --- module/hyperparameters.amlg | 22 +++++++++++++++++----- module/react_utilities.amlg | 2 +- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/module/hyperparameters.amlg b/module/hyperparameters.amlg index af0e89201..62cb7b40c 100644 --- a/module/hyperparameters.amlg +++ b/module/hyperparameters.amlg @@ -124,7 +124,10 @@ (if (~ (assoc) (first (current_value))) (append (list (get (current_value 1) (list 0 "sdm"))) (last (current_value)) ) - (append (first (current_value)) (last (current_value)) ) + (if (contains_index ratios_map (current_index )) + (append (get ratios_map (current_index)) (last (current_value)) ) + (append (first (current_value)) (last (current_value)) ) + ) ) ;else return the deviation value without the known-unkwnown uncertainties @@ -133,7 +136,11 @@ ;since sdm is a tuple, must wrap it in a list so the tuple itself is treated as the deviation [(get (current_value 1) [0 "sdm"])] - (first (current_value)) + ;if continuous_ratio, store the ratio, else store the deviation + (if (contains_index ratios_map (current_index )) + (get ratios_map (current_index)) + (first (current_value)) + ) ) ) ) @@ -149,7 +156,11 @@ ;since sdm is a tuple, must wrap it in a list so the tuple itself is treated as the deviation [(get (current_value 1) "sdm")] - (current_value) + ;if continuous_ratio, store the ratio, else store the deviation + (if (contains_index ratios_map (current_index )) + (get ratios_map (current_index)) + (current_value) + ) ) ) feature_deviations @@ -162,7 +173,7 @@ (assoc "featureDeviations" ;prepend a null to represent the undefined feature deviation - (map (lambda (append (list .null) (modify (current_value)))) null_deviations) + (map (lambda (append [.null] (modify (current_value)))) null_deviations) ) (assoc) @@ -195,8 +206,9 @@ (assoc) ) + ;;;TODO: remove this as we shouldn't need to store ratios separately (outside of featureDeviations) (if (size ratios_map) - ;;;TODO: add featureDeviationRatios to return type for hyperparameters + (assoc "featureDeviationRatios" ratios_map) (assoc) ) diff --git a/module/react_utilities.amlg b/module/react_utilities.amlg index d28b55898..80af50f49 100644 --- a/module/react_utilities.amlg +++ b/module/react_utilities.amlg @@ -881,7 +881,7 @@ continuous_context_features ;filter the context features to get just continuous (filter - (lambda (contains_index !continuousMap (current_value)) + (lambda (contains_index !continuousMap (current_value))) context_features ) )) From 5a40bf75ee011e3fcfa792e5d7eb2552e260e06e Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 25 Jun 2026 15:35:39 -0400 Subject: [PATCH 11/14] update to continuous_interval --- module/attributes.amlg | 11 ++++++++--- performance_tests/adult_test.amlg | 12 ++++++------ performance_tests/asteroid_test.amlg | 4 ++-- performance_tests/online_retail_test.amlg | 6 +++--- performance_tests/time_series_test.amlg | 4 ++-- unit_tests/ut_h_ablate_ts.amlg | 2 +- unit_tests/ut_h_analyze.amlg | 2 +- unit_tests/ut_h_box_conviction.amlg | 4 ++-- unit_tests/ut_h_constraints.amlg | 14 +++++++------- unit_tests/ut_h_cyclic_dataset.amlg | 14 +++++++------- unit_tests/ut_h_datetimeformat.amlg | 4 ++-- unit_tests/ut_h_density_conviction.amlg | 4 ++-- unit_tests/ut_h_dependent_features.amlg | 6 +++--- unit_tests/ut_h_derive_custom.amlg | 12 ++++++------ unit_tests/ut_h_derive_start_end.amlg | 4 ++-- unit_tests/ut_h_edit_dist_features.amlg | 10 +++++----- unit_tests/ut_h_input_validation.amlg | 16 ++++++++-------- unit_tests/ut_h_matrix.amlg | 6 +++--- unit_tests/ut_h_null_null_react.amlg | 2 +- unit_tests/ut_h_one_value_cont.amlg | 2 +- unit_tests/ut_h_ordinal.amlg | 4 ++-- unit_tests/ut_h_post_process.amlg | 10 +++++----- unit_tests/ut_h_preserve_rare_values.amlg | 12 ++++++------ unit_tests/ut_h_react_distance_ratio.amlg | 4 ++-- unit_tests/ut_h_react_into_features.amlg | 16 ++++++++-------- unit_tests/ut_h_rebuild_recommendations.amlg | 4 ++-- unit_tests/ut_h_rounding.amlg | 4 ++-- unit_tests/ut_h_scale.amlg | 4 ++-- unit_tests/ut_h_scale_ablation.amlg | 4 ++-- unit_tests/ut_h_shared_deviations.amlg | 4 ++-- unit_tests/ut_h_shared_deviations_series.amlg | 8 ++++---- unit_tests/ut_h_string_ordinals.amlg | 4 ++-- unit_tests/ut_h_synchronous_cases.amlg | 4 ++-- unit_tests/ut_h_time_format.amlg | 6 +++--- unit_tests/ut_h_time_series.amlg | 8 ++++---- unit_tests/ut_h_time_series_datetime.amlg | 4 ++-- unit_tests/ut_h_time_series_gravity.amlg | 4 ++-- unit_tests/ut_h_time_series_stock.amlg | 16 ++++++++-------- unit_tests/ut_h_unique_ids.amlg | 4 ++-- unit_tests/ut_h_value_contributions.amlg | 8 ++++---- unit_tests/ut_h_value_evaluation_code.amlg | 2 +- 41 files changed, 139 insertions(+), 134 deletions(-) diff --git a/module/attributes.amlg b/module/attributes.amlg index 698736033..f15b5b8b6 100644 --- a/module/attributes.amlg +++ b/module/attributes.amlg @@ -698,6 +698,8 @@ !sharedDeviationsPrimaryFeatures (get shared_deviations_map "shared_deviations_primary_features") !sharedDeviationsNonPrimaryFeatures (get shared_deviations_map "shared_deviations_non_primary_features") + ;;;TODO: may need to make a pass on unit tests to ensure they set feature attributes as continuous for all features as needed, + ;and/or modify train and react_into_features so that anytime there are continuous features that have been added, they are also added to this assoc !continuousMap (zip (indices continuous_map)) )) (call !UpdateDefinedFeatures) @@ -706,12 +708,15 @@ (call !SetNominalFeatures (assoc nominal_features nominals)) (call !SetCyclicFeatures (assoc feature_attributes cyclics_map)) - ;;;TODO: add logic to parse ratio feature attribute, for now set all non-ordinal and non-cyclic continuous as ratio (declare (assoc - ratio_features_map (remove (zip !trainedFeatures) (append nominals ordinals)) + ratio_features_map + (filter + (lambda (= "continuous_ratio" (get (current_value) "type"))) + continuous_map + ) )) (if (size ratio_features_map) - (assign_to_entities (assoc !ratioMeasureMap ratio_features_map )) + (assign_to_entities (assoc !ratioMeasureMap (zip (indices ratio_features_map)) )) ) ;overwrite computed deviations with user specified ones if user specified ones were larger diff --git a/performance_tests/adult_test.amlg b/performance_tests/adult_test.amlg index 3dfe162dc..02085613f 100644 --- a/performance_tests/adult_test.amlg +++ b/performance_tests/adult_test.amlg @@ -33,19 +33,19 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "age" (assoc "type" "continuous") + "age" (assoc "type" "continuous_interval") "workclass" (assoc "type" "nominal" "data_type" "number") - "fnlwgt" (assoc "type" "continuous") + "fnlwgt" (assoc "type" "continuous_interval") "education" (assoc "type" "ordinal" "data_type" "number") - "education-num" (assoc "type" "continuous") + "education-num" (assoc "type" "continuous_interval") "marital-status" (assoc "type" "nominal" "data_type" "number") "occupation" (assoc "type" "nominal" "data_type" "number") "relationship" (assoc "type" "nominal" "data_type" "number") "race" (assoc "type" "nominal" "data_type" "number") "sex" (assoc "type" "nominal" "data_type" "number") - "capital-gain" (assoc "type" "continuous") - "capital-loss" (assoc "type" "continuous") - "hours-per-week" (assoc "type" "continuous") + "capital-gain" (assoc "type" "continuous_ratio") + "capital-loss" (assoc "type" "continuous_interval") + "hours-per-week" (assoc "type" "continuous_interval") "native-country" (assoc "type" "nominal" "data_type" "number") "target" (assoc "type" "nominal" "data_type" "number") ) diff --git a/performance_tests/asteroid_test.amlg b/performance_tests/asteroid_test.amlg index 2909fa03d..d3bddd68a 100644 --- a/performance_tests/asteroid_test.amlg +++ b/performance_tests/asteroid_test.amlg @@ -7,7 +7,7 @@ {escape_resource_name .false escape_contained_resource_names .false} ) (set_entity_permissions "howso" .true) - (call_entity "howso" "initialize" (assoc trainee_id "model" filepath "../")) + (call_entity "howso" "initialize" (assoc trainee_id "model" filepath "../" print_progress .true)) (declare (assoc test_start (system_time))) @@ -15,7 +15,7 @@ ;set to true to see full output verbose .false ;set to null to do full dataset - submodel_size .null; 100000 + submodel_size .null ;set to true to use case weights in analyze use_case_weights .false diff --git a/performance_tests/online_retail_test.amlg b/performance_tests/online_retail_test.amlg index acc61ed10..cf22f3986 100644 --- a/performance_tests/online_retail_test.amlg +++ b/performance_tests/online_retail_test.amlg @@ -37,9 +37,9 @@ "Description" (assoc "type" "nominal") "CustomerID" (assoc "type" "nominal") "Country" (assoc "type" "nominal") - "Quantity" (assoc "type" "continuous") - "UnitPrice" (assoc "type" "continuous") - "InvoiceDate" (assoc "type" "continuous" "date_time_format" "%m/%d/%Y %H:%M") + "Quantity" (assoc "type" "continuous_interval") + "UnitPrice" (assoc "type" "continuous_interval") + "InvoiceDate" (assoc "type" "continuous_interval" "date_time_format" "%m/%d/%Y %H:%M") ) )) diff --git a/performance_tests/time_series_test.amlg b/performance_tests/time_series_test.amlg index 0cc471703..669654157 100644 --- a/performance_tests/time_series_test.amlg +++ b/performance_tests/time_series_test.amlg @@ -37,7 +37,7 @@ "id" (assoc "type" "nominal" "id_feature" .true) "x" (assoc - type "continuous" + type "continuous_interval" time_series { type "rate" @@ -46,7 +46,7 @@ ) "time" (assoc - type "continuous" + type "continuous_interval" date_time_format "%Y-%m-%d" original_type {"data_type" "datetime"} time_series {"time_feature" .true "type" "delta"} diff --git a/unit_tests/ut_h_ablate_ts.amlg b/unit_tests/ut_h_ablate_ts.amlg index b9edd2a61..b1c37ab35 100644 --- a/unit_tests/ut_h_ablate_ts.amlg +++ b/unit_tests/ut_h_ablate_ts.amlg @@ -12,7 +12,7 @@ { "index" { - "type" "continuous" + "type" "continuous_interval" "data_type" "number" "decimal_places" 0 "bounds" {"allow_null" .false} diff --git a/unit_tests/ut_h_analyze.amlg b/unit_tests/ut_h_analyze.amlg index 40a995469..d93ab78c5 100644 --- a/unit_tests/ut_h_analyze.amlg +++ b/unit_tests/ut_h_analyze.amlg @@ -208,7 +208,7 @@ feature_name "TEST" default_value 2 condition_session "unit_test" - feature_attributes (assoc "type" "continuous") + feature_attributes (assoc "type" "continuous_interval") )) )) (print "test feature is NOT added and error is returned: ") diff --git a/unit_tests/ut_h_box_conviction.amlg b/unit_tests/ut_h_box_conviction.amlg index aaa4c0f75..748e898ac 100644 --- a/unit_tests/ut_h_box_conviction.amlg +++ b/unit_tests/ut_h_box_conviction.amlg @@ -123,7 +123,7 @@ ;VERIFY CYCLIC CONVICTIONS (call_entity "howso" "set_feature_attributes" (assoc - feature_attributes (assoc "x" (assoc "type" "continuous" "cycle_length" 3.5) ) + feature_attributes (assoc "x" (assoc "type" "continuous_interval" "cycle_length" 3.5) ) )) (assign (assoc @@ -148,7 +148,7 @@ (call assert_same (assoc obs (get result (list "distance_contribution" 0) ) exp 0.5)) (call_entity "howso" "set_feature_attributes" (assoc - feature_attributes (assoc "x" (assoc "type" "continuous" "cycle_length" 6) ) + feature_attributes (assoc "x" (assoc "type" "continuous_interval" "cycle_length" 6) ) )) (print "\nsame point at 3,0 with large cycle to not affect distances: ") diff --git a/unit_tests/ut_h_constraints.amlg b/unit_tests/ut_h_constraints.amlg index 7f62c912f..ea4e3f162 100644 --- a/unit_tests/ut_h_constraints.amlg +++ b/unit_tests/ut_h_constraints.amlg @@ -14,7 +14,7 @@ (assoc "x" (assoc - "type" "continuous" + "type" "continuous_interval" "bounds" (assoc "min" 0 @@ -24,7 +24,7 @@ ) ) "z" - (assoc "type" "continuous" ) + (assoc "type" "continuous_interval" ) "w" (assoc "type" "nominal" ) ) )) @@ -43,17 +43,17 @@ (assoc "x" (assoc - "type" "continuous" + "type" "continuous_interval" "bounds" { "min" 0 "max" 300 "allow_null" .false } ) "y" (assoc - "type" "continuous" + "type" "continuous_interval" "bounds" { "min" 0 "max" 300 "allow_null" .false } ) "z" (assoc - "type" "continuous" + "type" "continuous_interval" "bounds" (assoc "min" 0 @@ -203,12 +203,12 @@ (assoc "x" (assoc - "type" "continuous" + "type" "continuous_interval" "bounds" { "min" 0 "max" 300 "allow_null" .false } ) "y" (assoc - "type" "continuous" + "type" "continuous_interval" "bounds" { "min" 0 "max" 300 "allow_null" .false } ) "z" diff --git a/unit_tests/ut_h_cyclic_dataset.amlg b/unit_tests/ut_h_cyclic_dataset.amlg index a483b481f..c4f187dc7 100644 --- a/unit_tests/ut_h_cyclic_dataset.amlg +++ b/unit_tests/ut_h_cyclic_dataset.amlg @@ -47,9 +47,9 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "x" (assoc "type" "continuous") - "day" (assoc "type" "continuous" "cycle_length" 7) - "degree" (assoc "type" "continuous" "cycle_length" 360) + "x" (assoc "type" "continuous_interval") + "day" (assoc "type" "continuous_interval" "cycle_length" 7) + "degree" (assoc "type" "continuous_interval" "cycle_length" 360) ) )) @@ -139,16 +139,16 @@ obs result exp (assoc - "x" (assoc "type" "continuous" ) + "x" (assoc "type" "continuous_interval" ) "day" (assoc "cycle_length" 7 - "type" "continuous" + "type" "continuous_interval" ) "degree" (assoc "cycle_length" 360 - "type" "continuous" + "type" "continuous_interval" ) ) )) @@ -286,7 +286,7 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "x" (assoc "type" "continuous" ) + "x" (assoc "type" "continuous_interval" ) "day" (assoc "cycle_length" 7 diff --git a/unit_tests/ut_h_datetimeformat.amlg b/unit_tests/ut_h_datetimeformat.amlg index 78d40ac5b..34d8a1705 100644 --- a/unit_tests/ut_h_datetimeformat.amlg +++ b/unit_tests/ut_h_datetimeformat.amlg @@ -43,11 +43,11 @@ feature_attributes (assoc "id" (assoc "type" "nominal") - "val" (assoc "type" "continuous" "decimal_places" 2) + "val" (assoc "type" "continuous_interval" "decimal_places" 2) "nom" (assoc "type" "nominal") "datetime" (assoc - "type" "continuous" + "type" "continuous_interval" "date_time_format" "%Y-%m-%d %A %H.%M.%S" ;should be overwritten with 'formatted_date_time' "data_type" "string" diff --git a/unit_tests/ut_h_density_conviction.amlg b/unit_tests/ut_h_density_conviction.amlg index b0e4ee9a9..c686ecbee 100644 --- a/unit_tests/ut_h_density_conviction.amlg +++ b/unit_tests/ut_h_density_conviction.amlg @@ -56,8 +56,8 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "x" (assoc "type" "continuous") - "y" (assoc "type" "continuous") + "x" (assoc "type" "continuous_interval") + "y" (assoc "type" "continuous_interval") "color" (assoc "type" "nominal" "data_type" "string") ) )) diff --git a/unit_tests/ut_h_dependent_features.amlg b/unit_tests/ut_h_dependent_features.amlg index 7ca1b001a..485b5633c 100644 --- a/unit_tests/ut_h_dependent_features.amlg +++ b/unit_tests/ut_h_dependent_features.amlg @@ -65,7 +65,7 @@ ) "measure_value" (assoc - "type" "continuous" + "type" "continuous_interval" "bounds" (assoc "min" 0 "max" 220) "null_is_dependent" .true ) @@ -476,10 +476,10 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "random" { "type" "continuous" "bounds" { "min" 0 "max" 100000 "allow_null" .true}} + "random" { "type" "continuous_interval" "bounds" { "min" 0 "max" 100000 "allow_null" .true}} "type" { "type" "nominal" } "value" { - "type" "continuous" + "type" "continuous_interval" "bounds" { "min" 0 "max" 100000 "allow_null" .true} "dependent_features" ["type"] "null_is_dependent" .true diff --git a/unit_tests/ut_h_derive_custom.amlg b/unit_tests/ut_h_derive_custom.amlg index 3486853f9..ca16c634d 100644 --- a/unit_tests/ut_h_derive_custom.amlg +++ b/unit_tests/ut_h_derive_custom.amlg @@ -43,7 +43,7 @@ (assoc ".custom2" { - "type" "continuous" + "type" "continuous_interval" ;bad opcode "derived_feature_code" "(bad opcode)" ;bad feature, mismatching parenthesis @@ -74,12 +74,12 @@ (assoc "sender" (assoc "type" "nominal" ) "reciever" (assoc "type" "nominal" ) - "time" (assoc "type" "continuous" "bounds" (assoc "min" 900 "max" 4000 "allow_null" .true) "derived_feature_code" "(* (call value {feature \"mult\"}) (call value {feature \"number\"}))") - "number" (assoc "type" "continuous" "decimal_places" 0 "bounds" (assoc "min" 5.0 "max" 50 "allow_null" .true)) - "mult" (assoc "type" "continuous" "derived_feature_code" "(* 10 (call value {feature \"number\"}))") + "time" (assoc "type" "continuous_interval" "bounds" (assoc "min" 900 "max" 4000 "allow_null" .true) "derived_feature_code" "(* (call value {feature \"mult\"}) (call value {feature \"number\"}))") + "number" (assoc "type" "continuous_interval" "decimal_places" 0 "bounds" (assoc "min" 5.0 "max" 50 "allow_null" .true)) + "mult" (assoc "type" "continuous_interval" "derived_feature_code" "(* 10 (call value {feature \"number\"}))") ".time_delta" (assoc - "type" "continuous" + "type" "continuous_interval" "auto_derive_on_train" (assoc "derive_type" "custom" @@ -92,7 +92,7 @@ ) ".custom2" (assoc - "type" "continuous" + "type" "continuous_interval" "auto_derive_on_train" (assoc "derive_type" "custom" diff --git a/unit_tests/ut_h_derive_start_end.amlg b/unit_tests/ut_h_derive_start_end.amlg index a7d4ecf6e..8fe1a6923 100644 --- a/unit_tests/ut_h_derive_start_end.amlg +++ b/unit_tests/ut_h_derive_start_end.amlg @@ -30,13 +30,13 @@ "name" (assoc "type" "nominal" "id_feature" .true ) "time" (assoc - "type" "continuous" + "type" "continuous_interval" "bounds" (assoc "min" 900 "max" 4000 "allow_null" .true) "time_series" (assoc "time_feature" .true "type" "delta") ) "pts" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "bounds" (assoc "min" 10.0 "max" 40 "allow_null" .true) "time_series" (assoc "type" "delta") diff --git a/unit_tests/ut_h_edit_dist_features.amlg b/unit_tests/ut_h_edit_dist_features.amlg index c87c449fe..fd221736a 100644 --- a/unit_tests/ut_h_edit_dist_features.amlg +++ b/unit_tests/ut_h_edit_dist_features.amlg @@ -8,11 +8,11 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "x" (assoc "type" "continuous" ) - "string" (assoc "type" "continuous" "data_type" "string_mixable") - "amalgam" (assoc "type" "continuous" "data_type" "amalgam") - "json" (assoc "type" "continuous" "data_type" "json") - "yaml" (assoc "type" "continuous" "data_type" "yaml") + "x" (assoc "type" "continuous_interval" ) + "string" (assoc "type" "continuous_interval" "data_type" "string_mixable") + "amalgam" (assoc "type" "continuous_interval" "data_type" "amalgam") + "json" (assoc "type" "continuous_interval" "data_type" "json") + "yaml" (assoc "type" "continuous_interval" "data_type" "yaml") ) )) diff --git a/unit_tests/ut_h_input_validation.amlg b/unit_tests/ut_h_input_validation.amlg index de9f2bace..163893ed0 100644 --- a/unit_tests/ut_h_input_validation.amlg +++ b/unit_tests/ut_h_input_validation.amlg @@ -24,13 +24,13 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "x" (assoc "type" "continuous") - "y" (assoc "type" "continuous") - ".none" (assoc "type" "continuous") + "x" (assoc "type" "continuous_interval") + "y" (assoc "type" "continuous_interval") + ".none" (assoc "type" "continuous_interval") ; INVALID FEATURES - "#case_weight" (assoc "type" "continuous") - "^session" (assoc "type" "continuous") + "#case_weight" (assoc "type" "continuous_interval") + "^session" (assoc "type" "continuous_interval") ) )) (declare (assoc result (call_entity "howso" "get_feature_attributes"))) @@ -38,7 +38,7 @@ (print "Did not store attributes for invalid feature: ") (call assert_same (assoc obs (get result (list 1 "payload")) - exp (assoc "y" (assoc "type" "continuous") ".none" (assoc "type" "continuous" ) "x" (assoc "type" "continuous")) + exp (assoc "y" (assoc "type" "continuous_interval") ".none" (assoc "type" "continuous_interval" ) "x" (assoc "type" "continuous_interval")) )) (assign (assoc @@ -212,8 +212,8 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "x" (assoc "type" "continuous") - "y" (assoc "type" "continuous") + "x" (assoc "type" "continuous_interval") + "y" (assoc "type" "continuous_interval") ) )) diff --git a/unit_tests/ut_h_matrix.amlg b/unit_tests/ut_h_matrix.amlg index e5d6a87cf..8c2a5bbc6 100644 --- a/unit_tests/ut_h_matrix.amlg +++ b/unit_tests/ut_h_matrix.amlg @@ -6,9 +6,9 @@ feature_attributes (assoc "a" { "type" "nominal" } - "b" { "type" "continuous"} - "c" { "type" "continuous"} - "d" { "type" "continuous"} + "b" { "type" "continuous_interval"} + "c" { "type" "continuous_interval"} + "d" { "type" "continuous_interval"} ) )) diff --git a/unit_tests/ut_h_null_null_react.amlg b/unit_tests/ut_h_null_null_react.amlg index 8b504bffe..a09dea803 100644 --- a/unit_tests/ut_h_null_null_react.amlg +++ b/unit_tests/ut_h_null_null_react.amlg @@ -159,7 +159,7 @@ feature_attributes (assoc "C" (assoc "type" "nominal") - "A" (assoc "type" "continuous" "bounds" (assoc "min" 0 "max" 120)) + "A" (assoc "type" "continuous_interval" "bounds" (assoc "min" 0 "max" 120)) ) )) diff --git a/unit_tests/ut_h_one_value_cont.amlg b/unit_tests/ut_h_one_value_cont.amlg index a980b261b..c248fcdd9 100644 --- a/unit_tests/ut_h_one_value_cont.amlg +++ b/unit_tests/ut_h_one_value_cont.amlg @@ -29,7 +29,7 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - up_b {"type" "continuous" "data_type" "number" "bounds" {"max" 2 "min" 0}} + up_b {"type" "continuous_interval" "data_type" "number" "bounds" {"max" 2 "min" 0}} ;rest are just continuous values so no need to specify ) )) diff --git a/unit_tests/ut_h_ordinal.amlg b/unit_tests/ut_h_ordinal.amlg index 72ad2b9ba..1564d09e6 100644 --- a/unit_tests/ut_h_ordinal.amlg +++ b/unit_tests/ut_h_ordinal.amlg @@ -28,7 +28,7 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "k" (assoc "type" "continuous") + "k" (assoc "type" "continuous_interval") "y" (assoc "type" "ordinal" "data_type" "number") ) )) @@ -43,7 +43,7 @@ obs result exp (assoc - "k" (assoc "type" "continuous") + "k" (assoc "type" "continuous_interval") "y" (assoc "type" "ordinal" "data_type" "number") ) )) diff --git a/unit_tests/ut_h_post_process.amlg b/unit_tests/ut_h_post_process.amlg index 80a92ddcf..8f0172875 100644 --- a/unit_tests/ut_h_post_process.amlg +++ b/unit_tests/ut_h_post_process.amlg @@ -28,7 +28,7 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "petal_length" (assoc "type" "continuous" "bounds" (assoc min 0 max 10.0) "post_process" "(+ (call value {feature \"petal_len\" }) 40.0)") + "petal_length" (assoc "type" "continuous_interval" "bounds" (assoc min 0 max 10.0) "post_process" "(+ (call value {feature \"petal_len\" }) 40.0)") "target" (assoc "type" "nominal" @@ -50,12 +50,12 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "petal_length" (assoc "type" "continuous" "bounds" (assoc max 10.0) "post_process" "(+ (call value {feature \"petal_length\" }) 40.0)") - "sepal_length" (assoc "type" "continuous") - "sepal_width" (assoc "type" "continuous") + "petal_length" (assoc "type" "continuous_interval" "bounds" (assoc max 10.0) "post_process" "(+ (call value {feature \"petal_length\" }) 40.0)") + "sepal_length" (assoc "type" "continuous_interval") + "sepal_width" (assoc "type" "continuous_interval") "petal_width" (assoc - "type" "continuous" + "type" "continuous_interval" ; Assigning a post_process that stores digits 1,3,2,0 in base 4 between digits 2 and 5 (indexed from most significant digit) ; into the value of petal_width after synth. "post_process" "(set_digits (call value {feature \"petal_width\"}) 4 (list 1 3 2 0) 2 5 .false)" diff --git a/unit_tests/ut_h_preserve_rare_values.amlg b/unit_tests/ut_h_preserve_rare_values.amlg index 2715d9ce9..fbdf9b1b2 100644 --- a/unit_tests/ut_h_preserve_rare_values.amlg +++ b/unit_tests/ut_h_preserve_rare_values.amlg @@ -16,18 +16,18 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "age" (assoc "type" "continuous") - "fnlwgt" (assoc "type" "continuous") + "age" (assoc "type" "continuous_interval") + "fnlwgt" (assoc "type" "continuous_interval") "education" (assoc "type" "ordinal" "data_type" "number") - "education-num" (assoc "type" "continuous") + "education-num" (assoc "type" "continuous_interval") "marital-status" (assoc "type" "nominal" "data_type" "number") "occupation" (assoc "type" "nominal" "data_type" "number") "relationship" (assoc "type" "nominal" "data_type" "number") "race" (assoc "type" "nominal" "data_type" "number") "sex" (assoc "type" "nominal" "data_type" "number") - "capital-gain" (assoc "type" "continuous") - "capital-loss" (assoc "type" "continuous") - "hours-per-week" (assoc "type" "continuous") + "capital-gain" (assoc "type" "continuous_interval") + "capital-loss" (assoc "type" "continuous_interval") + "hours-per-week" (assoc "type" "continuous_interval") "native-country" (assoc "type" "nominal" "data_type" "number") "target" (assoc "type" "nominal" "data_type" "number") "workclass" diff --git a/unit_tests/ut_h_react_distance_ratio.amlg b/unit_tests/ut_h_react_distance_ratio.amlg index e45beeccc..51e5f7dcb 100644 --- a/unit_tests/ut_h_react_distance_ratio.amlg +++ b/unit_tests/ut_h_react_distance_ratio.amlg @@ -16,8 +16,8 @@ context_features (list "con_A" "con_B" "nom_A") features (assoc - "con_A" (assoc type "continuous") - "con_B" (assoc type "continuous") + "con_A" (assoc type "continuous_interval") + "con_B" (assoc type "continuous_interval") "nom_A" (assoc type "nominal" data_type "string") ) default_hyperparameter_map diff --git a/unit_tests/ut_h_react_into_features.amlg b/unit_tests/ut_h_react_into_features.amlg index 758679737..abf2eb864 100644 --- a/unit_tests/ut_h_react_into_features.amlg +++ b/unit_tests/ut_h_react_into_features.amlg @@ -113,14 +113,14 @@ exp { color {type "nominal"} - distance_contribution { bounds {max .infinity min 0} data_type "number" type "continuous" } - dc { bounds {max .infinity min 0} data_type "number" type "continuous" } - fc { bounds {max .infinity min 0} data_type "number" type "continuous" } - if { bounds {max .infinity min 0} data_type "number" type "continuous" } - sc { bounds {max .infinity min 0} data_type "number" type "continuous" } - similarity_conviction { bounds {max .infinity min 0} data_type "number" type "continuous" } - x { bounds {allow_null .true} type "continuous" } - y { bounds {allow_null .true} type "continuous" } + distance_contribution { bounds {max .infinity min 0} data_type "number" type "continuous_interval" } + dc { bounds {max .infinity min 0} data_type "number" type "continuous_interval" } + fc { bounds {max .infinity min 0} data_type "number" type "continuous_interval" } + if { bounds {max .infinity min 0} data_type "number" type "continuous_interval" } + sc { bounds {max .infinity min 0} data_type "number" type "continuous_interval" } + similarity_conviction { bounds {max .infinity min 0} data_type "number" type "continuous_interval" } + x { bounds {allow_null .true} type "continuous_interval" } + y { bounds {allow_null .true} type "continuous_interval" } } )) diff --git a/unit_tests/ut_h_rebuild_recommendations.amlg b/unit_tests/ut_h_rebuild_recommendations.amlg index 56577c1c2..d4ab2674a 100644 --- a/unit_tests/ut_h_rebuild_recommendations.amlg +++ b/unit_tests/ut_h_rebuild_recommendations.amlg @@ -7,8 +7,8 @@ result .null attributes (assoc - "x" (assoc "type" "continuous") - "y" (assoc "type" "continuous") + "x" (assoc "type" "continuous_interval") + "y" (assoc "type" "continuous_interval") "label" (assoc "type" "nominal") ) )) diff --git a/unit_tests/ut_h_rounding.amlg b/unit_tests/ut_h_rounding.amlg index 09466547b..69a3f7ab7 100644 --- a/unit_tests/ut_h_rounding.amlg +++ b/unit_tests/ut_h_rounding.amlg @@ -44,7 +44,7 @@ feature_attributes (assoc "id" (assoc "type" "nominal") - "val" (assoc "type" "continuous" "significant_digits" 3) + "val" (assoc "type" "continuous_interval" "significant_digits" 3) "nom" (assoc "type" "nominal") ) )) @@ -74,7 +74,7 @@ feature_attributes (assoc "id" (assoc "type" "nominal") - "val" (assoc "type" "continuous" "decimal_places" 2) + "val" (assoc "type" "continuous_interval" "decimal_places" 2) "nom" (assoc "type" "nominal") ) )) diff --git a/unit_tests/ut_h_scale.amlg b/unit_tests/ut_h_scale.amlg index ff9903d91..6bca6832b 100644 --- a/unit_tests/ut_h_scale.amlg +++ b/unit_tests/ut_h_scale.amlg @@ -5,8 +5,8 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "x" (assoc "type" "continuous") - "y" (assoc "type" "continuous") + "x" (assoc "type" "continuous_interval") + "y" (assoc "type" "continuous_interval") ) )) diff --git a/unit_tests/ut_h_scale_ablation.amlg b/unit_tests/ut_h_scale_ablation.amlg index 233304988..e11ad60ea 100644 --- a/unit_tests/ut_h_scale_ablation.amlg +++ b/unit_tests/ut_h_scale_ablation.amlg @@ -27,8 +27,8 @@ (print "set_feature_attributes\n") (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "x" (assoc "type" "continuous") - "y" (assoc "type" "continuous") + "x" (assoc "type" "continuous_interval") + "y" (assoc "type" "continuous_interval") ) )) diff --git a/unit_tests/ut_h_shared_deviations.amlg b/unit_tests/ut_h_shared_deviations.amlg index fb601bad6..e27fd5138 100644 --- a/unit_tests/ut_h_shared_deviations.amlg +++ b/unit_tests/ut_h_shared_deviations.amlg @@ -77,8 +77,8 @@ "fruit" (assoc "type" "nominal") "color1" (assoc "type" "nominal") "color2" (assoc "type" "nominal" "unique" .true) - "weight" (assoc "type" "continuous" "shared_deviations" (list "height" "tart")) - "length" (assoc "type" "continuous" "shared_deviations" (list "sweet")) + "weight" (assoc "type" "continuous_interval" "shared_deviations" (list "height" "tart")) + "length" (assoc "type" "continuous_interval" "shared_deviations" (list "sweet")) "size" (assoc "type" "nominal" "shared_deviations" (list "fruit" "size")) ) )) diff --git a/unit_tests/ut_h_shared_deviations_series.amlg b/unit_tests/ut_h_shared_deviations_series.amlg index 3c1123404..3bd817c4f 100644 --- a/unit_tests/ut_h_shared_deviations_series.amlg +++ b/unit_tests/ut_h_shared_deviations_series.amlg @@ -61,7 +61,7 @@ ) "time" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "time_series" (assoc "time_feature" .true "type" "delta" "universal" .false) "bounds" @@ -73,7 +73,7 @@ ) "value" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "time_series" (assoc "type" "delta" "num_lags" 2) "bounds" @@ -131,7 +131,7 @@ ) "time" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "time_series" (assoc "time_feature" .true "type" "delta" "universal" .false) "bounds" @@ -143,7 +143,7 @@ ) "value" (assoc - "type" "continuous" + "type" "continuous_interval" "shared_deviations" .false "decimal_places" 0 "time_series" (assoc "type" "delta" "num_lags" 2) diff --git a/unit_tests/ut_h_string_ordinals.amlg b/unit_tests/ut_h_string_ordinals.amlg index 76c8205f8..001414135 100644 --- a/unit_tests/ut_h_string_ordinals.amlg +++ b/unit_tests/ut_h_string_ordinals.amlg @@ -47,11 +47,11 @@ (assoc "id" (assoc "type" "nominal" "data_type" "number" "unique" .true) "value" (assoc "type" "ordinal" "bounds" (assoc "allowed" (list "one" "two" "three" "four" "five"))) - "count" (assoc "type" "continuous") + "count" (assoc "type" "continuous_interval") "num_nom" (assoc "type" "nominal" "data_type" "number") "bool" (assoc "type" "nominal") ;stored as stringified booleans, ie "true" and "false" "bool_nom" (assoc "type" "nominal" "data_type" "boolean") - "inactive" (assoc "type" "continuous") + "inactive" (assoc "type" "continuous_interval") ) )) diff --git a/unit_tests/ut_h_synchronous_cases.amlg b/unit_tests/ut_h_synchronous_cases.amlg index 387e70c13..39e2fa2ad 100644 --- a/unit_tests/ut_h_synchronous_cases.amlg +++ b/unit_tests/ut_h_synchronous_cases.amlg @@ -23,7 +23,7 @@ "time" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "time_series" (assoc "time_feature" .true "type" "delta" "universal" .false) "bounds" @@ -40,7 +40,7 @@ ) "value" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "time_series" (assoc "type" "rate" "num_lags" 2) "bounds" diff --git a/unit_tests/ut_h_time_format.amlg b/unit_tests/ut_h_time_format.amlg index 4222d105c..15a225f02 100644 --- a/unit_tests/ut_h_time_format.amlg +++ b/unit_tests/ut_h_time_format.amlg @@ -30,12 +30,12 @@ (call_entity "howso" "set_feature_attributes" (assoc feature_attributes (assoc - "id" (assoc "type" "continuous") - "val" (assoc "type" "continuous" ) + "id" (assoc "type" "continuous_interval") + "val" (assoc "type" "continuous_interval" ) "nom" (assoc "type" "nominal") "time" (assoc - "type" "continuous" + "type" "continuous_interval" ;should be set automatically ;"date_time_format" "%H.%M.%S" "data_type" "formatted_time" diff --git a/unit_tests/ut_h_time_series.amlg b/unit_tests/ut_h_time_series.amlg index 9cd06a44c..af5a114d1 100644 --- a/unit_tests/ut_h_time_series.amlg +++ b/unit_tests/ut_h_time_series.amlg @@ -124,7 +124,7 @@ "f1" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 4 "original_type" (assoc "data_type" "numeric" "size" 8) "bounds" (assoc "min" 20.085536923187668 "max" 8103.083927575384) @@ -138,7 +138,7 @@ "f2" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 4 "original_type" (assoc "data_type" "numeric" "size" 8) "bounds" (assoc "min" 20.085536923187668 "max" 8103.083927575384) @@ -153,7 +153,7 @@ "f3" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 4 "original_type" (assoc "data_type" "numeric" "size" 8) "bounds" (assoc "min" 20.085536923187668 "max" 8103.083927575384) @@ -168,7 +168,7 @@ "date" (assoc - "type" "continuous" + "type" "continuous_interval" "date_time_format" "%Y-%m-%d" "original_type" (assoc "data_type" "datetime") "bounds" (assoc "min" "1985-05-17" "max" "2083-08-08" "allow_null" .false) diff --git a/unit_tests/ut_h_time_series_datetime.amlg b/unit_tests/ut_h_time_series_datetime.amlg index aa2c92964..cf414ac42 100644 --- a/unit_tests/ut_h_time_series_datetime.amlg +++ b/unit_tests/ut_h_time_series_datetime.amlg @@ -52,7 +52,7 @@ "time" (assoc - "type" "continuous" + "type" "continuous_interval" "time_series" (assoc "time_feature" .true "type" "delta") ;should auto-set "date_time_format" to "%Y-%m-%dT%H:%M:%S" "data_type" "formatted_date_time" @@ -71,7 +71,7 @@ "owner" (assoc "type" "nominal") "value" (assoc - "type" "continuous" + "type" "continuous_interval" "time_series" (assoc "type" "delta") "bounds" (assoc "allow_null" .true diff --git a/unit_tests/ut_h_time_series_gravity.amlg b/unit_tests/ut_h_time_series_gravity.amlg index 9268c6e04..68f9b880a 100644 --- a/unit_tests/ut_h_time_series_gravity.amlg +++ b/unit_tests/ut_h_time_series_gravity.amlg @@ -23,7 +23,7 @@ ) "time" (assoc - "type" "continuous" + "type" "continuous_interval" "time_series" (assoc "time_feature" .true "type" "delta" "universal" .false) "bounds" (assoc @@ -34,7 +34,7 @@ ) "y" (assoc - "type" "continuous" + "type" "continuous_interval" "time_series" (assoc "type" "covariate" "num_lags" 2) "bounds" (assoc diff --git a/unit_tests/ut_h_time_series_stock.amlg b/unit_tests/ut_h_time_series_stock.amlg index 7f8ca688d..5594bf4d0 100644 --- a/unit_tests/ut_h_time_series_stock.amlg +++ b/unit_tests/ut_h_time_series_stock.amlg @@ -65,7 +65,7 @@ "time" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "time_series" (assoc "time_feature" .true "type" "delta" "universal" .false) "bounds" @@ -77,7 +77,7 @@ ) "value" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "time_series" (assoc "type" "delta" "num_lags" 2) "bounds" @@ -89,7 +89,7 @@ ) "over106" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "auto_derive_on_train" (assoc @@ -410,21 +410,21 @@ "time" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "time_series" (assoc "time_feature" .true "type" "delta") "bounds" (assoc "allow_null" .false "min" 1 "max" 30) ) "value" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "time_series" (assoc "type" "delta" "num_lags" 2) "bounds" (assoc "allow_null" .true "min" 60 "max" 125) ) "over106" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "auto_derive_on_train" (assoc @@ -602,7 +602,7 @@ "time" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "time_series" (assoc @@ -614,7 +614,7 @@ ) "value" (assoc - "type" "continuous" + "type" "continuous_interval" "decimal_places" 0 "time_series" (assoc "type" "delta" "num_lags" 2) "bounds" (assoc "allow_null" .true "min" 60 "max" 125) diff --git a/unit_tests/ut_h_unique_ids.amlg b/unit_tests/ut_h_unique_ids.amlg index 61e8ddedb..2f96fb367 100644 --- a/unit_tests/ut_h_unique_ids.amlg +++ b/unit_tests/ut_h_unique_ids.amlg @@ -46,8 +46,8 @@ (assoc "n_id" (assoc "type" "nominal" "data_type" "number" "unique" .true) "s_id" (assoc "type" "nominal" "unique" .true) - "value" (assoc "type" "continuous") - "count" (assoc "type" "continuous") + "value" (assoc "type" "continuous_interval") + "count" (assoc "type" "continuous_interval") ) )) diff --git a/unit_tests/ut_h_value_contributions.amlg b/unit_tests/ut_h_value_contributions.amlg index 56d5b8770..061c7c768 100644 --- a/unit_tests/ut_h_value_contributions.amlg +++ b/unit_tests/ut_h_value_contributions.amlg @@ -19,10 +19,10 @@ "name" (assoc "type" "nominal" "data_type" "string") "subject" (assoc "type" "nominal" "data_type" "string") "is_multiple_choice" (assoc "type" "nominal" "data_type" "string") - "num_questions" (assoc "type" "continuous") - "minutes" (assoc "type" "continuous") - "study_time" (assoc "type" "continuous") - "score" (assoc "type" "continuous") + "num_questions" (assoc "type" "continuous_interval") + "minutes" (assoc "type" "continuous_interval") + "study_time" (assoc "type" "continuous_interval") + "score" (assoc "type" "continuous_interval") ) )) diff --git a/unit_tests/ut_h_value_evaluation_code.amlg b/unit_tests/ut_h_value_evaluation_code.amlg index f4a707fd0..a685f0a09 100644 --- a/unit_tests/ut_h_value_evaluation_code.amlg +++ b/unit_tests/ut_h_value_evaluation_code.amlg @@ -19,7 +19,7 @@ "target" (assoc "type" "nominal" "data_type" "number") "sepal_width" (assoc - "type" "continuous" + "type" "continuous_interval" "value_evaluation_code" "(call_on_entity .null (pow 2 sepal_width)))" ) ) From 399bc564d88e3ad9817c680d0b166e7762e593ce Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:46:28 -0400 Subject: [PATCH 12/14] cleanup, speed improvements --- module/analysis_weights.amlg | 2 +- module/hyperparameters.amlg | 19 +++++++++---------- module/mda_weight.amlg | 4 ++-- module/react.amlg | 10 ++++------ module/residuals.amlg | 12 ++++++------ module/trainee.amlg | 2 +- 6 files changed, 23 insertions(+), 26 deletions(-) diff --git a/module/analysis_weights.amlg b/module/analysis_weights.amlg index 81d4a9b39..9a8725052 100644 --- a/module/analysis_weights.amlg +++ b/module/analysis_weights.amlg @@ -430,7 +430,7 @@ ;;;TODO: use generalized mean of initial_deviation / case value, for now just use initial deviation / median (/ (get initial_residual_values_map (current_index)) - (compute_on_contained_entities (query_quantile (current_index) 0.5)) + (compute_on_contained_entities (query_quantile (current_index) 0.98)) ) ) !ratioMeasureMap diff --git a/module/hyperparameters.amlg b/module/hyperparameters.amlg index 62cb7b40c..c7390ca37 100644 --- a/module/hyperparameters.amlg +++ b/module/hyperparameters.amlg @@ -124,10 +124,10 @@ (if (~ (assoc) (first (current_value))) (append (list (get (current_value 1) (list 0 "sdm"))) (last (current_value)) ) - (if (contains_index ratios_map (current_index )) - (append (get ratios_map (current_index)) (last (current_value)) ) + ;;;(if (contains_index ratios_map (current_index)) + ;;;(append (get ratios_map (current_index)) (last (current_value)) ) (append (first (current_value)) (last (current_value)) ) - ) + ;;;) ) ;else return the deviation value without the known-unkwnown uncertainties @@ -137,10 +137,10 @@ [(get (current_value 1) [0 "sdm"])] ;if continuous_ratio, store the ratio, else store the deviation - (if (contains_index ratios_map (current_index )) - (get ratios_map (current_index)) + ;;; (if (contains_index ratios_map (current_index)) + ;;; (get ratios_map (current_index)) (first (current_value)) - ) + ;;;) ) ) ) @@ -157,10 +157,10 @@ [(get (current_value 1) "sdm")] ;if continuous_ratio, store the ratio, else store the deviation - (if (contains_index ratios_map (current_index )) - (get ratios_map (current_index)) + ;;; (if (contains_index ratios_map (current_index)) + ;;; (get ratios_map (current_index)) (current_value) - ) + ;;;) ) ) feature_deviations @@ -208,7 +208,6 @@ ;;;TODO: remove this as we shouldn't need to store ratios separately (outside of featureDeviations) (if (size ratios_map) - (assoc "featureDeviationRatios" ratios_map) (assoc) ) diff --git a/module/mda_weight.amlg b/module/mda_weight.amlg index 80604cf96..8250ba8c5 100644 --- a/module/mda_weight.amlg +++ b/module/mda_weight.amlg @@ -103,7 +103,7 @@ updated_deviation (* (max - (abs (current_value 1)) + (abs (get case_values_map (current_index 1))) (get !cachedFeatureHalfMinGapMap (current_index 1)) ) (get feature_deviation_ratios_map (current_index 1)) @@ -116,7 +116,7 @@ ) )) - (keep (filter case_values_map) (indices !ratioMeasureMap)) + !ratioMeasureMap ) ) )) diff --git a/module/react.amlg b/module/react.amlg index 0cf70d9e6..686b8563f 100644 --- a/module/react.amlg +++ b/module/react.amlg @@ -1602,11 +1602,9 @@ ) )) - (if feature_deviations + (if (and feature_deviations !ratioMeasureMap) (let - (assoc - ctx_value_map (remove (zip context_features context_values) (indices !nominalsMap)) - ) + (assoc ctx_value_map (zip context_features context_values) ) ;overwrite continuous deviations with ratio from case value (assign (assoc @@ -1620,7 +1618,7 @@ updated_deviation (* (max - (abs (current_value 1)) + (abs (get ctx_value_map (current_index 1))) (get !cachedFeatureHalfMinGapMap (current_index 1)) ) (get hyperparam_map ["featureDeviationRatios" (current_index 2)]) @@ -1632,7 +1630,7 @@ updated_deviation ) )) - (keep (filter ctx_value_map) (indices !ratioMeasureMap)) + !ratioMeasureMap ) ) )) diff --git a/module/residuals.amlg b/module/residuals.amlg index 1ab3942ac..07660a02c 100644 --- a/module/residuals.amlg +++ b/module/residuals.amlg @@ -689,7 +689,7 @@ ) ) - (if feature_deviations + (if (and feature_deviations !ratioMeasureMap) ;overwrite continuous deviations with ratio from case value (assign (assoc feature_deviations @@ -702,7 +702,7 @@ updated_deviation (* (max - (abs (current_value 1)) + (abs (get case_values_map (current_index 1))) (get !cachedFeatureHalfMinGapMap (current_index 1)) ) (get feature_deviation_ratios_map (current_index 1)) @@ -715,7 +715,7 @@ updated_deviation ) )) - (keep case_values_map (indices !ratioMeasureMap)) + !ratioMeasureMap ) ) )) @@ -1099,7 +1099,7 @@ )) ) - (if feature_deviations + (if (and feature_deviations !ratioMeasureMap) ;overwrite continuous deviations with ratio from case value (assign (assoc feature_deviations @@ -1112,7 +1112,7 @@ updated_deviation (* (max - (abs (current_value 1)) + (abs (get case_values_map (current_index 1))) (get !cachedFeatureHalfMinGapMap (current_index 1)) ) (get feature_deviation_ratios_map (current_index 1)) @@ -1125,7 +1125,7 @@ updated_deviation ) )) - (keep (filter case_values_map) (indices !ratioMeasureMap)) + !ratioMeasureMap ) ) )) diff --git a/module/trainee.amlg b/module/trainee.amlg index 572498969..93df277af 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -281,7 +281,7 @@ ;store class probabilities for each nominal feature; weight feature -> feature -> probability !nominalClassProbabilitiesMap (assoc) - ;map of feature -> query attribute types for use in queries. i.e., "nominal", "continuous_intetrval", specifying 'data_type', etc + ;map of feature -> query attribute types for use in queries. i.e., "nominal", "continuous_interval", specifying 'data_type', etc !queryFeatureAttributeTypeMap (assoc) ;set of ordinal features From aed1608d444c748fce3455924041149b7cd38e1e Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:57:29 -0400 Subject: [PATCH 13/14] improve ratio computation --- module/analysis_weights.amlg | 2 +- module/feature_residuals.amlg | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/module/analysis_weights.amlg b/module/analysis_weights.amlg index 9a8725052..81d4a9b39 100644 --- a/module/analysis_weights.amlg +++ b/module/analysis_weights.amlg @@ -430,7 +430,7 @@ ;;;TODO: use generalized mean of initial_deviation / case value, for now just use initial deviation / median (/ (get initial_residual_values_map (current_index)) - (compute_on_contained_entities (query_quantile (current_index) 0.98)) + (compute_on_contained_entities (query_quantile (current_index) 0.5)) ) ) !ratioMeasureMap diff --git a/module/feature_residuals.amlg b/module/feature_residuals.amlg index bc998b542..16fe0bdcc 100644 --- a/module/feature_residuals.amlg +++ b/module/feature_residuals.amlg @@ -809,6 +809,7 @@ (lambda (retrieve_from_entity (current_value) cont_feature_map) ) case_ids ) + feature_index_map (zip features (indices features)) )) (map @@ -816,6 +817,12 @@ (assoc feature (current_index 1) deviation_value (current_value 1) + ;get the corresponding half of deviation value for each case + deviations + (map + (lambda (/ (first (current_value)) 2)) + (get feature_residuals_lists (get feature_index_map (current_index 1))) + ) ) ;does not affect nominal features @@ -825,10 +832,10 @@ (generalized_mean (map (lambda - ;divide global deviation by feature value + ;divide deviation by feature value (/ ;;;TODO: handle null tuples and edit features - deviation_value + (or (get deviations (current_index)) deviation_value) ;don't allow 0s by replacing them with half min gaps (max From f6941ad074725eb9b17a1a70b8592756c93c78b8 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 16 Jul 2026 09:19:00 -0400 Subject: [PATCH 14/14] improve initial ratio --- module/analysis_weights.amlg | 40 ++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/module/analysis_weights.amlg b/module/analysis_weights.amlg index 81d4a9b39..1033e4ddf 100644 --- a/module/analysis_weights.amlg +++ b/module/analysis_weights.amlg @@ -425,15 +425,39 @@ (declare (assoc initial_ratios_map (if !ratioMeasureMap - (map - (lambda - ;;;TODO: use generalized mean of initial_deviation / case value, for now just use initial deviation / median - (/ - (get initial_residual_values_map (current_index)) - (compute_on_contained_entities (query_quantile (current_index) 0.5)) - ) + (let + (assoc + f_values_maps + (map + (lambda (retrieve_from_entity (current_value) !ratioMeasureMap) ) + (call !AllCases) + ) + ) + ;initial ratio computed as: harmonic mean of initial_deviation / case value for all cases + (map + (lambda (let + (assoc feature (current_index 1)) + (generalized_mean + (map + (lambda + (/ + ;;;TODO: handle nulls and edit features + (get initial_residual_values_map feature) + + ;don't allow 0's by replacing them with half min gaps + (max + (abs (get (current_value) feature)) + (get !cachedFeatureHalfMinGapMap feature) + ) + ) + ) + f_values_maps + ) + -1 + ) + )) + !ratioMeasureMap ) - !ratioMeasureMap ) ) ))