|
| 1 | +""" |
| 2 | +Booleans in a metrics dict are FLAGS, not scores. |
| 3 | +
|
| 4 | +`bool` is a subclass of `int`, so a naive `isinstance(value, (int, float))` check |
| 5 | +silently treats True/False as 1.0/0.0. That matters in two places: |
| 6 | +
|
| 7 | + * display - `timeout=1.0000` instead of `timeout=True` |
| 8 | + * FITNESS - a program that timed out returns {"error": 0.0, "timeout": True} |
| 9 | + (openevolve/evaluator.py), which averaged to 0.5 instead of 0.0, |
| 10 | + handing a failed program a mid-range score. |
| 11 | +
|
| 12 | +OpenEvolve's own evaluator emits `timeout: True` in several places, so this is not |
| 13 | +a hypothetical input. |
| 14 | +""" |
| 15 | + |
| 16 | +import os |
| 17 | +import unittest |
| 18 | + |
| 19 | +os.environ.setdefault("OPENAI_API_KEY", "test") |
| 20 | + |
| 21 | +from openevolve.utils.format_utils import format_improvement_safe, format_metrics_safe |
| 22 | +from openevolve.utils.metrics_utils import get_fitness_score, safe_numeric_average |
| 23 | + |
| 24 | + |
| 25 | +class TestBooleanMetricsFormatting(unittest.TestCase): |
| 26 | + def test_bool_rendered_as_true_false(self): |
| 27 | + self.assertEqual( |
| 28 | + format_metrics_safe({"valid": True, "timeout": False, "score": 0.25}), |
| 29 | + "valid=True, timeout=False, score=0.2500", |
| 30 | + ) |
| 31 | + |
| 32 | + def test_bool_excluded_from_improvement(self): |
| 33 | + """A boolean flipping False->True is not a '+1.0000' improvement.""" |
| 34 | + self.assertEqual( |
| 35 | + format_improvement_safe( |
| 36 | + {"valid": False, "score": 0.25}, |
| 37 | + {"valid": True, "score": 0.5}, |
| 38 | + ), |
| 39 | + "score=+0.2500", |
| 40 | + ) |
| 41 | + |
| 42 | + def test_numeric_formatting_unchanged(self): |
| 43 | + self.assertEqual(format_metrics_safe({"score": 0.5, "n": 3}), "score=0.5000, n=3.0000") |
| 44 | + |
| 45 | + |
| 46 | +class TestBooleanMetricsExcludedFromFitness(unittest.TestCase): |
| 47 | + def test_timed_out_program_scores_zero(self): |
| 48 | + """The exact dict openevolve/evaluator.py returns on timeout.""" |
| 49 | + metrics = {"error": 0.0, "timeout": True} |
| 50 | + # Before the fix both of these returned 0.5. |
| 51 | + self.assertEqual(safe_numeric_average(metrics), 0.0) |
| 52 | + self.assertEqual(get_fitness_score(metrics), 0.0) |
| 53 | + |
| 54 | + def test_bool_does_not_inflate_average(self): |
| 55 | + # Without the guard this would be (0.4 + 1.0) / 2 = 0.7 |
| 56 | + self.assertAlmostEqual(safe_numeric_average({"score": 0.4, "valid": True}), 0.4) |
| 57 | + |
| 58 | + def test_all_boolean_metrics_average_to_zero(self): |
| 59 | + self.assertEqual(safe_numeric_average({"valid": True, "timeout": False}), 0.0) |
| 60 | + |
| 61 | + def test_combined_score_still_takes_precedence(self): |
| 62 | + self.assertAlmostEqual(get_fitness_score({"combined_score": 0.9, "timeout": True}), 0.9) |
| 63 | + |
| 64 | + def test_ordinary_metrics_unaffected(self): |
| 65 | + self.assertAlmostEqual(safe_numeric_average({"a": 0.8, "b": 0.6}), 0.7) |
| 66 | + self.assertAlmostEqual(get_fitness_score({"a": 0.8, "b": 0.6}), 0.7) |
| 67 | + |
| 68 | + def test_feature_dimensions_still_excluded(self): |
| 69 | + """Bool handling must not disturb the existing feature-dimension exclusion.""" |
| 70 | + metrics = {"score": 0.8, "complexity": 100.0} |
| 71 | + self.assertAlmostEqual(get_fitness_score(metrics, ["complexity"]), 0.8) |
| 72 | + |
| 73 | + |
| 74 | +if __name__ == "__main__": |
| 75 | + unittest.main() |
0 commit comments