Feature/persistency improvements new

.gitignore

@@ -203,3 +203,6 @@ CLAUDE.md
 docs/superpowers/
 docs/design/
 .claude/
+
+# Leaked local persistency state (see PersistConfig default ./state)
+/state/

docs/detectors/bigram_frequency.md

@@ -19,7 +19,10 @@ detectors:
     BigramFrequencyDetector:
         method_type: bigram_frequency_detector
         auto_config: False
-        params: {}
+        params:
+            prob_thresh: 0.05        # limit for the average probability of character pairs for which anomalies are reported.
+            default_freqs: False     # initializes the probabilities with default values from https://github.com/markbaggett/freq.
+            skip_repetitions: False  # boolean that determines whether only distinct values are used for character pair counting. This counteracts the problem of imbalanced word frequencies that distort the frequency table generated in a single aminer run.
         events:
             1:
                 test:

src/detectmatelibrary/detectors/__init__.py

@@ -1,3 +1,4 @@
+from .bigram_frequency_detector import BigramFrequencyDetector, BigramFrequencyDetectorConfig
 from .random_detector import RandomDetector, RandomDetectorConfig
 from .new_value_detector import NewValueDetector, NewValueDetectorConfig
 from .new_event_detector import NewEventDetector, NewEventDetectorConfig
@@ -15,5 +16,7 @@
     "ValueRangeDetector",
     "ValueRangeDetectorConfig",
     "CharsetDetector",
-    "CharsetDetectorConfig"
+    "CharsetDetectorConfig",
+    "BigramFrequencyDetector",
+    "BigramFrequencyDetectorConfig"
 ]

src/detectmatelibrary/detectors/bigram_frequency_detector.py

@@ -1,5 +1,4 @@
 from typing import Any, cast
-
 from detectmatelibrary.common._config._compile import generate_detector_config
 from detectmatelibrary.common._config._formats import EventsConfig
 from detectmatelibrary.common.detector import (
@@ -48,12 +47,18 @@ def _default_freq_tables() -> tuple[dict[str, dict[str, int]], dict[str, int]]:
 
 
 class BigramFrequencyDetectorConfig(CoreDetectorConfig):
-    # documentation see: https://github.com/ernstleierzopf/logdata-anomaly-miner/blob/main/source
-    # /root/usr/lib/logdata-anomaly-miner/aminer/analysis/EntropyDetector.py
+    """
+    @param prob_thresh limit for the average probability of character pairs for which anomalies are reported.
+    @param default_freqs initializes the probabilities with default values from
+           https://github.com/markbaggett/freq.
+    @param skip_repetitions boolean that determines whether only distinct values are used for character pair
+           counting. This counteracts the problem of imbalanced word frequencies that distort the frequency
+           table generated in a single run.
+    """
     method_type: str = "bigram_frequency_detector"
     prob_thresh: float = 0.05
     default_freqs: bool = False
-    skip_repetitions: bool = False
+    skip_repetitions: bool = True
 
     use_stable_vars: bool = True
     use_static_vars: bool = True
@@ -67,18 +72,57 @@ def __init__(
         name: str = "BigramFrequencyDetector",
         config: BigramFrequencyDetectorConfig = BigramFrequencyDetectorConfig()
     ) -> None:
-
         if isinstance(config, dict):
             config = BigramFrequencyDetectorConfig.from_dict(config, name)
 
+        def add_value(cls: SingleStabilityTracker, value: Any) -> None:
+            """Add a new value to the tracker."""
+            change = False
+            default_freq, default_total = (_default_freq_tables() if self.config.default_freqs else ({}, {}))
+            freq: dict[Any, dict[Any, int]] = cls.extra_state.get("freq", {})
+            total_freq: dict[Any, int] = cls.extra_state.get("total_freq", {})
+            probs: list[float] = []
+            for i in range(-1, len(value)):
+                first: Any = -1 if i == -1 else value[i]
+                second: Any = -1 if i == len(value) - 1 else value[i + 1]
+                prob = 0.0
+                if first in freq and second in freq[first] and total_freq.get(first, 0) > 0:
+                    prob = freq[first][second] / total_freq[first]
+                elif self.config.default_freqs:
+                    if (first in default_freq and second in default_freq[first]
+                            and default_total.get(first, 0) > 0):
+                        prob = default_freq[first][second] / default_total[first]
+                probs.append(prob)
+            if probs:
+                critical_val = sum(probs) / len(probs)
+                change = critical_val > self.config.prob_thresh or any(x == 0.0 for x in probs)
+
+            if self.config.skip_repetitions and value in cls.unique_set:
+                change = False
+            else:
+                for i in range(-1, len(value)):
+                    first = -1 if i == -1 else value[i]
+                    second = -1 if i == len(value) - 1 else value[i + 1]
+                    row = freq.setdefault(first, {})
+                    row[second] = row.get(second, 0) + 1
+                    total_freq[first] = total_freq.get(first, 0) + 1
+            cls.unique_set.add(value)
+            cls.change_series.append(change)
+        self.add_value_fn = add_value
+
         super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config)
         self.config: BigramFrequencyDetectorConfig  # type narrowing for IDE
+        kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict(
+            method_id="BigramFrequencyDetector")}
         self.persistency = EventPersistency(
             event_data_class=EventStabilityTracker,
+            event_data_kwargs=kwargs
+
         )
-        # auto config checks if individual variables are stable to select combos from
+        # auto config checks if individual variables are stable
         self.auto_conf_persistency = EventPersistency(
-            event_data_class=EventStabilityTracker
+            event_data_class=EventStabilityTracker,
+            event_data_kwargs=kwargs
         )
         self._register_persistency(self.persistency)
 
@@ -99,17 +143,13 @@ def train(self, input_: ParserSchema) -> None:  # type: ignore
             named_variables=configured_variables,
         )
         if configured_variables:
-            known_events = cast(
-                dict[int | str, EventStabilityTracker], self.persistency.get_events_data()
-            )
+            known_events = cast(dict[int | str, EventStabilityTracker], self.persistency.get_events_data())
             self.train_helper(configured_variables, current_event_id, known_events, pre_unique)
 
         if self.config.global_instances:
             global_vars = get_global_variables(input_, self.config.global_instances)
             if global_vars:
-                pre_unique_global = self._snapshot_unique_sets(
-                    known_events.get(GLOBAL_EVENT_ID), global_vars
-                )
+                pre_unique_global = self._snapshot_unique_sets(known_events.get(GLOBAL_EVENT_ID), global_vars)
                 self.persistency.ingest_event(
                     event_id=GLOBAL_EVENT_ID,
                     event_template=input_["template"],
@@ -175,9 +215,7 @@ def detect(
         configured_variables = get_configured_variables(input_, self.config.events)
         overall_score = 0.0
         current_event_id = input_["EventID"]
-        known_events = cast(
-            dict[int | str, EventStabilityTracker], self.persistency.get_events_data()
-        )
+        known_events = cast(dict[int | str, EventStabilityTracker], self.persistency.get_events_data())
         if current_event_id in known_events:
             overall_score = self.detect_helper(
                 alerts, configured_variables, current_event_id, known_events, overall_score
@@ -203,12 +241,8 @@ def detect_helper(
         overall_score: float,
     ) -> float:
         anomaly = False
-        default_freq, default_total = (
-            _default_freq_tables() if self.config.default_freqs else ({}, {})
-        )
-        var_trackers = cast(
-            dict[str, SingleStabilityTracker], known_events[event_id].get_data()
-        )
+        default_freq, default_total = (_default_freq_tables() if self.config.default_freqs else ({}, {}))
+        var_trackers = cast(dict[str, SingleStabilityTracker], known_events[event_id].get_data())
         for var_name, single_tracker in var_trackers.items():
             value: Any = variables.get(var_name)
             if value is None:

src/detectmatelibrary/detectors/charset_detector.py

@@ -1,3 +1,4 @@
+from typing import Any
 from detectmatelibrary.common._config._compile import generate_detector_config
 from detectmatelibrary.common._config._formats import EventsConfig
 from detectmatelibrary.common.detector import (
@@ -8,7 +9,8 @@
     validate_config_coverage,
 )
 from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import (
-    EventStabilityTracker
+    EventStabilityTracker,
+    SingleStabilityTracker
 )
 from detectmatelibrary.utils.persistency.event_persistency import EventPersistency
 from detectmatelibrary.utils.data_buffer import BufferMode
@@ -37,14 +39,26 @@ def __init__(
         if isinstance(config, dict):
             config = CharsetDetectorConfig.from_dict(config, name)
 
+        def add_value(cls: SingleStabilityTracker, value: Any) -> None:
+            """Add a new value to the tracker."""
+            before = len(cls.unique_set)
+            cls.unique_set.update(value)
+            cls.change_series.append(len(cls.unique_set) > before)
+        self.add_value_fn = add_value
+
         super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config)
         self.config: CharsetDetectorConfig  # type narrowing for IDE
+        kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict(
+            method_id="CharsetDetector")}
         self.persistency = EventPersistency(
             event_data_class=EventStabilityTracker,
-            event_data_kwargs={"expand_value": True},
+            event_data_kwargs=kwargs
+        )
+        # auto config checks if individual variables are stable to select characters from
+        self.auto_conf_persistency = EventPersistency(
+            event_data_class=EventStabilityTracker,
+            event_data_kwargs=kwargs
         )
-        # auto config checks if individual variables are stable to select combos from
-        self.auto_conf_persistency = EventPersistency(event_data_class=EventStabilityTracker)
         self._register_persistency(self.persistency)
 
     def train(self, input_: ParserSchema) -> None:  # type: ignore

src/detectmatelibrary/detectors/value_range_detector.py

@@ -8,7 +8,8 @@
     validate_config_coverage,
 )
 from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import (
-    EventStabilityTracker
+    EventStabilityTracker,
+    SingleStabilityTracker
 )
 from detectmatelibrary.utils.persistency.event_persistency import EventPersistency
 from detectmatelibrary.utils.data_buffer import BufferMode
@@ -17,6 +18,7 @@
 from typing_extensions import override
 from detectmatelibrary.tools.logging import logger
 from typing import Dict, List, Any
+import sys
 
 
 class ValueRangeDetectorConfig(CoreDetectorConfig):
@@ -39,32 +41,51 @@ def __init__(
         if isinstance(config, dict):
             config = ValueRangeDetectorConfig.from_dict(config, name)
 
+        def add_value(cls: SingleStabilityTracker, value: int | float) -> None:
+            """Add a new value to the tracker."""
+            try:
+                value = float(value)
+                value = int(value) if value.is_integer() else value
+            except ValueError:
+                return
+            if len(cls.unique_set) > 0:
+                min_ = min(cls.unique_set)
+                max_ = max(cls.unique_set)
+                cls.change_series.append(value < min_ or value > max_)
+            else:
+                cls.change_series.append(True)
+            cls.unique_set.add(value)
+        self.add_value_fn = add_value
+
         super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config)
         self.config: ValueRangeDetectorConfig  # type narrowing for IDE
+        kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict(
+            method_id="ValueRangeDetector")}
         self.persistency = EventPersistency(
             event_data_class=EventStabilityTracker,
+            event_data_kwargs=kwargs
         )
-        # auto config checks if individual variables are stable to select combos from
+        # auto config checks if individual variables are stable to select value ranges from
         self.auto_conf_persistency = EventPersistency(
-            event_data_class=EventStabilityTracker
+            event_data_class=EventStabilityTracker,
+            event_data_kwargs=kwargs
         )
 
     def cast_val_to_numeric(self, configured_variables: Dict[str, Any], k: str, remove: List[str],
                             stage: str) -> bool:
         v = configured_variables[k]
         if not isinstance(v, (int, float)):
             try:
-                configured_variables[k] = int(v)
+                configured_variables[k] = float(v)
+                configured_variables[k] = int(configured_variables[k])\
+                    if configured_variables[k].is_integer() else configured_variables[k]
             except ValueError:
-                try:
-                    configured_variables[k] = float(v)
-                except ValueError:
-                    logger.error(f"Non-numeric value '{v}' appeared in {stage} of {self.__class__.__name__}"
-                                 f" with the name {self.name}.")
-                    if not self.config.ignore_non_numerical_val:
-                        exit(1)
-                    remove.append(k)
-                    return False
+                logger.error(f"Non-numeric value '{v}' appeared in {stage} of {self.__class__.__name__}"
+                             f" with the name {self.name}.")
+                if not self.config.ignore_non_numerical_val:
+                    sys.exit(1)
+                remove.append(k)
+                return False
         return True
 
     def train(self, input_: ParserSchema) -> None:  # type: ignore

src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py

@@ -1,34 +1,44 @@
 """Tracks whether a variable is converging to a constant value."""
 
-from typing import Any, Callable, Dict, List, Literal, Set
-
+import importlib
+from typing import Any, Callable, Dict, List, Literal, Set, TYPE_CHECKING
 from detectmatelibrary.utils.preview_helpers import list_preview_str
 from detectmatelibrary.utils.persistency.rle_list import RLEList
-
 from ..base import SingleTracker, MultiTracker, EventTracker, Classification
 from .stability_classifier import StabilityClassifier
 
+if TYPE_CHECKING:
+    from detectmatelibrary.common.detector import CoreDetectorConfig
+
 
 class SingleStabilityTracker(SingleTracker):
     """Tracks stability of a single feature."""
 
-    def __init__(self, min_samples: int = 3, expand_value: bool = False) -> None:
+    def __init__(self, min_samples: int = 3, add_value_fn: str = "default",
+                 detector_config: "CoreDetectorConfig | None" = None) -> None:
         self.min_samples = min_samples
-        self.expand_value = expand_value
         self.change_series: RLEList[bool] = RLEList()
         self.unique_set: Set[Any] = set()
         self.stability_classifier: StabilityClassifier = StabilityClassifier(
             segment_thresholds=[1.1, 0.3, 0.1, 0.01],
         )
-        self._accum = set.update if expand_value else set.add
         # Opaque slot for detectors to stash per-variable model state that
         # must survive save/load. Schema-free; the tracker does not interpret it.
         self.extra_state: Dict[str, Any] = {}
+        self.add_value_fn = add_value_fn
+        self.detector_config = detector_config
+        if add_value_fn != "default":
+            detector = getattr(importlib.import_module("detectmatelibrary.detectors"), add_value_fn)
+            if detector_config is not None:
+                detector = detector(config=detector_config)
+            else:
+                detector = detector()
+            self.add_value = detector.add_value_fn.__get__(self, type(self))  # type: ignore[method-assign]
 
     def add_value(self, value: Any) -> None:
         """Add a new value to the tracker."""
         before = len(self.unique_set)
-        self._accum(self.unique_set, value)
+        self.unique_set.add(value)
         self.change_series.append(len(self.unique_set) > before)
 
     def classify(self) -> Classification:
@@ -69,7 +79,8 @@ def to_state(self) -> Dict[str, Any]:
             "type": self.__class__.__name__,
             "module": self.__class__.__module__,
             "min_samples": self.min_samples,
-            "expand_value": self.expand_value,
+            "add_value_fn": self.add_value_fn,
+            "detector_config": self.detector_config,
             "runs": self.change_series.runs(),
             "unique_set": list(self.unique_set),
             "segment_thresholds": self.stability_classifier.segment_threshs,
@@ -81,7 +92,8 @@ def from_state(cls, state: Dict[str, Any]) -> "SingleStabilityTracker":
         """Restore tracker from a state dict produced by to_state()."""
         tracker = cls(
             min_samples=state["min_samples"],
-            expand_value=state.get("expand_value", False),
+            add_value_fn=state["add_value_fn"],
+            detector_config=state["detector_config"]
         )
         runs = [(bool(r[0]), int(r[1])) for r in state["runs"]]
         tracker.change_series._runs = runs
@@ -131,12 +143,14 @@ class EventStabilityTracker(EventTracker):
     def __init__(
         self,
         converter_function: Callable[[Any], Any] = lambda x: x,
-        expand_value: bool = False,
+        add_value_fn: str = "default",
+        detector_config: "CoreDetectorConfig | None" = None
+
     ) -> None:
         self.multi_tracker: MultiStabilityTracker  # for type hinting
 
         def make_tracker() -> SingleStabilityTracker:
-            return SingleStabilityTracker(expand_value=expand_value)
+            return SingleStabilityTracker(add_value_fn=add_value_fn, detector_config=detector_config)
 
         # Mirror class identity onto the closure so dump()/load() can resolve
         # the underlying SingleStabilityTracker via its module + qualname.