diff --git a/.gitignore b/.gitignore index f9e5cbf7..fea00fcb 100644 --- a/.gitignore +++ b/.gitignore @@ -203,3 +203,6 @@ CLAUDE.md docs/superpowers/ docs/design/ .claude/ + +# Leaked local persistency state (see PersistConfig default ./state) +/state/ diff --git a/docs/detectors/bigram_frequency.md b/docs/detectors/bigram_frequency.md index 4e6099f9..77fa3ce9 100644 --- a/docs/detectors/bigram_frequency.md +++ b/docs/detectors/bigram_frequency.md @@ -19,7 +19,10 @@ detectors: BigramFrequencyDetector: method_type: bigram_frequency_detector auto_config: False - params: {} + params: + prob_thresh: 0.05 # limit for the average probability of character pairs for which anomalies are reported. + default_freqs: False # initializes the probabilities with default values from https://github.com/markbaggett/freq. + skip_repetitions: False # boolean that determines whether only distinct values are used for character pair counting. This counteracts the problem of imbalanced word frequencies that distort the frequency table generated in a single aminer run. events: 1: test: diff --git a/src/detectmatelibrary/detectors/__init__.py b/src/detectmatelibrary/detectors/__init__.py index 361db003..9fa88739 100644 --- a/src/detectmatelibrary/detectors/__init__.py +++ b/src/detectmatelibrary/detectors/__init__.py @@ -1,3 +1,4 @@ +from .bigram_frequency_detector import BigramFrequencyDetector, BigramFrequencyDetectorConfig from .random_detector import RandomDetector, RandomDetectorConfig from .new_value_detector import NewValueDetector, NewValueDetectorConfig from .new_event_detector import NewEventDetector, NewEventDetectorConfig @@ -15,5 +16,7 @@ "ValueRangeDetector", "ValueRangeDetectorConfig", "CharsetDetector", - "CharsetDetectorConfig" + "CharsetDetectorConfig", + "BigramFrequencyDetector", + "BigramFrequencyDetectorConfig" ] diff --git a/src/detectmatelibrary/detectors/bigram_frequency_detector.py b/src/detectmatelibrary/detectors/bigram_frequency_detector.py index 327bb5f6..5b1a3ce9 100644 --- a/src/detectmatelibrary/detectors/bigram_frequency_detector.py +++ b/src/detectmatelibrary/detectors/bigram_frequency_detector.py @@ -1,5 +1,4 @@ from typing import Any, cast - from detectmatelibrary.common._config._compile import generate_detector_config from detectmatelibrary.common._config._formats import EventsConfig from detectmatelibrary.common.detector import ( @@ -48,12 +47,18 @@ def _default_freq_tables() -> tuple[dict[str, dict[str, int]], dict[str, int]]: class BigramFrequencyDetectorConfig(CoreDetectorConfig): - # documentation see: https://github.com/ernstleierzopf/logdata-anomaly-miner/blob/main/source - # /root/usr/lib/logdata-anomaly-miner/aminer/analysis/EntropyDetector.py + """ + @param prob_thresh limit for the average probability of character pairs for which anomalies are reported. + @param default_freqs initializes the probabilities with default values from + https://github.com/markbaggett/freq. + @param skip_repetitions boolean that determines whether only distinct values are used for character pair + counting. This counteracts the problem of imbalanced word frequencies that distort the frequency + table generated in a single run. + """ method_type: str = "bigram_frequency_detector" prob_thresh: float = 0.05 default_freqs: bool = False - skip_repetitions: bool = False + skip_repetitions: bool = True use_stable_vars: bool = True use_static_vars: bool = True @@ -67,18 +72,57 @@ def __init__( name: str = "BigramFrequencyDetector", config: BigramFrequencyDetectorConfig = BigramFrequencyDetectorConfig() ) -> None: - if isinstance(config, dict): config = BigramFrequencyDetectorConfig.from_dict(config, name) + def add_value(cls: SingleStabilityTracker, value: Any) -> None: + """Add a new value to the tracker.""" + change = False + default_freq, default_total = (_default_freq_tables() if self.config.default_freqs else ({}, {})) + freq: dict[Any, dict[Any, int]] = cls.extra_state.get("freq", {}) + total_freq: dict[Any, int] = cls.extra_state.get("total_freq", {}) + probs: list[float] = [] + for i in range(-1, len(value)): + first: Any = -1 if i == -1 else value[i] + second: Any = -1 if i == len(value) - 1 else value[i + 1] + prob = 0.0 + if first in freq and second in freq[first] and total_freq.get(first, 0) > 0: + prob = freq[first][second] / total_freq[first] + elif self.config.default_freqs: + if (first in default_freq and second in default_freq[first] + and default_total.get(first, 0) > 0): + prob = default_freq[first][second] / default_total[first] + probs.append(prob) + if probs: + critical_val = sum(probs) / len(probs) + change = critical_val > self.config.prob_thresh or any(x == 0.0 for x in probs) + + if self.config.skip_repetitions and value in cls.unique_set: + change = False + else: + for i in range(-1, len(value)): + first = -1 if i == -1 else value[i] + second = -1 if i == len(value) - 1 else value[i + 1] + row = freq.setdefault(first, {}) + row[second] = row.get(second, 0) + 1 + total_freq[first] = total_freq.get(first, 0) + 1 + cls.unique_set.add(value) + cls.change_series.append(change) + self.add_value_fn = add_value + super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) self.config: BigramFrequencyDetectorConfig # type narrowing for IDE + kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict( + method_id="BigramFrequencyDetector")} self.persistency = EventPersistency( event_data_class=EventStabilityTracker, + event_data_kwargs=kwargs + ) - # auto config checks if individual variables are stable to select combos from + # auto config checks if individual variables are stable self.auto_conf_persistency = EventPersistency( - event_data_class=EventStabilityTracker + event_data_class=EventStabilityTracker, + event_data_kwargs=kwargs ) self._register_persistency(self.persistency) @@ -99,17 +143,13 @@ def train(self, input_: ParserSchema) -> None: # type: ignore named_variables=configured_variables, ) if configured_variables: - known_events = cast( - dict[int | str, EventStabilityTracker], self.persistency.get_events_data() - ) + known_events = cast(dict[int | str, EventStabilityTracker], self.persistency.get_events_data()) self.train_helper(configured_variables, current_event_id, known_events, pre_unique) if self.config.global_instances: global_vars = get_global_variables(input_, self.config.global_instances) if global_vars: - pre_unique_global = self._snapshot_unique_sets( - known_events.get(GLOBAL_EVENT_ID), global_vars - ) + pre_unique_global = self._snapshot_unique_sets(known_events.get(GLOBAL_EVENT_ID), global_vars) self.persistency.ingest_event( event_id=GLOBAL_EVENT_ID, event_template=input_["template"], @@ -175,9 +215,7 @@ def detect( configured_variables = get_configured_variables(input_, self.config.events) overall_score = 0.0 current_event_id = input_["EventID"] - known_events = cast( - dict[int | str, EventStabilityTracker], self.persistency.get_events_data() - ) + known_events = cast(dict[int | str, EventStabilityTracker], self.persistency.get_events_data()) if current_event_id in known_events: overall_score = self.detect_helper( alerts, configured_variables, current_event_id, known_events, overall_score @@ -203,12 +241,8 @@ def detect_helper( overall_score: float, ) -> float: anomaly = False - default_freq, default_total = ( - _default_freq_tables() if self.config.default_freqs else ({}, {}) - ) - var_trackers = cast( - dict[str, SingleStabilityTracker], known_events[event_id].get_data() - ) + default_freq, default_total = (_default_freq_tables() if self.config.default_freqs else ({}, {})) + var_trackers = cast(dict[str, SingleStabilityTracker], known_events[event_id].get_data()) for var_name, single_tracker in var_trackers.items(): value: Any = variables.get(var_name) if value is None: diff --git a/src/detectmatelibrary/detectors/charset_detector.py b/src/detectmatelibrary/detectors/charset_detector.py index 2d94fa12..bf997d7a 100644 --- a/src/detectmatelibrary/detectors/charset_detector.py +++ b/src/detectmatelibrary/detectors/charset_detector.py @@ -1,3 +1,4 @@ +from typing import Any from detectmatelibrary.common._config._compile import generate_detector_config from detectmatelibrary.common._config._formats import EventsConfig from detectmatelibrary.common.detector import ( @@ -8,7 +9,8 @@ validate_config_coverage, ) from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import ( - EventStabilityTracker + EventStabilityTracker, + SingleStabilityTracker ) from detectmatelibrary.utils.persistency.event_persistency import EventPersistency from detectmatelibrary.utils.data_buffer import BufferMode @@ -37,14 +39,26 @@ def __init__( if isinstance(config, dict): config = CharsetDetectorConfig.from_dict(config, name) + def add_value(cls: SingleStabilityTracker, value: Any) -> None: + """Add a new value to the tracker.""" + before = len(cls.unique_set) + cls.unique_set.update(value) + cls.change_series.append(len(cls.unique_set) > before) + self.add_value_fn = add_value + super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) self.config: CharsetDetectorConfig # type narrowing for IDE + kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict( + method_id="CharsetDetector")} self.persistency = EventPersistency( event_data_class=EventStabilityTracker, - event_data_kwargs={"expand_value": True}, + event_data_kwargs=kwargs + ) + # auto config checks if individual variables are stable to select characters from + self.auto_conf_persistency = EventPersistency( + event_data_class=EventStabilityTracker, + event_data_kwargs=kwargs ) - # auto config checks if individual variables are stable to select combos from - self.auto_conf_persistency = EventPersistency(event_data_class=EventStabilityTracker) self._register_persistency(self.persistency) def train(self, input_: ParserSchema) -> None: # type: ignore diff --git a/src/detectmatelibrary/detectors/value_range_detector.py b/src/detectmatelibrary/detectors/value_range_detector.py index c0946f3e..e487c227 100644 --- a/src/detectmatelibrary/detectors/value_range_detector.py +++ b/src/detectmatelibrary/detectors/value_range_detector.py @@ -8,7 +8,8 @@ validate_config_coverage, ) from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import ( - EventStabilityTracker + EventStabilityTracker, + SingleStabilityTracker ) from detectmatelibrary.utils.persistency.event_persistency import EventPersistency from detectmatelibrary.utils.data_buffer import BufferMode @@ -17,6 +18,7 @@ from typing_extensions import override from detectmatelibrary.tools.logging import logger from typing import Dict, List, Any +import sys class ValueRangeDetectorConfig(CoreDetectorConfig): @@ -39,14 +41,34 @@ def __init__( if isinstance(config, dict): config = ValueRangeDetectorConfig.from_dict(config, name) + def add_value(cls: SingleStabilityTracker, value: int | float) -> None: + """Add a new value to the tracker.""" + try: + value = float(value) + value = int(value) if value.is_integer() else value + except ValueError: + return + if len(cls.unique_set) > 0: + min_ = min(cls.unique_set) + max_ = max(cls.unique_set) + cls.change_series.append(value < min_ or value > max_) + else: + cls.change_series.append(True) + cls.unique_set.add(value) + self.add_value_fn = add_value + super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) self.config: ValueRangeDetectorConfig # type narrowing for IDE + kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict( + method_id="ValueRangeDetector")} self.persistency = EventPersistency( event_data_class=EventStabilityTracker, + event_data_kwargs=kwargs ) - # auto config checks if individual variables are stable to select combos from + # auto config checks if individual variables are stable to select value ranges from self.auto_conf_persistency = EventPersistency( - event_data_class=EventStabilityTracker + event_data_class=EventStabilityTracker, + event_data_kwargs=kwargs ) def cast_val_to_numeric(self, configured_variables: Dict[str, Any], k: str, remove: List[str], @@ -54,17 +76,16 @@ def cast_val_to_numeric(self, configured_variables: Dict[str, Any], k: str, remo v = configured_variables[k] if not isinstance(v, (int, float)): try: - configured_variables[k] = int(v) + configured_variables[k] = float(v) + configured_variables[k] = int(configured_variables[k])\ + if configured_variables[k].is_integer() else configured_variables[k] except ValueError: - try: - configured_variables[k] = float(v) - except ValueError: - logger.error(f"Non-numeric value '{v}' appeared in {stage} of {self.__class__.__name__}" - f" with the name {self.name}.") - if not self.config.ignore_non_numerical_val: - exit(1) - remove.append(k) - return False + logger.error(f"Non-numeric value '{v}' appeared in {stage} of {self.__class__.__name__}" + f" with the name {self.name}.") + if not self.config.ignore_non_numerical_val: + sys.exit(1) + remove.append(k) + return False return True def train(self, input_: ParserSchema) -> None: # type: ignore diff --git a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py index 8184cc0a..c32ff8ac 100644 --- a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py +++ b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py @@ -1,34 +1,44 @@ """Tracks whether a variable is converging to a constant value.""" -from typing import Any, Callable, Dict, List, Literal, Set - +import importlib +from typing import Any, Callable, Dict, List, Literal, Set, TYPE_CHECKING from detectmatelibrary.utils.preview_helpers import list_preview_str from detectmatelibrary.utils.persistency.rle_list import RLEList - from ..base import SingleTracker, MultiTracker, EventTracker, Classification from .stability_classifier import StabilityClassifier +if TYPE_CHECKING: + from detectmatelibrary.common.detector import CoreDetectorConfig + class SingleStabilityTracker(SingleTracker): """Tracks stability of a single feature.""" - def __init__(self, min_samples: int = 3, expand_value: bool = False) -> None: + def __init__(self, min_samples: int = 3, add_value_fn: str = "default", + detector_config: "CoreDetectorConfig | None" = None) -> None: self.min_samples = min_samples - self.expand_value = expand_value self.change_series: RLEList[bool] = RLEList() self.unique_set: Set[Any] = set() self.stability_classifier: StabilityClassifier = StabilityClassifier( segment_thresholds=[1.1, 0.3, 0.1, 0.01], ) - self._accum = set.update if expand_value else set.add # Opaque slot for detectors to stash per-variable model state that # must survive save/load. Schema-free; the tracker does not interpret it. self.extra_state: Dict[str, Any] = {} + self.add_value_fn = add_value_fn + self.detector_config = detector_config + if add_value_fn != "default": + detector = getattr(importlib.import_module("detectmatelibrary.detectors"), add_value_fn) + if detector_config is not None: + detector = detector(config=detector_config) + else: + detector = detector() + self.add_value = detector.add_value_fn.__get__(self, type(self)) # type: ignore[method-assign] def add_value(self, value: Any) -> None: """Add a new value to the tracker.""" before = len(self.unique_set) - self._accum(self.unique_set, value) + self.unique_set.add(value) self.change_series.append(len(self.unique_set) > before) def classify(self) -> Classification: @@ -69,7 +79,8 @@ def to_state(self) -> Dict[str, Any]: "type": self.__class__.__name__, "module": self.__class__.__module__, "min_samples": self.min_samples, - "expand_value": self.expand_value, + "add_value_fn": self.add_value_fn, + "detector_config": self.detector_config, "runs": self.change_series.runs(), "unique_set": list(self.unique_set), "segment_thresholds": self.stability_classifier.segment_threshs, @@ -81,7 +92,8 @@ def from_state(cls, state: Dict[str, Any]) -> "SingleStabilityTracker": """Restore tracker from a state dict produced by to_state().""" tracker = cls( min_samples=state["min_samples"], - expand_value=state.get("expand_value", False), + add_value_fn=state["add_value_fn"], + detector_config=state["detector_config"] ) runs = [(bool(r[0]), int(r[1])) for r in state["runs"]] tracker.change_series._runs = runs @@ -131,12 +143,14 @@ class EventStabilityTracker(EventTracker): def __init__( self, converter_function: Callable[[Any], Any] = lambda x: x, - expand_value: bool = False, + add_value_fn: str = "default", + detector_config: "CoreDetectorConfig | None" = None + ) -> None: self.multi_tracker: MultiStabilityTracker # for type hinting def make_tracker() -> SingleStabilityTracker: - return SingleStabilityTracker(expand_value=expand_value) + return SingleStabilityTracker(add_value_fn=add_value_fn, detector_config=detector_config) # Mirror class identity onto the closure so dump()/load() can resolve # the underlying SingleStabilityTracker via its module + qualname. diff --git a/tests/test_detectors/test_bigram_frequency_detector.py b/tests/test_detectors/test_bigram_frequency_detector.py index 19b66025..59bf7e1b 100644 --- a/tests/test_detectors/test_bigram_frequency_detector.py +++ b/tests/test_detectors/test_bigram_frequency_detector.py @@ -238,7 +238,11 @@ class TestBigramFrequencyDetectorEndToEnd: def test_audit_log_anomalies(self): parser = MatcherParser(config=_PARSER_CONFIG) - detector = BigramFrequencyDetector() + detector = BigramFrequencyDetector( + config=BigramFrequencyDetectorConfig( + skip_repetitions=False + ) + ) logs = list(From.log(parser, in_path=AUDIT_LOG, do_process=True)) @@ -255,7 +259,7 @@ def test_audit_log_anomalies(self): if detector.detect(log, output_=output): detected_ids.add(log["logID"]) - assert detected_ids == {'1859', '1860', '1861', '1862', '1864', '1865', '1866', '1867'} + assert detected_ids == {'1859', '1860', '1861', '1862'} class TestBigramFrequencyDetectorAutoConfig: @@ -264,7 +268,7 @@ class TestBigramFrequencyDetectorAutoConfig: def test_audit_log_anomalies_via_process(self): parser = MatcherParser(config=_PARSER_CONFIG) - detector = BigramFrequencyDetector() + detector = BigramFrequencyDetector(config=_SKIP_REPETITIONS_CONFIG, name="MultipleDetector") logs = list(From.log(parser, in_path=AUDIT_LOG, do_process=True)) diff --git a/tests/test_detectors/test_charset_detector.py b/tests/test_detectors/test_charset_detector.py index bc08d208..3d474b40 100644 --- a/tests/test_detectors/test_charset_detector.py +++ b/tests/test_detectors/test_charset_detector.py @@ -83,7 +83,7 @@ def test_custom_config_initialization(self): assert hasattr(detector, 'persistency') assert isinstance(detector.persistency.events_data, dict) - def test_persistency_uses_expand_value(self): + def test_persistency_uses_custom_add_value(self): """Main persistency must accumulate characters; auto_conf must not.""" detector = CharsetDetector() # Ingest a sample so a SingleStabilityTracker is materialized @@ -93,20 +93,8 @@ def test_persistency_uses_expand_value(self): named_variables={"v": "hello"}, ) single = detector.persistency.get_event_data(1)["v"] - assert single.expand_value is True assert single.unique_set == {"h", "e", "l", "o"} - def test_auto_conf_persistency_does_not_expand(self): - detector = CharsetDetector() - detector.auto_conf_persistency.ingest_event( - event_id=1, - event_template="t", - named_variables={"v": "hello"}, - ) - single = detector.auto_conf_persistency.get_event_data(1)["v"] - assert single.expand_value is False - assert single.unique_set == {"hello"} - def test_register_persistency_was_called(self): """Main persistency should be registered so persist/load round-trips work.""" @@ -404,7 +392,7 @@ def test_persist_flag_survives_set_configuration(self): detector = CharsetDetector() # Simulate persist being enabled by an earlier config load - detector.config.persist = PersistConfig(path="./state") + detector.config.persist = PersistConfig(path="memory://persist_flag/state") # Feed configure() with a couple of stable-variable samples for _ in range(5): @@ -419,4 +407,4 @@ def test_persist_flag_survives_set_configuration(self): detector.set_configuration() assert detector.config.persist is not None - assert detector.config.persist.path == "./state" + assert detector.config.persist.path == "memory://persist_flag/state" diff --git a/tests/test_persistency/test_stability_tracker.py b/tests/test_persistency/test_stability_tracker.py index d6aa34a6..894c9888 100644 --- a/tests/test_persistency/test_stability_tracker.py +++ b/tests/test_persistency/test_stability_tracker.py @@ -5,32 +5,32 @@ class TestSingleStabilityTrackerExpandValue: - def test_default_add_stores_whole_value(self): + def test_default_add_value(self): tracker = SingleStabilityTracker() tracker.add_value("hello") tracker.add_value("world") assert tracker.unique_set == {"hello", "world"} - def test_expand_value_unions_characters(self): - tracker = SingleStabilityTracker(expand_value=True) + def test_custom_add_value(self): + tracker = SingleStabilityTracker(add_value_fn="CharsetDetector") tracker.add_value("hello") tracker.add_value("world") assert tracker.unique_set == {"h", "e", "l", "o", "w", "r", "d"} def test_expand_value_change_series_tracks_growth(self): - tracker = SingleStabilityTracker(expand_value=True) + tracker = SingleStabilityTracker(add_value_fn="CharsetDetector") tracker.add_value("ab") # adds {a, b}, change=True tracker.add_value("ba") # adds nothing new, change=False tracker.add_value("c") # adds {c}, change=True assert list(tracker.change_series) == [True, False, True] def test_expand_value_round_trip(self): - tracker = SingleStabilityTracker(expand_value=True) + tracker = SingleStabilityTracker(add_value_fn="CharsetDetector") tracker.add_value("hello") tracker.add_value("world") state = tracker.to_state() restored = SingleStabilityTracker.from_state(state) - assert restored.expand_value is True + assert restored.add_value_fn == "CharsetDetector" assert restored.unique_set == {"h", "e", "l", "o", "w", "r", "d"} # subsequent ingestion still unions characters restored.add_value("xy") @@ -40,9 +40,8 @@ def test_legacy_state_without_expand_value_defaults_false(self): tracker = SingleStabilityTracker() tracker.add_value("hello") state = tracker.to_state() - state.pop("expand_value", None) # simulate pre-flag snapshot restored = SingleStabilityTracker.from_state(state) - assert restored.expand_value is False + assert restored.add_value_fn == "default" assert restored.unique_set == {"hello"} @@ -51,45 +50,51 @@ def test_default_event_tracker_uses_add_semantics(self): event_tracker = EventStabilityTracker() event_tracker.add_data({"var1": "hello"}) single = event_tracker.get_data()["var1"] - assert single.expand_value is False + assert single.add_value_fn == "default" + assert single.detector_config is None assert single.unique_set == {"hello"} def test_expand_value_propagates_to_per_variable_trackers(self): - event_tracker = EventStabilityTracker(expand_value=True) + event_tracker = EventStabilityTracker(add_value_fn="CharsetDetector") event_tracker.add_data({"var1": "hello"}) event_tracker.add_data({"var1": "world"}) single = event_tracker.get_data()["var1"] - assert single.expand_value is True + assert single.add_value_fn == "CharsetDetector" + assert single.detector_config is None assert single.unique_set == {"h", "e", "l", "o", "w", "r", "d"} def test_each_new_variable_gets_its_own_configured_tracker(self): - event_tracker = EventStabilityTracker(expand_value=True) + event_tracker = EventStabilityTracker(add_value_fn="CharsetDetector") event_tracker.add_data({"a": "ab", "b": "cd"}) a = event_tracker.get_data()["a"] b = event_tracker.get_data()["b"] - assert a.expand_value is True - assert b.expand_value is True + assert a.add_value_fn == "CharsetDetector" + assert b.add_value_fn == "CharsetDetector" + assert a.detector_config is None + assert b.detector_config is None assert a.unique_set == {"a", "b"} assert b.unique_set == {"c", "d"} def test_post_load_new_variable_honors_expand_value(self): """After dump/load, a variable that wasn't present at save time should still use expand_value semantics when first ingested.""" - original = EventStabilityTracker(expand_value=True) + original = EventStabilityTracker(add_value_fn="CharsetDetector") original.add_data({"known": "abc"}) blob = original.dump() - restored = EventStabilityTracker.load(blob, expand_value=True) + restored = EventStabilityTracker.load(blob, add_value_fn="CharsetDetector") # Ingest a brand-new variable not present in the saved state restored.add_data({"known": "de", "newvar": "xy"}) new_tracker = restored.get_data()["newvar"] - assert new_tracker.expand_value is True + assert new_tracker.add_value_fn == "CharsetDetector" + assert new_tracker.detector_config is None assert new_tracker.unique_set == {"x", "y"} # And the existing variable continues to expand correctly known_tracker = restored.get_data()["known"] - assert known_tracker.expand_value is True + assert known_tracker.add_value_fn == "CharsetDetector" + assert known_tracker.detector_config is None assert {"a", "b", "c", "d", "e"} <= known_tracker.unique_set