diff --git a/src/detectmatelibrary/common/variable_detector.py b/src/detectmatelibrary/common/variable_detector.py new file mode 100644 index 00000000..8b57395a --- /dev/null +++ b/src/detectmatelibrary/common/variable_detector.py @@ -0,0 +1,216 @@ +from detectmatelibrary.common._config._formats import EventsConfig +from detectmatelibrary.common._config._compile import generate_detector_config +from detectmatelibrary.common.detector import ( + CoreDetectorConfig, + CoreDetector, + get_configured_variables, + get_global_variables, + validate_config_coverage, +) +from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import ( + EventStabilityTracker, + SingleStabilityTracker, +) +from detectmatelibrary.utils.persistency.event_persistency import EventPersistency +from detectmatelibrary.utils.data_buffer import BufferMode +from detectmatelibrary.schemas import ParserSchema, DetectorSchema +from detectmatelibrary.constants import GLOBAL_EVENT_ID +from detectmatelibrary.tools.logging import logger + +from typing import Any, Dict, Optional, cast +from typing_extensions import override + + +class VariableDetectorConfig(CoreDetectorConfig): + use_stable_vars: bool = True + use_static_vars: bool = True + + +class VariableDetector(CoreDetector): + """Abstract base for detectors that learn a per-variable model from + configured log variables and flag anomalous values at detection time. + + Subclasses override a small set of hooks: + - ``_check_variable`` (required): the per-variable anomaly test. + - ``_prepare_variables`` (optional): transform variables per stage. + - ``_event_data_kwargs`` / ``_auto_conf_kwargs`` (optional): tracker + construction kwargs. + - ``_description`` / ``_alert_key`` (optional): output formatting. + + The five lifecycle methods (train/detect/configure/post_train/ + set_configuration) live here and are shared by all subclasses. + """ + + def __init__(self, name: str, config: VariableDetectorConfig) -> None: + super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) + self.config: VariableDetectorConfig # type narrowing for IDE + self.persistency = EventPersistency( + event_data_class=self._event_data_class(), + event_data_kwargs=self._event_data_kwargs(), + ) + # auto config checks individual-variable stability to select features + self.auto_conf_persistency = EventPersistency( + event_data_class=self._event_data_class(), + event_data_kwargs=self._auto_conf_kwargs(), + ) + self._register_persistency(self.persistency) + + # ---- construction hooks ------------------------------------------------- + + def _event_data_class(self) -> type: + return EventStabilityTracker + + def _event_data_kwargs(self) -> Optional[Dict[str, Any]]: + return None + + def _auto_conf_kwargs(self) -> Optional[Dict[str, Any]]: + return self._event_data_kwargs() + + def _stability_kwargs(self) -> Dict[str, Any]: + """Kwargs for detectors whose tracker rebinds a per-detector + ``add_value`` closure (charset / value_range / bigram).""" + name = type(self).__name__ + return { + "add_value_fn": name, + "detector_config": self.config.to_dict(method_id=name), + } + + # ---- per-detector hooks ------------------------------------------------- + + def _prepare_variables(self, variables: Dict[str, Any], stage: str) -> Dict[str, Any]: + """Transform extracted variables. + + ``stage`` is "training" or "detection". + """ + return variables + + def _check_variable( + self, tracker: SingleStabilityTracker, value: Any, key: Any + ) -> Optional[str]: + """Return an alert message if ``value`` is anomalous for ``tracker``, + else None.""" + raise NotImplementedError + + def _alert_key(self, event_id: Any, key: Any, is_global: bool) -> str: + return f"Global - {key}" if is_global else f"EventID {event_id} - {key}" + + def _description(self) -> str: + return f"{self.name} detected anomalies." + + # ---- shared lifecycle --------------------------------------------------- + + def train(self, input_: ParserSchema) -> None: # type: ignore + self._ingest(input_, get_configured_variables(input_, self.config.events), input_["EventID"]) + if self.config.global_instances: + global_vars = get_global_variables(input_, self.config.global_instances) + if global_vars: + self._ingest(input_, global_vars, GLOBAL_EVENT_ID) + + def _ingest(self, input_: ParserSchema, variables: Dict[str, Any], event_id: Any) -> None: + variables = self._prepare_variables(variables, "training") + self.persistency.ingest_event( + event_id=event_id, + event_template=input_["template"], + named_variables=variables, + ) + + def detect(self, input_: ParserSchema, output_: DetectorSchema) -> bool: # type: ignore + alerts: Dict[str, str] = {} + overall_score = 0.0 + known_events = self.persistency.get_events_data() + current_event_id = input_["EventID"] + + if current_event_id in known_events: + variables = self._prepare_variables( + get_configured_variables(input_, self.config.events), "detection" + ) + event_tracker = cast(EventStabilityTracker, known_events[current_event_id]) + overall_score += self._check_event( + alerts, current_event_id, event_tracker, variables, is_global=False + ) + if self.config.global_instances and GLOBAL_EVENT_ID in known_events: + global_vars = self._prepare_variables( + get_global_variables(input_, self.config.global_instances), "detection" + ) + global_tracker = cast(EventStabilityTracker, known_events[GLOBAL_EVENT_ID]) + overall_score += self._check_event( + alerts, GLOBAL_EVENT_ID, global_tracker, global_vars, is_global=True + ) + + if overall_score > 0: + output_["score"] = overall_score + output_["description"] = self._description() + output_["alertsObtain"].update(alerts) + return True + return False + + def _check_event( + self, + alerts: Dict[str, str], + event_id: Any, + event_tracker: EventStabilityTracker, + variables: Dict[str, Any], + is_global: bool, + ) -> float: + """Loop the event's per-variable trackers, accumulate alerts, score +1 + per anomalous variable. + + Bigram overrides this for event-level scoring. + """ + score = 0.0 + var_trackers = cast(Dict[str, SingleStabilityTracker], event_tracker.get_data()) + for key, tracker in var_trackers.items(): + value = variables.get(key) + if value is None: + continue + message = self._check_variable(tracker, value, key) + if message: + alerts[self._alert_key(event_id, key, is_global)] = message + score += 1.0 + return score + + def configure(self, input_: ParserSchema) -> None: # type: ignore + self.auto_conf_persistency.ingest_event( + event_id=input_["EventID"], + event_template=input_["template"], + variables=input_["variables"], + named_variables=input_["logFormatVariables"], + ) + + @override + def post_train(self) -> None: + if not self.config.auto_config: + validate_config_coverage(self.name, self.config.events, self.persistency) + + def set_configuration(self) -> None: + variables: Dict[Any, Any] = {} + for event_id, tracker in self.auto_conf_persistency.get_events_data().items(): + stability_tracker = cast(EventStabilityTracker, tracker) + stable = ( + stability_tracker.get_features_by_classification("STABLE") + if self.config.use_stable_vars + else [] + ) + static = ( + stability_tracker.get_features_by_classification("STATIC") + if self.config.use_static_vars + else [] + ) + selected = stable + static + if selected: + variables[event_id] = selected + old_persist = self.config.persist + config_dict = generate_detector_config( + variable_selection=variables, + detector_name=self.name, + method_type=self.config.method_type, + ) + self.config = type(self.config).from_dict(config_dict, self.name) + self.config.persist = old_persist + events = self.config.events + if isinstance(events, EventsConfig) and not events.events: + logger.warning( + f"[{self.name}] auto_config=True generated an empty configuration. " + "No stable variables were found in configure-phase data. " + "The detector will produce no alerts." + ) diff --git a/src/detectmatelibrary/detectors/bigram_frequency_detector.py b/src/detectmatelibrary/detectors/bigram_frequency_detector.py index 5b1a3ce9..82c9b1f0 100644 --- a/src/detectmatelibrary/detectors/bigram_frequency_detector.py +++ b/src/detectmatelibrary/detectors/bigram_frequency_detector.py @@ -1,23 +1,13 @@ -from typing import Any, cast -from detectmatelibrary.common._config._compile import generate_detector_config -from detectmatelibrary.common._config._formats import EventsConfig -from detectmatelibrary.common.detector import ( - CoreDetectorConfig, - CoreDetector, - get_configured_variables, - get_global_variables, - validate_config_coverage, -) +from typing import Any, Dict, Optional, cast + +from detectmatelibrary.common.variable_detector import VariableDetector, VariableDetectorConfig +from detectmatelibrary.common.detector import get_configured_variables, get_global_variables from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import ( EventStabilityTracker, SingleStabilityTracker, ) -from detectmatelibrary.utils.persistency.event_persistency import EventPersistency -from detectmatelibrary.utils.data_buffer import BufferMode -from detectmatelibrary.schemas import ParserSchema, DetectorSchema +from detectmatelibrary.schemas import ParserSchema from detectmatelibrary.constants import GLOBAL_EVENT_ID, DEFAULT_FREQUENCIES -from typing_extensions import override -from detectmatelibrary.tools.logging import logger _DEFAULT_FREQ: dict[str, dict[str, int]] | None = None @@ -46,7 +36,7 @@ def _default_freq_tables() -> tuple[dict[str, dict[str, int]], dict[str, int]]: return _DEFAULT_FREQ, _DEFAULT_TOTAL_FREQ -class BigramFrequencyDetectorConfig(CoreDetectorConfig): +class BigramFrequencyDetectorConfig(VariableDetectorConfig): """ @param prob_thresh limit for the average probability of character pairs for which anomalies are reported. @param default_freqs initializes the probabilities with default values from @@ -60,71 +50,62 @@ class BigramFrequencyDetectorConfig(CoreDetectorConfig): default_freqs: bool = False skip_repetitions: bool = True - use_stable_vars: bool = True - use_static_vars: bool = True - -class BigramFrequencyDetector(CoreDetector): +class BigramFrequencyDetector(VariableDetector): """Detect bigram-frequency-based anomalies in log data.""" def __init__( self, name: str = "BigramFrequencyDetector", - config: BigramFrequencyDetectorConfig = BigramFrequencyDetectorConfig() + config: BigramFrequencyDetectorConfig = BigramFrequencyDetectorConfig(), ) -> None: if isinstance(config, dict): config = BigramFrequencyDetectorConfig.from_dict(config, name) - def add_value(cls: SingleStabilityTracker, value: Any) -> None: - """Add a new value to the tracker.""" + super().__init__(name=name, config=config) + self.config: BigramFrequencyDetectorConfig # type narrowing for IDE + + def add_value(self, tracker: SingleStabilityTracker, value: Any) -> None: + """Add a new value to the tracker (bigram-frequency semantics).""" + change = False + default_freq, default_total = (_default_freq_tables() if self.config.default_freqs else ({}, {})) + freq: dict[Any, dict[Any, int]] = tracker.extra_state.get("freq", {}) + total_freq: dict[Any, int] = tracker.extra_state.get("total_freq", {}) + probs: list[float] = [] + for i in range(-1, len(value)): + first: Any = -1 if i == -1 else value[i] + second: Any = -1 if i == len(value) - 1 else value[i + 1] + prob = 0.0 + if first in freq and second in freq[first] and total_freq.get(first, 0) > 0: + prob = freq[first][second] / total_freq[first] + elif self.config.default_freqs: + if (first in default_freq and second in default_freq[first] + and default_total.get(first, 0) > 0): + prob = default_freq[first][second] / default_total[first] + probs.append(prob) + if probs: + critical_val = sum(probs) / len(probs) + change = critical_val > self.config.prob_thresh or any(x == 0.0 for x in probs) + + if self.config.skip_repetitions and value in tracker.unique_set: change = False - default_freq, default_total = (_default_freq_tables() if self.config.default_freqs else ({}, {})) - freq: dict[Any, dict[Any, int]] = cls.extra_state.get("freq", {}) - total_freq: dict[Any, int] = cls.extra_state.get("total_freq", {}) - probs: list[float] = [] + else: for i in range(-1, len(value)): - first: Any = -1 if i == -1 else value[i] - second: Any = -1 if i == len(value) - 1 else value[i + 1] - prob = 0.0 - if first in freq and second in freq[first] and total_freq.get(first, 0) > 0: - prob = freq[first][second] / total_freq[first] - elif self.config.default_freqs: - if (first in default_freq and second in default_freq[first] - and default_total.get(first, 0) > 0): - prob = default_freq[first][second] / default_total[first] - probs.append(prob) - if probs: - critical_val = sum(probs) / len(probs) - change = critical_val > self.config.prob_thresh or any(x == 0.0 for x in probs) + first = -1 if i == -1 else value[i] + second = -1 if i == len(value) - 1 else value[i + 1] + row = freq.setdefault(first, {}) + row[second] = row.get(second, 0) + 1 + total_freq[first] = total_freq.get(first, 0) + 1 + tracker.unique_set.add(value) + tracker.change_series.append(change) - if self.config.skip_repetitions and value in cls.unique_set: - change = False - else: - for i in range(-1, len(value)): - first = -1 if i == -1 else value[i] - second = -1 if i == len(value) - 1 else value[i + 1] - row = freq.setdefault(first, {}) - row[second] = row.get(second, 0) + 1 - total_freq[first] = total_freq.get(first, 0) + 1 - cls.unique_set.add(value) - cls.change_series.append(change) - self.add_value_fn = add_value + def _event_data_kwargs(self) -> Optional[Dict[str, Any]]: + return self._stability_kwargs() - super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) - self.config: BigramFrequencyDetectorConfig # type narrowing for IDE - kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict( - method_id="BigramFrequencyDetector")} - self.persistency = EventPersistency( - event_data_class=EventStabilityTracker, - event_data_kwargs=kwargs + def _description(self) -> str: + return f"{self.name} anomalies in the bigram frequencies." - ) - # auto config checks if individual variables are stable - self.auto_conf_persistency = EventPersistency( - event_data_class=EventStabilityTracker, - event_data_kwargs=kwargs - ) - self._register_persistency(self.persistency) + # ---- training (overrides base: needs pre-ingest snapshot + freq update) -- def train(self, input_: ParserSchema) -> None: # type: ignore """Train the detector by updating per-variable bigram frequencies.""" @@ -207,42 +188,19 @@ def train_helper( row[second] = row.get(second, 0) + 1 total_freq[first] = total_freq.get(first, 0) + 1 - def detect( - self, input_: ParserSchema, output_: DetectorSchema # type: ignore - ) -> bool: - """Detect bigram-frequency anomalies in the input data.""" - alerts: dict[str, str] = {} - configured_variables = get_configured_variables(input_, self.config.events) - overall_score = 0.0 - current_event_id = input_["EventID"] - known_events = cast(dict[int | str, EventStabilityTracker], self.persistency.get_events_data()) - if current_event_id in known_events: - overall_score = self.detect_helper( - alerts, configured_variables, current_event_id, known_events, overall_score - ) - if self.config.global_instances and GLOBAL_EVENT_ID in known_events: - global_vars = get_global_variables(input_, self.config.global_instances) - overall_score = self.detect_helper( - alerts, global_vars, GLOBAL_EVENT_ID, known_events, overall_score - ) - if overall_score > 0: - output_["score"] = overall_score - output_["description"] = f"{self.name} anomalies in the bigram frequencies." - output_["alertsObtain"].update(alerts) - return True - return False + # ---- detection (overrides _check_event: event-level +1 scoring) ---------- - def detect_helper( + def _check_event( self, - alerts: dict[str, str], - variables: dict[str, Any], - event_id: "int | str", - known_events: "dict[int | str, EventStabilityTracker]", - overall_score: float, + alerts: Dict[str, str], + event_id: Any, + event_tracker: EventStabilityTracker, + variables: Dict[str, Any], + is_global: bool, ) -> float: anomaly = False default_freq, default_total = (_default_freq_tables() if self.config.default_freqs else ({}, {})) - var_trackers = cast(dict[str, SingleStabilityTracker], known_events[event_id].get_data()) + var_trackers = cast(dict[str, SingleStabilityTracker], event_tracker.get_data()) for var_name, single_tracker in var_trackers.items(): value: Any = variables.get(var_name) if value is None: @@ -265,55 +223,9 @@ def detect_helper( continue critical_val = sum(probs) / len(probs) if critical_val < self.config.prob_thresh: - k = f"EventID {event_id} - {var_name}" - if event_id == GLOBAL_EVENT_ID: - k = f"Global - {var_name}" - alerts[k] = ( + alerts[self._alert_key(event_id, var_name, is_global)] = ( f"Bigram frequency anomaly with value {value}, critical_val {critical_val} and " f"threshold {self.config.prob_thresh}." ) anomaly = True - if anomaly: - overall_score += 1.0 - return overall_score - - def configure(self, input_: ParserSchema) -> None: # type: ignore - self.auto_conf_persistency.ingest_event( - event_id=input_["EventID"], - event_template=input_["template"], - variables=input_["variables"], - named_variables=input_["logFormatVariables"], - ) - - @override - def post_train(self) -> None: - if not self.config.auto_config: - validate_config_coverage(self.name, self.config.events, self.persistency) - - def set_configuration(self) -> None: - variables = {} - for event_id, tracker in self.auto_conf_persistency.get_events_data().items(): - stable = [] - if self.config.use_stable_vars: - stable = tracker.get_features_by_classification("STABLE") # type: ignore - static = [] - if self.config.use_static_vars: - static = tracker.get_features_by_classification("STATIC") # type: ignore - vars_ = stable + static - if len(vars_) > 0: - variables[event_id] = vars_ - config_dict = generate_detector_config( - variable_selection=variables, - detector_name=self.name, - method_type=self.config.method_type, - ) - old_persist = self.config.persist - self.config = BigramFrequencyDetectorConfig.from_dict(config_dict, self.name) - self.config.persist = old_persist - events = self.config.events - if isinstance(events, EventsConfig) and not events.events: - logger.warning( - f"[{self.name}] auto_config=True generated an empty configuration. " - "No stable variables were found in configure-phase data. " - "The detector will produce no alerts." - ) + return 1.0 if anomaly else 0.0 diff --git a/src/detectmatelibrary/detectors/charset_detector.py b/src/detectmatelibrary/detectors/charset_detector.py index bf997d7a..c2356b5a 100644 --- a/src/detectmatelibrary/detectors/charset_detector.py +++ b/src/detectmatelibrary/detectors/charset_detector.py @@ -1,172 +1,45 @@ -from typing import Any -from detectmatelibrary.common._config._compile import generate_detector_config -from detectmatelibrary.common._config._formats import EventsConfig -from detectmatelibrary.common.detector import ( - CoreDetectorConfig, - CoreDetector, - get_configured_variables, - get_global_variables, - validate_config_coverage, -) +from detectmatelibrary.common.variable_detector import VariableDetector, VariableDetectorConfig from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import ( - EventStabilityTracker, - SingleStabilityTracker + SingleStabilityTracker, ) -from detectmatelibrary.utils.persistency.event_persistency import EventPersistency -from detectmatelibrary.utils.data_buffer import BufferMode -from detectmatelibrary.schemas import ParserSchema, DetectorSchema -from detectmatelibrary.constants import GLOBAL_EVENT_ID -from typing_extensions import override -from detectmatelibrary.tools.logging import logger +from typing import Any, Dict, Optional -class CharsetDetectorConfig(CoreDetectorConfig): - method_type: str = "charset_detector" - use_stable_vars: bool = True - use_static_vars: bool = True +class CharsetDetectorConfig(VariableDetectorConfig): + method_type: str = "charset_detector" -class CharsetDetector(CoreDetector): - """Detect new values in log data as anomalies based on learned values.""" +class CharsetDetector(VariableDetector): + """Detect characters in log data not seen in training as anomalies.""" def __init__( self, name: str = "CharsetDetector", - config: CharsetDetectorConfig = CharsetDetectorConfig() + config: CharsetDetectorConfig = CharsetDetectorConfig(), ) -> None: - if isinstance(config, dict): config = CharsetDetectorConfig.from_dict(config, name) - def add_value(cls: SingleStabilityTracker, value: Any) -> None: - """Add a new value to the tracker.""" - before = len(cls.unique_set) - cls.unique_set.update(value) - cls.change_series.append(len(cls.unique_set) > before) - self.add_value_fn = add_value - - super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) + super().__init__(name=name, config=config) self.config: CharsetDetectorConfig # type narrowing for IDE - kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict( - method_id="CharsetDetector")} - self.persistency = EventPersistency( - event_data_class=EventStabilityTracker, - event_data_kwargs=kwargs - ) - # auto config checks if individual variables are stable to select characters from - self.auto_conf_persistency = EventPersistency( - event_data_class=EventStabilityTracker, - event_data_kwargs=kwargs - ) - self._register_persistency(self.persistency) - - def train(self, input_: ParserSchema) -> None: # type: ignore - """Train the detector by learning characters from the input data.""" - configured_variables = get_configured_variables(input_, self.config.events) - self.persistency.ingest_event( - event_id=input_["EventID"], - event_template=input_["template"], - named_variables=configured_variables - ) - if self.config.global_instances: - global_vars = get_global_variables(input_, self.config.global_instances) - if global_vars: - self.persistency.ingest_event( - event_id=GLOBAL_EVENT_ID, - event_template=input_["template"], - named_variables=global_vars - ) - - def detect( - self, input_: ParserSchema, output_: DetectorSchema # type: ignore - ) -> bool: - """Detect characters in the input data that were not seen in - training.""" - alerts: dict[str, str] = {} - configured_variables = get_configured_variables(input_, self.config.events) - overall_score = 0.0 - - current_event_id = input_["EventID"] - known_events = self.persistency.get_events_data() - - if current_event_id in known_events: - event_tracker = known_events[current_event_id] - for var_name, single_tracker in event_tracker.get_data().items(): - v = configured_variables.get(var_name) - if v is None: - continue - unknown = set(v) - single_tracker.unique_set - if unknown: - alerts[f"EventID {current_event_id} - {var_name}"] = ( - "Unknown character(s): " - + ", ".join(f"'{c}'" for c in sorted(unknown)) - ) - overall_score += 1.0 - - if self.config.global_instances and GLOBAL_EVENT_ID in known_events: - global_vars = get_global_variables(input_, self.config.global_instances) - global_tracker = known_events[GLOBAL_EVENT_ID] - for var_name, single_tracker in global_tracker.get_data().items(): - v = global_vars.get(var_name) - if v is None: - continue - unknown = set(v) - single_tracker.unique_set - if unknown: - alerts[f"Global - {var_name}"] = ( - "Unknown character(s): " - + ", ".join(f"'{c}'" for c in sorted(unknown)) - ) - overall_score += 1.0 - - if overall_score > 0: - output_["score"] = overall_score - output_["description"] = ( - f"{self.name} detects characters not encountered in training as anomalies." - ) - output_["alertsObtain"].update(alerts) - return True - - return False - - def configure(self, input_: ParserSchema) -> None: # type: ignore - self.auto_conf_persistency.ingest_event( - event_id=input_["EventID"], - event_template=input_["template"], - variables=input_["variables"], - named_variables=input_["logFormatVariables"], - ) - - @override - def post_train(self) -> None: - if not self.config.auto_config: - validate_config_coverage(self.name, self.config.events, self.persistency) - def set_configuration(self) -> None: - variables = {} - for event_id, tracker in self.auto_conf_persistency.get_events_data().items(): - stable = [] - if self.config.use_stable_vars: - stable = tracker.get_features_by_classification("STABLE") # type: ignore - static = [] - if self.config.use_static_vars: - static = tracker.get_features_by_classification("STATIC") # type: ignore - vars_ = stable + static - if len(vars_) > 0: - variables[event_id] = vars_ - old_persist = self.config.persist - config_dict = generate_detector_config( - variable_selection=variables, - detector_name=self.name, - method_type=self.config.method_type, - ) - # Update the config object from the dictionary instead of replacing it - self.config = CharsetDetectorConfig.from_dict(config_dict, self.name) - self.config.persist = old_persist - events = self.config.events - if isinstance(events, EventsConfig) and not events.events: - logger.warning( - f"[{self.name}] auto_config=True generated an empty configuration. " - "No stable variables were found in configure-phase data. " - "The detector will produce no alerts." - ) + def add_value(self, tracker: SingleStabilityTracker, value: Any) -> None: + """Add a new value to the tracker (character-set semantics).""" + before = len(tracker.unique_set) + tracker.unique_set.update(value) + tracker.change_series.append(len(tracker.unique_set) > before) + + def _event_data_kwargs(self) -> Optional[Dict[str, Any]]: + return self._stability_kwargs() + + def _check_variable( + self, tracker: SingleStabilityTracker, value: Any, key: Any + ) -> Optional[str]: + unknown = set(value) - tracker.unique_set + if unknown: + return "Unknown character(s): " + ", ".join(f"'{c}'" for c in sorted(unknown)) + return None + + def _description(self) -> str: + return f"{self.name} detects characters not encountered in training as anomalies." diff --git a/src/detectmatelibrary/detectors/new_value_combo_detector.py b/src/detectmatelibrary/detectors/new_value_combo_detector.py index f732100d..e166250b 100644 --- a/src/detectmatelibrary/detectors/new_value_combo_detector.py +++ b/src/detectmatelibrary/detectors/new_value_combo_detector.py @@ -1,24 +1,18 @@ from detectmatelibrary.common._config import generate_detector_config from detectmatelibrary.common._config._formats import EventsConfig +from detectmatelibrary.common.variable_detector import VariableDetector, VariableDetectorConfig +from detectmatelibrary.common.detector import get_configured_variables -from detectmatelibrary.common.detector import ( - CoreDetectorConfig, - CoreDetector, - get_configured_variables, - get_global_variables, - validate_config_coverage, -) - -from detectmatelibrary.utils.data_buffer import BufferMode from detectmatelibrary.utils import persistency +from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import ( + SingleStabilityTracker, +) -from detectmatelibrary.schemas import ParserSchema, DetectorSchema -from detectmatelibrary.constants import GLOBAL_EVENT_ID +from detectmatelibrary.schemas import ParserSchema -from typing import Any, Dict, Sequence, cast, Tuple +from typing import Any, Dict, Optional, Sequence, Tuple, cast from itertools import combinations -from typing_extensions import override from detectmatelibrary.tools.logging import logger @@ -27,14 +21,11 @@ def get_combo(variables: Dict[str, Any]) -> Dict[Tuple[str, ...], Tuple[Any, ... return {tuple(variables.keys()): tuple(variables.values())} -def _combine( - iterable: Sequence[str], max_combo_length: int = 2 -) -> list[Tuple[str, ...]]: +def _combine(iterable: Sequence[str], max_combo_length: int = 2) -> list[Tuple[str, ...]]: """Get all possible combinations of an iterable.""" combos: list[Tuple[str, ...]] = [] for i in range(2, min(len(iterable), max_combo_length, 5) + 1): - combo = list(combinations(iterable, i)) - combos.extend(combo) + combos.extend(list(combinations(iterable, i))) return combos @@ -43,146 +34,74 @@ def get_all_possible_combos( ) -> Dict[Tuple[str, ...], Tuple[Any, ...]]: """Get all combinations of specified variables as key-value pairs.""" combo_dict = {} - combos = _combine(list(variables.keys()), max_combo_length) - for combo in combos: - key = tuple(combo) # Use tuple of variable names as key - value = tuple(variables[var] for var in combo) - combo_dict[key] = value + for combo in _combine(list(variables.keys()), max_combo_length): + combo_dict[tuple(combo)] = tuple(variables[var] for var in combo) return combo_dict -class NewValueComboDetectorConfig(CoreDetectorConfig): +class NewValueComboDetectorConfig(VariableDetectorConfig): method_type: str = "new_value_combo_detector" max_combo_size: int = 3 - use_stable_vars: bool = True use_static_vars: bool = False -class NewValueComboDetector(CoreDetector): +class NewValueComboDetector(VariableDetector): def __init__( self, name: str = "NewValueComboDetector", - config: NewValueComboDetectorConfig = NewValueComboDetectorConfig() + config: NewValueComboDetectorConfig = NewValueComboDetectorConfig(), ) -> None: - if isinstance(config, dict): config = NewValueComboDetectorConfig.from_dict(config, name) - super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) - - self.config = cast(NewValueComboDetectorConfig, self.config) - self.persistency = persistency.EventPersistency( - event_data_class=persistency.EventStabilityTracker, - event_data_kwargs={"converter_function": get_combo} - ) - # auto config checks if individual variables are stable to select combos from - self.auto_conf_persistency = persistency.EventPersistency( - event_data_class=persistency.EventStabilityTracker - ) + super().__init__(name=name, config=config) + self.config: NewValueComboDetectorConfig # type narrowing for IDE + # second-pass persistency to learn stability of variable combinations self.auto_conf_persistency_combos = persistency.EventPersistency( event_data_class=persistency.EventStabilityTracker, - event_data_kwargs={"converter_function": get_all_possible_combos} + event_data_kwargs={"converter_function": get_all_possible_combos}, ) self.inputs: list[ParserSchema] = [] - self._register_persistency(self.persistency) - - def train(self, input_: ParserSchema) -> None: # type: ignore - config = cast(NewValueComboDetectorConfig, self.config) - configured_variables = get_configured_variables(input_, config.events) - self.persistency.ingest_event( - event_id=input_["EventID"], - event_template=input_["template"], - named_variables=configured_variables - ) - if config.global_instances: - global_vars = get_global_variables(input_, config.global_instances) - if global_vars: - self.persistency.ingest_event( - event_id=GLOBAL_EVENT_ID, - event_template=input_["template"], - named_variables=global_vars - ) - - def detect( - self, input_: ParserSchema, output_: DetectorSchema # type: ignore - ) -> bool: - alerts: Dict[str, str] = {} - config = cast(NewValueComboDetectorConfig, self.config) - configured_variables = get_configured_variables(input_, config.events) - combo_dict = get_combo(configured_variables) - - overall_score = 0.0 - current_event_id = input_["EventID"] - known_events = self.persistency.get_events_data() - - if current_event_id in known_events: - event_tracker = known_events[current_event_id] - for combo_key, multi_tracker in event_tracker.get_data().items(): - value_tuple = combo_dict.get(combo_key) - if value_tuple is None: - continue - if value_tuple not in multi_tracker.unique_set: - alerts[f"EventID {current_event_id} - {combo_key}"] = ( - f"Unknown value combination: {value_tuple}" - ) - overall_score += 1.0 - - if config.global_instances and GLOBAL_EVENT_ID in known_events: - global_vars = get_global_variables(input_, config.global_instances) - global_combo_dict = get_combo(global_vars) - global_tracker = known_events[GLOBAL_EVENT_ID] - for combo_key, multi_tracker in global_tracker.get_data().items(): - value_tuple = global_combo_dict.get(combo_key) - if value_tuple is None: - continue - if value_tuple not in multi_tracker.unique_set: - alerts[f"Global - {combo_key}"] = f"Unknown value combination: {value_tuple}" - overall_score += 1.0 - - if overall_score > 0: - output_["score"] = overall_score - output_["description"] = ( - f"{self.name} detects value combinations not encountered " - "in training as anomalies." - ) - output_["alertsObtain"].update(alerts) - return True - return False - @override - def post_train(self) -> None: - config = cast(NewValueComboDetectorConfig, self.config) - if not config.auto_config: - validate_config_coverage(self.name, config.events, self.persistency) + def _event_data_kwargs(self) -> Optional[Dict[str, Any]]: + return {"converter_function": get_combo} - def configure(self, input_: ParserSchema) -> None: # type: ignore - """Configure the detector based on the stability of individual - variables, then learn value combinations based on that - configuration.""" - # store inputs to re-ingest after the first step of the configuration process - self.inputs.append(input_) + def _auto_conf_kwargs(self) -> Optional[Dict[str, Any]]: + return None # first-pass auto-config tracks individual variables + + def _prepare_variables(self, variables: Dict[str, Any], stage: str) -> Dict[str, Any]: + if stage == "detection": + return cast(Dict[str, Any], get_combo(variables)) + return variables - # first pass to learn variable stability of all variables - self.auto_conf_persistency.ingest_event( - event_id=input_["EventID"], - event_template=input_["template"], - variables=input_["variables"], - named_variables=input_["logFormatVariables"], + def _check_variable( + self, tracker: SingleStabilityTracker, value: Any, key: Any + ) -> Optional[str]: + if value not in tracker.unique_set: + return f"Unknown value combination: {value}" + return None + + def _description(self) -> str: + return ( + f"{self.name} detects value combinations not encountered " + "in training as anomalies." ) + def configure(self, input_: ParserSchema) -> None: # type: ignore + # store inputs to re-ingest after the first configuration pass + self.inputs.append(input_) + super().configure(input_) + def set_configuration(self, max_combo_size: int | None = None) -> None: - """Set the detector configuration based on the stability of variable - combinations. + """Set configuration based on the stability of variable combinations. - The process is as follows: - 1. Analyze the stability of individual variables to identify which are stable. + 1. Analyze individual-variable stability to identify stable variables. 2. Generate an initial config with combos of stable variables. - 3. Re-ingest all events to learn the stability of these combos (testing all possible combos right away - would explode combinatorially). + 3. Re-ingest all events to learn the stability of those combos (testing + every possible combo up front would explode combinatorially). """ - config = cast(NewValueComboDetectorConfig, self.config) - old_persist = config.persist - # run WITH auto_conf_persistency + old_persist = self.config.persist + # pass 1: stable individual variables -> combos variable_combos = {} for event_id, tracker in self.auto_conf_persistency.get_events_data().items(): stable_vars = tracker.get_features_by_classification("STABLE") # type: ignore @@ -192,40 +111,41 @@ def set_configuration(self, max_combo_size: int | None = None) -> None: variable_selection=variable_combos, detector_name=self.name, method_type=self.config.method_type, - max_combo_size=max_combo_size or config.max_combo_size + max_combo_size=max_combo_size or self.config.max_combo_size, ) - # Update the config object from the dictionary instead of replacing it self.config = NewValueComboDetectorConfig.from_dict(config_dict, self.name) - # Re-ingest all inputs to learn combos based on new configuration + # re-ingest all inputs to learn combos under the new configuration for input_ in self.inputs: configured_variables = get_configured_variables(input_, self.config.events) self.auto_conf_persistency_combos.ingest_event( event_id=input_["EventID"], event_template=input_["template"], - named_variables=configured_variables + named_variables=configured_variables, ) - # rerun to set final config WITH auto_conf_persistency_combos + # pass 2: stable/static combos -> final config combo_selection = {} for event_id, tracker in self.auto_conf_persistency_combos.get_events_data().items(): - stable_combos = [] - if self.config.use_stable_vars: - stable_combos = tracker.get_features_by_classification("STABLE") # type: ignore - static_combos = [] - if self.config.use_static_vars: - static_combos = tracker.get_features_by_classification("STATIC") # type: ignore + stable_combos = ( + tracker.get_features_by_classification("STABLE") # type: ignore + if self.config.use_stable_vars + else [] + ) + static_combos = ( + tracker.get_features_by_classification("STATIC") # type: ignore + if self.config.use_static_vars + else [] + ) combos = stable_combos + static_combos - # Keep combos as tuples - each will become a separate config entry - if len(combos) > 0: + if combos: combo_selection[event_id] = combos config_dict = generate_detector_config( variable_selection=combo_selection, detector_name=self.name, method_type=self.config.method_type, - max_combo_size=max_combo_size or self.config.max_combo_size + max_combo_size=max_combo_size or self.config.max_combo_size, ) - # Update the config object from the dictionary instead of replacing it self.config = NewValueComboDetectorConfig.from_dict(config_dict, self.name) self.config.persist = old_persist events = self.config.events diff --git a/src/detectmatelibrary/detectors/new_value_detector.py b/src/detectmatelibrary/detectors/new_value_detector.py index 390114fd..b704a7d2 100644 --- a/src/detectmatelibrary/detectors/new_value_detector.py +++ b/src/detectmatelibrary/detectors/new_value_detector.py @@ -1,150 +1,34 @@ -from detectmatelibrary.common._config._compile import generate_detector_config -from detectmatelibrary.common._config._formats import EventsConfig - -from detectmatelibrary.common.detector import ( - CoreDetectorConfig, - CoreDetector, - get_configured_variables, - get_global_variables, - validate_config_coverage, +from detectmatelibrary.common.variable_detector import VariableDetector, VariableDetectorConfig +from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import ( + SingleStabilityTracker, ) -from detectmatelibrary.utils import persistency -from detectmatelibrary.utils.data_buffer import BufferMode - -from detectmatelibrary.schemas import ParserSchema, DetectorSchema -from detectmatelibrary.constants import GLOBAL_EVENT_ID -from typing_extensions import override -from detectmatelibrary.tools.logging import logger +from typing import Any, Optional -class NewValueDetectorConfig(CoreDetectorConfig): +class NewValueDetectorConfig(VariableDetectorConfig): method_type: str = "new_value_detector" - use_stable_vars: bool = True - use_static_vars: bool = True - -class NewValueDetector(CoreDetector): +class NewValueDetector(VariableDetector): """Detect new values in log data as anomalies based on learned values.""" def __init__( self, name: str = "NewValueDetector", - config: NewValueDetectorConfig = NewValueDetectorConfig() + config: NewValueDetectorConfig = NewValueDetectorConfig(), ) -> None: - if isinstance(config, dict): config = NewValueDetectorConfig.from_dict(config, name) - - super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) + super().__init__(name=name, config=config) self.config: NewValueDetectorConfig # type narrowing for IDE - self.persistency = persistency.EventPersistency( - event_data_class=persistency.EventStabilityTracker, - ) - # auto config checks if individual variables are stable to select combos from - self.auto_conf_persistency = persistency.EventPersistency( - event_data_class=persistency.EventStabilityTracker - ) - self._register_persistency(self.persistency) - - def train(self, input_: ParserSchema) -> None: # type: ignore - """Train the detector by learning values from the input data.""" - configured_variables = get_configured_variables(input_, self.config.events) - self.persistency.ingest_event( - event_id=input_["EventID"], - event_template=input_["template"], - named_variables=configured_variables - ) - if self.config.global_instances: - global_vars = get_global_variables(input_, self.config.global_instances) - if global_vars: - self.persistency.ingest_event( - event_id=GLOBAL_EVENT_ID, - event_template=input_["template"], - named_variables=global_vars - ) - - def detect( - self, input_: ParserSchema, output_: DetectorSchema # type: ignore - ) -> bool: - """Detect new values in the input data.""" - alerts: dict[str, str] = {} - configured_variables = get_configured_variables(input_, self.config.events) - overall_score = 0.0 - - current_event_id = input_["EventID"] - known_events = self.persistency.get_events_data() - - if current_event_id in known_events: - event_tracker = known_events[current_event_id] - for var_name, multi_tracker in event_tracker.get_data().items(): - value = configured_variables.get(var_name) - if value is None: - continue - if value not in multi_tracker.unique_set: - alerts[f"EventID {current_event_id} - {var_name}"] = ( - f"Unknown value: '{value}'" - ) - overall_score += 1.0 - - if self.config.global_instances and GLOBAL_EVENT_ID in known_events: - global_vars = get_global_variables(input_, self.config.global_instances) - global_tracker = known_events[GLOBAL_EVENT_ID] - for var_name, multi_tracker in global_tracker.get_data().items(): - value = global_vars.get(var_name) - if value is None: - continue - if value not in multi_tracker.unique_set: - alerts[f"Global - {var_name}"] = f"Unknown value: '{value}'" - overall_score += 1.0 - - if overall_score > 0: - output_["score"] = overall_score - output_["description"] = f"{self.name} detects values not encountered in training as anomalies." - output_["alertsObtain"].update(alerts) - return True - - return False - - def configure(self, input_: ParserSchema) -> None: # type: ignore - self.auto_conf_persistency.ingest_event( - event_id=input_["EventID"], - event_template=input_["template"], - variables=input_["variables"], - named_variables=input_["logFormatVariables"], - ) - @override - def post_train(self) -> None: - if not self.config.auto_config: - validate_config_coverage(self.name, self.config.events, self.persistency) + def _check_variable( + self, tracker: SingleStabilityTracker, value: Any, key: Any + ) -> Optional[str]: + if value not in tracker.unique_set: + return f"Unknown value: '{value}'" + return None - def set_configuration(self) -> None: - variables = {} - for event_id, tracker in self.auto_conf_persistency.get_events_data().items(): - stable = [] - if self.config.use_stable_vars: - stable = tracker.get_features_by_classification("STABLE") # type: ignore - static = [] - if self.config.use_static_vars: - static = tracker.get_features_by_classification("STATIC") # type: ignore - vars_ = stable + static - if len(vars_) > 0: - variables[event_id] = vars_ - old_persist = self.config.persist - config_dict = generate_detector_config( - variable_selection=variables, - detector_name=self.name, - method_type=self.config.method_type, - ) - # Update the config object from the dictionary instead of replacing it - self.config = NewValueDetectorConfig.from_dict(config_dict, self.name) - self.config.persist = old_persist - events = self.config.events - if isinstance(events, EventsConfig) and not events.events: - logger.warning( - f"[{self.name}] auto_config=True generated an empty configuration. " - "No stable variables were found in configure-phase data. " - "The detector will produce no alerts." - ) + def _description(self) -> str: + return f"{self.name} detects values not encountered in training as anomalies." diff --git a/src/detectmatelibrary/detectors/value_range_detector.py b/src/detectmatelibrary/detectors/value_range_detector.py index e487c227..7f61cc81 100644 --- a/src/detectmatelibrary/detectors/value_range_detector.py +++ b/src/detectmatelibrary/detectors/value_range_detector.py @@ -1,205 +1,81 @@ -from detectmatelibrary.common._config._compile import generate_detector_config -from detectmatelibrary.common._config._formats import EventsConfig -from detectmatelibrary.common.detector import ( - CoreDetectorConfig, - CoreDetector, - get_configured_variables, - get_global_variables, - validate_config_coverage, -) +from typing import Any, Dict, Optional + +from detectmatelibrary.common.variable_detector import VariableDetector, VariableDetectorConfig from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import ( - EventStabilityTracker, - SingleStabilityTracker + SingleStabilityTracker, ) -from detectmatelibrary.utils.persistency.event_persistency import EventPersistency -from detectmatelibrary.utils.data_buffer import BufferMode -from detectmatelibrary.schemas import ParserSchema, DetectorSchema -from detectmatelibrary.constants import GLOBAL_EVENT_ID -from typing_extensions import override from detectmatelibrary.tools.logging import logger -from typing import Dict, List, Any -import sys -class ValueRangeDetectorConfig(CoreDetectorConfig): +class ValueRangeDetectorConfig(VariableDetectorConfig): method_type: str = "value_range_detector" ignore_non_numerical_val: bool = True - use_stable_vars: bool = True - use_static_vars: bool = True -class ValueRangeDetector(CoreDetector): - """Detect new value ranges in logs as anomalies based on learned values.""" +class ValueRangeDetector(VariableDetector): + """Detect out-of-range numeric values in logs based on learned min/max.""" def __init__( self, name: str = "ValueRangeDetector", - config: ValueRangeDetectorConfig = ValueRangeDetectorConfig() + config: ValueRangeDetectorConfig = ValueRangeDetectorConfig(), ) -> None: - if isinstance(config, dict): config = ValueRangeDetectorConfig.from_dict(config, name) - def add_value(cls: SingleStabilityTracker, value: int | float) -> None: - """Add a new value to the tracker.""" - try: - value = float(value) - value = int(value) if value.is_integer() else value - except ValueError: - return - if len(cls.unique_set) > 0: - min_ = min(cls.unique_set) - max_ = max(cls.unique_set) - cls.change_series.append(value < min_ or value > max_) - else: - cls.change_series.append(True) - cls.unique_set.add(value) - self.add_value_fn = add_value - - super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) + super().__init__(name=name, config=config) self.config: ValueRangeDetectorConfig # type narrowing for IDE - kwargs = {"add_value_fn": self.__class__.__name__, "detector_config": self.config.to_dict( - method_id="ValueRangeDetector")} - self.persistency = EventPersistency( - event_data_class=EventStabilityTracker, - event_data_kwargs=kwargs - ) - # auto config checks if individual variables are stable to select value ranges from - self.auto_conf_persistency = EventPersistency( - event_data_class=EventStabilityTracker, - event_data_kwargs=kwargs - ) - def cast_val_to_numeric(self, configured_variables: Dict[str, Any], k: str, remove: List[str], - stage: str) -> bool: - v = configured_variables[k] - if not isinstance(v, (int, float)): + def add_value(self, tracker: SingleStabilityTracker, value: Any) -> None: + """Add a new value to the tracker (range semantics).""" + try: + value = float(value) + value = int(value) if value.is_integer() else value + except ValueError: + return + if len(tracker.unique_set) > 0: + min_ = min(tracker.unique_set) + max_ = max(tracker.unique_set) + tracker.change_series.append(value < min_ or value > max_) + else: + tracker.change_series.append(True) + tracker.unique_set.add(value) + + def _event_data_kwargs(self) -> Optional[Dict[str, Any]]: + return self._stability_kwargs() + + def _prepare_variables(self, variables: Dict[str, Any], stage: str) -> Dict[str, Any]: + """Cast values to numeric; drop (or exit on) non-numeric ones.""" + remove = [] + for key in list(variables.keys()): + v = variables[key] + if isinstance(v, (int, float)): + continue try: - configured_variables[k] = float(v) - configured_variables[k] = int(configured_variables[k])\ - if configured_variables[k].is_integer() else configured_variables[k] + casted = float(v) + variables[key] = int(casted) if casted.is_integer() else casted except ValueError: - logger.error(f"Non-numeric value '{v}' appeared in {stage} of {self.__class__.__name__}" - f" with the name {self.name}.") - if not self.config.ignore_non_numerical_val: - sys.exit(1) - remove.append(k) - return False - return True - - def train(self, input_: ParserSchema) -> None: # type: ignore - """Train the detector by learning values from the input data.""" - configured_variables = get_configured_variables(input_, self.config.events) - remove: List[str] = [] - for k in configured_variables.keys(): - self.cast_val_to_numeric(configured_variables, k, remove, "training") - for k in remove: - del configured_variables[k] - self.persistency.ingest_event( - event_id=input_["EventID"], - event_template=input_["template"], - named_variables=configured_variables - ) - if self.config.global_instances: - global_vars = get_global_variables(input_, self.config.global_instances) - if global_vars: - self.persistency.ingest_event( - event_id=GLOBAL_EVENT_ID, - event_template=input_["template"], - named_variables=global_vars + msg = ( + f"Non-numeric value '{v}' appeared in {stage} of {type(self).__name__}" + f" with the name {self.name}." ) - - def detect( - self, input_: ParserSchema, output_: DetectorSchema # type: ignore - ) -> bool: - """Detect new value ranges in the input data.""" - alerts: dict[str, str] = {} - configured_variables = get_configured_variables(input_, self.config.events) - overall_score = 0.0 - - current_event_id = input_["EventID"] - known_events = self.persistency.get_events_data() - - if current_event_id in known_events: - event_tracker = known_events[current_event_id] - for var_name, multi_tracker in event_tracker.get_data().items(): - cast = self.cast_val_to_numeric(configured_variables, var_name, [], "detection") - value = configured_variables.get(var_name) - if value is None or not cast: - continue - min_ = min(multi_tracker.unique_set) - max_ = max(multi_tracker.unique_set) - if value < min_ or value > max_: - alerts[f"EventID {current_event_id} - {var_name}"] = ( - f"Out of range value: '{value}' ({min_} - {max_})" - ) - overall_score += 1.0 - - if self.config.global_instances and GLOBAL_EVENT_ID in known_events: - global_vars = get_global_variables(input_, self.config.global_instances) - global_tracker = known_events[GLOBAL_EVENT_ID] - for var_name, multi_tracker in global_tracker.get_data().items(): - cast = self.cast_val_to_numeric(global_vars, var_name, [], "detection") - value = global_vars.get(var_name) - if value is None or not cast: - continue - min_ = min(multi_tracker.unique_set) - max_ = max(multi_tracker.unique_set) - if isinstance(value, float): - min_ = float(min_) - max_ = float(max_) - else: - min_ = int(min_) - max_ = int(max_) - if value < min_ or value > max_: - alerts[f"Global - {var_name}"] = f"Out of range value: '{value}' ({min_} - {max_})" - overall_score += 1.0 - - if overall_score > 0: - output_["score"] = overall_score - output_["description"] = f"{self.name} detects values not encountered in training as anomalies." - output_["alertsObtain"].update(alerts) - return True - - return False - - def configure(self, input_: ParserSchema) -> None: # type: ignore - self.auto_conf_persistency.ingest_event( - event_id=input_["EventID"], - event_template=input_["template"], - variables=input_["variables"], - named_variables=input_["logFormatVariables"], - ) - - @override - def post_train(self) -> None: - if not self.config.auto_config: - validate_config_coverage(self.name, self.config.events, self.persistency) - - def set_configuration(self) -> None: - variables = {} - for event_id, tracker in self.auto_conf_persistency.get_events_data().items(): - stable = [] - if self.config.use_stable_vars: - stable = tracker.get_features_by_classification("STABLE") # type: ignore - static = [] - if self.config.use_static_vars: - static = tracker.get_features_by_classification("STATIC") # type: ignore - vars_ = stable + static - if len(vars_) > 0: - variables[event_id] = vars_ - config_dict = generate_detector_config( - variable_selection=variables, - detector_name=self.name, - method_type=self.config.method_type, - ) - # Update the config object from the dictionary instead of replacing it - self.config = ValueRangeDetectorConfig.from_dict(config_dict, self.name) - events = self.config.events - if isinstance(events, EventsConfig) and not events.events: - logger.warning( - f"[{self.name}] auto_config=True generated an empty configuration. " - "No stable variables were found in configure-phase data. " - "The detector will produce no alerts." - ) + if not self.config.ignore_non_numerical_val: + raise ValueError(msg) + logger.error(msg) + remove.append(key) + for key in remove: + del variables[key] + return variables + + def _check_variable( + self, tracker: SingleStabilityTracker, value: Any, key: Any + ) -> Optional[str]: + min_ = min(tracker.unique_set) + max_ = max(tracker.unique_set) + if value < min_ or value > max_: + return f"Out of range value: '{value}' ({min_} - {max_})" + return None + + def _description(self) -> str: + return f"{self.name} detects values not encountered in training as anomalies." diff --git a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py index c32ff8ac..72e79d58 100644 --- a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py +++ b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py @@ -1,6 +1,7 @@ """Tracks whether a variable is converging to a constant value.""" import importlib +from functools import partial from typing import Any, Callable, Dict, List, Literal, Set, TYPE_CHECKING from detectmatelibrary.utils.preview_helpers import list_preview_str from detectmatelibrary.utils.persistency.rle_list import RLEList @@ -11,6 +12,34 @@ from detectmatelibrary.common.detector import CoreDetectorConfig +def _strip_persist(detector_config: Any, method_id: str) -> Any: + """Return a copy of a serialized detector_config with its persist section + removed. + + A tracker reconstructs a throwaway detector from this config solely + to recover its add_value_fn closure. If the config still carries + persist, that throwaway instance starts a PersistencySaver -- + leaking a saver thread per variable and writing empty state over the + real detector's file. No add_value_fn closure reads persist, so + dropping it is behaviour-preserving. + """ + if not isinstance(detector_config, dict): + return detector_config + detectors = detector_config.get("detectors") + if not isinstance(detectors, dict): + return detector_config + entry = detectors.get(method_id) + if not (isinstance(entry, dict) and "persist" in entry): + return detector_config + return { + **detector_config, + "detectors": { + **detectors, + method_id: {k: v for k, v in entry.items() if k != "persist"}, + }, + } + + class SingleStabilityTracker(SingleTracker): """Tracks stability of a single feature.""" @@ -28,12 +57,12 @@ def __init__(self, min_samples: int = 3, add_value_fn: str = "default", self.add_value_fn = add_value_fn self.detector_config = detector_config if add_value_fn != "default": - detector = getattr(importlib.import_module("detectmatelibrary.detectors"), add_value_fn) + detector_cls = getattr(importlib.import_module("detectmatelibrary.detectors"), add_value_fn) if detector_config is not None: - detector = detector(config=detector_config) + detector = detector_cls(config=_strip_persist(detector_config, add_value_fn)) else: - detector = detector() - self.add_value = detector.add_value_fn.__get__(self, type(self)) # type: ignore[method-assign] + detector = detector_cls() + self.add_value = partial(detector.add_value, self) # type: ignore[method-assign] def add_value(self, value: Any) -> None: """Add a new value to the tracker.""" diff --git a/tests/test_detectors/test_bigram_frequency_detector.py b/tests/test_detectors/test_bigram_frequency_detector.py index 01f590b1..399b1bd8 100644 --- a/tests/test_detectors/test_bigram_frequency_detector.py +++ b/tests/test_detectors/test_bigram_frequency_detector.py @@ -8,9 +8,12 @@ - Input/output schema validation """ +from unittest.mock import patch +from detectmatelibrary.common.detector import PersistConfig from detectmatelibrary.detectors.bigram_frequency_detector import ( - BigramFrequencyDetector, BigramFrequencyDetectorConfig, BufferMode + BigramFrequencyDetector, BigramFrequencyDetectorConfig ) +from detectmatelibrary.utils.data_buffer import BufferMode from detectmatelibrary.common._core_op._fit_logic import EnumState from detectmatelibrary.constants import GLOBAL_EVENT_ID from detectmatelibrary.parsers.template_matcher import MatcherParser @@ -339,8 +342,6 @@ def test_global_instance_detects_new_type(self): class TestBigramFrequencyDetectorPersistencyRegistration: def test_register_persistency_is_called(self): """Persistency must be registered so `persist:` config takes effect.""" - from unittest.mock import patch - with patch.object( BigramFrequencyDetector, "_register_persistency", @@ -356,7 +357,6 @@ def test_register_persistency_is_called(self): class TestBigramFrequencyDetectorSetConfigurationPersist: def test_set_configuration_preserves_persist(self): """Auto-config must not silently drop the persist sub-config.""" - from detectmatelibrary.common.detector import PersistConfig detector = BigramFrequencyDetector() sentinel_persist = PersistConfig(path="/tmp/sentinel") detector.config.persist = sentinel_persist diff --git a/tests/test_detectors/test_charset_detector.py b/tests/test_detectors/test_charset_detector.py index a53fa2dd..7d113f20 100644 --- a/tests/test_detectors/test_charset_detector.py +++ b/tests/test_detectors/test_charset_detector.py @@ -8,7 +8,9 @@ - Input/output schema validation """ -from detectmatelibrary.detectors.charset_detector import CharsetDetector, CharsetDetectorConfig, BufferMode +from detectmatelibrary.common.detector import PersistConfig +from detectmatelibrary.detectors.charset_detector import CharsetDetector, CharsetDetectorConfig +from detectmatelibrary.utils.data_buffer import BufferMode from detectmatelibrary.common._core_op._fit_logic import EnumState from detectmatelibrary.constants import GLOBAL_EVENT_ID from detectmatelibrary.parsers.template_matcher import MatcherParser @@ -97,7 +99,6 @@ def test_persistency_uses_custom_add_value(self): def test_register_persistency_was_called(self): """Main persistency should be registered so persist/load round-trips work.""" - from detectmatelibrary.common.detector import PersistConfig cfg = CharsetDetectorConfig( persist=PersistConfig(path="memory://charset_regpersist/state") ) @@ -387,8 +388,6 @@ def test_global_instance_detects_new_type(self): class TestCharsetDetectorSetConfigurationPreservesPersist: def test_persist_flag_survives_set_configuration(self): - from detectmatelibrary.common.detector import PersistConfig - detector = CharsetDetector() # Simulate persist being enabled by an earlier config load detector.config.persist = PersistConfig(path="memory://persist_flag/state") diff --git a/tests/test_detectors/test_new_value_combo_detector.py b/tests/test_detectors/test_new_value_combo_detector.py index aeec8c18..5d5fcd79 100644 --- a/tests/test_detectors/test_new_value_combo_detector.py +++ b/tests/test_detectors/test_new_value_combo_detector.py @@ -1,5 +1,6 @@ -from detectmatelibrary.detectors.new_value_combo_detector import NewValueComboDetector, BufferMode +from detectmatelibrary.detectors.new_value_combo_detector import NewValueComboDetector +from detectmatelibrary.utils.data_buffer import BufferMode from detectmatelibrary.common._config import generate_detector_config from detectmatelibrary.parsers.template_matcher import MatcherParser from detectmatelibrary.helper.from_to import From diff --git a/tests/test_detectors/test_new_value_detector.py b/tests/test_detectors/test_new_value_detector.py index b66a4c11..62af1816 100644 --- a/tests/test_detectors/test_new_value_detector.py +++ b/tests/test_detectors/test_new_value_detector.py @@ -8,8 +8,8 @@ - Input/output schema validation """ -from detectmatelibrary.detectors.new_value_detector import NewValueDetector, NewValueDetectorConfig, \ - BufferMode +from detectmatelibrary.detectors.new_value_detector import NewValueDetector, NewValueDetectorConfig +from detectmatelibrary.utils.data_buffer import BufferMode from detectmatelibrary.common._core_op._fit_logic import EnumState from detectmatelibrary.constants import GLOBAL_EVENT_ID from detectmatelibrary.parsers.template_matcher import MatcherParser diff --git a/tests/test_detectors/test_value_range_detector.py b/tests/test_detectors/test_value_range_detector.py index ace3c604..052b9bf5 100644 --- a/tests/test_detectors/test_value_range_detector.py +++ b/tests/test_detectors/test_value_range_detector.py @@ -10,8 +10,13 @@ import logging import random import pytest -from detectmatelibrary.detectors.value_range_detector import (ValueRangeDetector, ValueRangeDetectorConfig, - BufferMode) +from detectmatelibrary.common.detector import PersistConfig +from detectmatelibrary.detectors.value_range_detector import ValueRangeDetector, ValueRangeDetectorConfig +from detectmatelibrary.utils.persistency import PersistencySaver +from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import ( + SingleStabilityTracker, +) +from detectmatelibrary.utils.data_buffer import BufferMode from detectmatelibrary.common._core_op._fit_logic import EnumState from detectmatelibrary.constants import GLOBAL_EVENT_ID from detectmatelibrary.parsers.template_matcher import MatcherParser @@ -82,6 +87,18 @@ def test_custom_config_initialization(self): assert hasattr(detector, 'persistency') assert isinstance(detector.persistency.events_data, dict) + def test_add_value_updates_tracker(self): + """add_value (now a method) applies range semantics to a tracker.""" + detector = ValueRangeDetector() + tracker = SingleStabilityTracker() + + detector.add_value(tracker, "5") # first value -> change + detector.add_value(tracker, "7") # within/extends range + detector.add_value(tracker, "nope") # non-numeric -> ignored + + assert tracker.unique_set == {5, 7} + assert list(tracker.change_series) == [True, True] + class TestValueRangeDetectorTraining: """Test ValueRangeDetector training functionality.""" @@ -137,8 +154,9 @@ def test_train_multiple_values(self): assert min(event_data["test"].unique_set) == min_val assert max(event_data["test"].unique_set) == max_val - def test_train_detect_non_numeric_exit(self): - """Test training with non-numeric values and not ignoring them.""" + def test_train_detect_non_numeric_raises(self): + """Non-numeric values raise ValueError when ignore_non_numerical_val is + False.""" detector = ValueRangeDetector(config=config, name="CustomInit") # Train with multiple values (the minimum and maximum value should be captured) @@ -153,9 +171,8 @@ def test_train_detect_non_numeric_exit(self): "log": "test log message", "logFormatVariables": {} }) - with pytest.raises(SystemExit) as excinfo: + with pytest.raises(ValueError): detector.train(parser_data) - assert excinfo.value.code == 1 normal_data = schemas.ParserSchema({ "parserType": "test", "EventID": 1, @@ -168,10 +185,9 @@ def test_train_detect_non_numeric_exit(self): "logFormatVariables": {} }) detector.train(normal_data) - with pytest.raises(SystemExit) as excinfo: + with pytest.raises(ValueError): output = schemas.DetectorSchema() detector.detect(parser_data, output) - assert excinfo.value.code == 1 def test_train_detect_non_numeric_ignore(self): """Test training with non-numeric values and ignoring them.""" @@ -424,3 +440,64 @@ def test_global_instance_detects_new_type(self): detected_ids.add(log["logID"]) assert len(detected_ids) > 0 + + +class TestValueRangeDetectorPersistFixes: + def test_registers_persistency_saver(self): + """The detector wires persistency into the base saver hook (was + missing).""" + detector = ValueRangeDetector( + config=ValueRangeDetectorConfig( + persist=PersistConfig(path="memory://value_range_regpersist/state") + ) + ) + # _register_persistency builds a PersistencySaver bound to detector.persistency + assert detector.saver is not None + assert detector.saver._persistency is detector.persistency + detector.saver.stop() + + def test_set_configuration_preserves_persist(self): + """set_configuration keeps the persist config (was dropped).""" + detector = ValueRangeDetector() + # Simulate persist being enabled by an earlier config load + detector.config.persist = PersistConfig(path="memory://value_range_persist_flag/state") + + # Feed configure() with a couple of stable-variable samples + for _ in range(5): + sample = schemas.ParserSchema({ + "parserType": "test", "EventID": 1, "template": "t", + "variables": ["123"], "logID": "x", "parsedLogID": "x", + "parserID": "p", "log": "l", + "logFormatVariables": {"level": "INFO"}, + }) + detector.configure(sample) + + detector.set_configuration() + + assert detector.config.persist is not None + assert detector.config.persist.path == "memory://value_range_persist_flag/state" + + def test_tracker_reconstruction_does_not_start_saver(self, monkeypatch): + """A tracker reconstructs a throwaway detector by name to recover its + add_value closure; that throwaway must not start a PersistencySaver + even when the serialized detector_config carries a persist section (it + would leak a saver thread per variable and clobber the real state + file).""" + starts: list = [] + monkeypatch.setattr(PersistencySaver, "start", lambda self: starts.append(self)) + + detector_config = ValueRangeDetectorConfig( + auto_config=False, + persist=PersistConfig(path="memory://value_range_recon_no_leak/state"), + ).to_dict(method_id="ValueRangeDetector") + # precondition: the serialized config carries a persist section + assert "persist" in detector_config["detectors"]["ValueRangeDetector"] + + tracker = SingleStabilityTracker( + add_value_fn="ValueRangeDetector", detector_config=detector_config + ) + # reconstruction recovered the range-semantics add_value closure... + tracker.add_value(5) + assert 5 in tracker.unique_set + # ...but started NO saver (the throwaway instance must not persist) + assert starts == [] diff --git a/tests/test_parsers/test_logbatcher_parser.py b/tests/test_parsers/test_logbatcher_parser.py index 24e09d5d..79aa5df2 100644 --- a/tests/test_parsers/test_logbatcher_parser.py +++ b/tests/test_parsers/test_logbatcher_parser.py @@ -9,6 +9,7 @@ import pytest import detectmatelibrary.schemas as schemas +from detectmatelibrary.common.parser import CoreParser from detectmatelibrary.parsers.logbatcher import LogBatcherParser, LogBatcherParserConfig from detectmatelibrary.utils.aux import time_test_mode @@ -32,7 +33,6 @@ def _make_parser(): class TestLogBatcherParserInit: def test_is_core_parser(self): - from detectmatelibrary.common.parser import CoreParser with patch("detectmatelibrary.parsers.logbatcher.engine.parser.OpenAI"): parser = LogBatcherParser(config=LogBatcherParserConfig(api_key="k")) assert isinstance(parser, CoreParser) diff --git a/tests/test_utils/test_persistency_saver.py b/tests/test_utils/test_persistency_saver.py index abcf9d98..a2b82a30 100644 --- a/tests/test_utils/test_persistency_saver.py +++ b/tests/test_utils/test_persistency_saver.py @@ -5,7 +5,10 @@ import fsspec import pytest -from detectmatelibrary.utils.persistency.event_data_structures.dataframes import EventDataFrame +from detectmatelibrary.utils.persistency.event_data_structures.dataframes import ( + EventDataFrame, + ChunkedEventDataFrame, +) from detectmatelibrary.utils.persistency.event_data_structures.trackers import EventStabilityTracker from detectmatelibrary.utils.persistency.event_persistency import EventPersistency from detectmatelibrary.utils.persistency.persistency_saver import ( @@ -164,8 +167,6 @@ def test_load_clears_stale_events_data(self): def test_load_restores_event_data_kwargs(self): """event_data_kwargs must be written back to ep after load.""" - from detectmatelibrary.utils.persistency.event_data_structures.dataframes import ChunkedEventDataFrame - p = EventPersistency( event_data_class=ChunkedEventDataFrame, event_data_kwargs={"max_rows": 500},