Feat/parser dp

docs/parsers.md

@@ -108,5 +108,6 @@ def test_my_parser_parse():
 - [Template Matcher](parsers/template_matcher.md): matches logs against a predefined set of `<*>` templates.
 - [Template Tree Matcher](parsers/template_tree_matcher.md): matches logs against a predefined set of `<*>` templates using a tree structure.
 - [LogBatcher Parser](parsers/logbatcher_parser.md): LLM-based parser that infers templates from raw logs with no training data.
+- [Drain parser](parsers/drain_parser.md): Parser inspired by [drain publication](https://ieeexplore.ieee.org/document/8029742).
 
 Go back to [Index](index.md)

docs/parsers/drain_parser.md

@@ -0,0 +1,109 @@
+# Drain parser
+
+The parsed is based in the official [Drain publication](https://ieeexplore.ieee.org/document/8029742).
+
+This parser wraps functionality from the DetectMatePerformance project: https://github.com/ait-detectmate/DetectMatePerformance. Prefer  use the performance implementation when parsing many log lines in non-stream (batch) mode.
+
+|            | Schema                     | Description        |
+|------------|----------------------------|--------------------|
+| **Input**  | [LogSchema](../schemas.md) | Unstructured log   |
+| **Output** | [ParserSchema](../schemas.md) | Structured log   |
+
+WARNING: This parser is not yet in a stable release and may behave differently across platforms or hardware.
+
+## Configuration
+
+Drain parser arguments:
+
+- `method_type` (string): parser type identifier (for example `"tree_matcher"`).
+- `depth` (int): Number of word layers.
+- `max_childs` (int): max number of childs allow in the length layer.
+- `sim_thres` (float): similarity threshold.
+- `reset_in_post_train` (bool): if true remove the logs in the train buffer when the templates are generated. Otherwise, it safe them for the next train.
+- `auto_config` (bool): whether to attempt an optional auto-configuration phase (not required).
+
+Example YAML fragment:
+```yaml
+parsers:
+  DrainParser:
+    method_type: drain_parser
+    auto_config: False
+    params:
+      depth: 2
+```
+
+## Usage example
+
+Simple usage (Reset = False):
+
+```python
+from detectmatelibrary.parsers.drain import DrainParser
+from detectmatelibrary import schemas
+
+# instantiate parser (config can be a dict or a config object)
+config_dict = {
+    "parsers": {
+        "DrainParser": {
+            "method_type": "drain_parser",
+            "data_use_training": 2,
+            "reset_in_post_train": False,
+        }
+    }
+}
+
+parser = DrainParser(config=config_dict)
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"}))
+print(parsed["template"])  # "templates not yet generated"
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+print(parsed["template"])  # "templates not yet generated"
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+print(parsed["template"])  # "hello there <*> kenobi"
+
+parser.update_state("keep_training")
+parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"}))
+parser.update_state("stop_training")
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+print(parsed["template"])  # "hello there <*> kenobi"
+```
+
+Simple usage (Reset = True):
+
+```python
+from detectmatelibrary.parsers.drain import DrainParser
+from detectmatelibrary import schemas
+
+# instantiate parser (config can be a dict or a config object)
+config_dict = {
+    "parsers": {
+        "DrainParser": {
+            "method_type": "drain_parser",
+            "data_use_training": 2,
+            "reset_in_post_train": False,
+        }
+    }
+}
+
+parser = DrainParser(config=config_dict)
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"}))
+print(parsed["template"])  # "templates not yet generated"
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+print(parsed["template"])  # "templates not yet generated"
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+print(parsed["template"])  # "hello there <*> kenobi"
+
+parser.update_state("keep_training")
+parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"}))
+parser.update_state("stop_training")
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+print(parsed["template"])  # "template not found"
+```
+
+Go back to [Index](../index.md)

mkdocs.yml

@@ -22,6 +22,7 @@ nav:
     - Template Tree Matcher: parsers/template_tree_matcher.md
     - Json Parser: parsers/json_parser.md
     - LogBatcher Parser: parsers/logbatcher_parser.md
+    - Drain Parser: parsers/drain_parser.md
   - Detectors Methods:
     - Random Detector: detectors/random_detector.md
     - New Value: detectors/new_value.md

pyproject.toml

@@ -13,7 +13,8 @@ dependencies = [
     "pyyaml>=6.0.3",
     "regex>=2025.11.3",
     "numpy>=2.3.2",
-    "detectmateperformance>=0.1.0",
+    #"detectmateperformance>=0.1.0",
+    "detectmateperformance @ git+https://github.com/ait-detectmate/DetectMatePerformance",
     "msgpack>=1.0.0",
     "fsspec>=2024.1.0",
     "pyarrow>=24.0.0",

src/detectmatelibrary/parsers/drain.py

@@ -0,0 +1,61 @@
+from detectmatelibrary.common.parser import CoreParser, CoreParserConfig
+from detectmatelibrary import schemas
+
+from detectmateperformance.match_tree import TreeMatcher
+from detectmateperformance.drain import Drain
+
+from typing import Any
+
+
+class DrainConfig(CoreParserConfig):
+    method_type: str = "drain_parser"
+
+    depth: int = 2
+    max_childs: int = 10
+    sim_thres: float = 0.2
+
+    reset_in_post_train: bool = False
+
+
+class DrainParser(CoreParser):
+    def __init__(
+        self,
+        name: str = "DrainParser",
+        config: DrainConfig | dict[str, Any] = DrainConfig()
+    ) -> None:
+
+        if isinstance(config, dict):
+            config = DrainConfig.from_dict(config, name)
+        super().__init__(name=name, config=config)
+
+        self.config: DrainConfig
+        self.drain_gen = Drain(
+            depth=self.config.depth,
+            max_child=self.config.max_childs,
+            sim=self.config.sim_thres,
+        )
+        self.tree_match: TreeMatcher | None = None
+
+    def train(self, input_: schemas.LogSchema) -> None:  # type: ignore
+        self.drain_gen.add(input_["log"])
+
+    def post_train(self) -> None:
+        self.tree_match = self.drain_gen.generate()
+        if self.config.reset_in_post_train:
+            self.drain_gen.reset()
+
+    def parse(
+        self,
+        input_: schemas.LogSchema,
+        output_: schemas.ParserSchema
+    ) -> None:
+
+        if self.tree_match is None:
+            output_["EventID"] = -1
+            output_["template"] = "templates not yet generated"
+        else:
+            parsed = self.tree_match.match_log(input_["log"], get_var=True)[0]
+
+            output_["EventID"] = parsed["EventID"]
+            output_["variables"].extend(parsed["ParamList"])
+            output_["template"] = parsed["Template"]

tests/test_parsers/test_drain.py

@@ -0,0 +1,96 @@
+"""Most of the functionality is test it in DetectMatePerformance."""
+from detectmatelibrary.parsers.drain import DrainParser
+from detectmatelibrary import schemas
+
+
+class TestDrainParser:
+    def test_train_process(self):
+        config_dict = {
+            "parsers": {
+                "DrainParser": {
+                    "method_type": "drain_parser",
+                    "data_use_training": 2,
+                }
+            }
+        }
+        parser = DrainParser(config=config_dict)
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "templates not yet generated"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "templates not yet generated"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+        assert parsed["EventID"] == 0
+        assert parsed["template"] == "hello there <*> kenobi"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, general R2D2!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "template not found"
+
+    def test_reset_after_train(self):
+        config_dict = {
+            "parsers": {
+                "DrainParser": {
+                    "method_type": "drain_parser",
+                    "data_use_training": 2,
+                    "reset_in_post_train": True,
+                }
+            }
+        }
+        parser = DrainParser(config=config_dict)
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "templates not yet generated"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "templates not yet generated"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+        assert parsed["EventID"] == 0
+        assert parsed["template"] == "hello there <*> kenobi"
+
+        parser.update_state("keep_training")
+        parsed = parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"}))
+        parser.update_state("stop_training")
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "template not found"
+
+    def test_not_reset_train(self):
+        config_dict = {
+            "parsers": {
+                "DrainParser": {
+                    "method_type": "drain_parser",
+                    "data_use_training": 2,
+                    "reset_in_post_train": False,
+                }
+            }
+        }
+        parser = DrainParser(config=config_dict)
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "templates not yet generated"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "templates not yet generated"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+        assert parsed["EventID"] == 0
+        assert parsed["template"] == "hello there <*> kenobi"
+
+        parser.update_state("keep_training")
+        parsed = parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"}))
+        parser.update_state("stop_training")
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+        assert parsed["EventID"] == 1
+        assert parsed["template"] == "hello there <*> kenobi"