Parse process

CMakeLists.txt

@@ -79,6 +79,12 @@ add_library(
   src/detectmateperformance/_core/template_matcher/match_tree.cpp
 )
 
+add_library(
+  drain_op
+  src/detectmateperformance/_core/parsers/drain.h
+  src/detectmateperformance/_core/parsers/drain.cpp
+)
+
 # match_tree depends on variable and tree_op; make those transitive so
 # consumers of match_tree (executables/tests) don't need to worry about
 # static library link ordering.
@@ -88,6 +94,7 @@ target_link_libraries(
   variable
   tree_op
   tree
+  drain_op
 )
 
 add_executable(
@@ -110,6 +117,11 @@ add_executable(
   tests/test_c/test_matcher_vars.cpp
 )
 
+add_executable(
+  test_drain_op
+  tests/test_c/test_drain_op.cpp
+)
+
 pybind11_add_module(
   bind_class
   src/detectmateperformance/_core/bind_class.cpp
@@ -129,6 +141,8 @@ pybind11_add_module(
   src/detectmateperformance/_core/template_matcher/tree_op.cpp
   src/detectmateperformance/_core/template_matcher/match_tree.h
   src/detectmateperformance/_core/template_matcher/match_tree.cpp
+  src/detectmateperformance/_core/parsers/drain.h
+  src/detectmateperformance/_core/parsers/drain.cpp
 )
 
 target_link_libraries(
@@ -172,8 +186,23 @@ target_link_libraries(
   aux
 )
 
+target_link_libraries(
+  test_drain_op
+  GTest::gtest_main
+  drain_op
+  templates
+  parsedelement
+  parsed
+  tree
+  variable
+  tree_op
+  match_tree
+  aux
+)
+
 include(GoogleTest)
 gtest_discover_tests(test_c)
 gtest_discover_tests(test_type)
 gtest_discover_tests(test_matcher_tree)
 gtest_discover_tests(test_matcher_vars)
+gtest_discover_tests(test_drain_op)

docs/drain.md

@@ -0,0 +1,43 @@
+
+# Drain
+
+This parsed implementation was based from this [publication](https://ieeexplore.ieee.org/document/8029742).
+
+```python
+class Drain:
+    def __init__(
+        self, depth: int = 2, max_child: int = 10, sim: float = 0.5
+    ) -> None:
+        pass
+
+    def __len__(self) -> int:
+       """Buffer size"""
+
+    def add(self, log: str) -> None:
+        """Add log to buffer"""
+
+    def reset(self) -> None:
+        """Reset train buffer"""
+
+    def generate(self) -> TreeMatcher:
+        """Generate Tree matcher"""
+
+    def generate_from_df(self, df: pl.DataFrame) -> TreeMatcher:
+        """Generate Tree matcher from df"""
+```
+
+
+## Usage
+
+```python
+from detectmateperformanc.drain import Drain
+
+
+rain_ = drain.Drain(depth=1, sim=0.3)
+
+drain_.add("hello tobias my man")
+drain_.add("hello guillermo my man")
+drain_.add("hello julie my woman")
+
+tree_match = drain_.generate()
+```

docs/tree_matcher.md

@@ -13,6 +13,9 @@ class TreeMatcher:
     def match_log(self, log: str, get_var: bool = False) -> ParsedLogs:
         """Match one preprocess log with the templates"""
 
+    def get_templates(self, do_print: bool = True) -> list[str]:
+        """Get the templates"""
+
     def match_batch(
         self, logs: list[str], get_var: bool = False, n_workers: int = 1
     ) -> ParsedLogs:

docs/types.md

@@ -17,6 +17,8 @@ class LogTemplates:
 
     def get_next(self) -> list[str]: pass
 
+    def get_templates(self, do_print: bool = True) -> list[str]: pass
+
     def __eq__(self, other: object) -> bool: pass
 
     def __str__(self) -> str: pass

mkdocs.yml

@@ -8,6 +8,7 @@ nav:
   - Home: index.md
   - Methods:
       - TreeMatcher: tree_matcher.md
+      - Drain: drain.md
   - Auxiliar:
       - Types: types.md
       - Metrics: metrics.md

src/detectmateperformance/_core/bind_class.cpp

@@ -1,6 +1,7 @@
 #include <pybind11/pybind11.h>
 #include <pybind11/stl.h>
 
+#include "parsers/drain.h"
 #include "template_matcher/match_tree.h"
 #include "_type/element.h"
 #include "_type/templates.h"
@@ -11,15 +12,19 @@ namespace py = pybind11;
 
 
 PYBIND11_MODULE(bind_class, m) {
+    m.def("drain_generator", &drain_generator);
+
     py::class_<Templates>(m, "Templates")
         .def(py::init<std::deque<std::string>>())
         .def("get_next_template", &Templates::getNext)
         .def("size", &Templates::size)
         .def("shape", &Templates::shape);
+
     py::class_<ParsedElement>(m, "ParsedElement")
         .def_readonly("event_id", &ParsedElement::event_id)
         .def_readonly("log_template", &ParsedElement::log_template)
         .def_readonly("variables", &ParsedElement::variables);
+
     py::class_<ParsedMessages>(m, "Parsed")
         .def(py::init<Templates*, int>())
         .def("get_elem_id", &ParsedMessages::getElemID)
@@ -32,6 +37,7 @@ PYBIND11_MODULE(bind_class, m) {
         .def("get_all_var", &ParsedMessages::getAllVar)
         .def("size", &ParsedMessages::size)
         .def("shape", &ParsedMessages::shape);
+
     py::class_<MatchTree>(m, "MatchTree")
         .def(py::init<Templates*>())
         .def("match_string", &MatchTree::match_string)

src/detectmateperformance/_core/parsers/drain.cpp

@@ -0,0 +1,170 @@
+#include "drain.h"
+
+#include <utility>
+#include <string>
+#include <vector>
+#include <sstream>
+#include <algorithm>
+#include <regex>
+#include <unordered_set>
+
+
+/////// Support methods
+
+std::vector<std::string> split_sentence(std::string sentence) {
+
+    std::istringstream iss(sentence);
+    std::vector<std::string> words;
+    std::string word;
+
+    while (iss >> word) {
+        words.push_back(word);
+    }
+
+    return words;
+}
+
+
+std::string join_sentence(std::vector<std::string> words) {
+    std::ostringstream oss;
+
+    for (size_t i = 0; i < words.size(); ++i) {
+        if (i != 0) oss << " ";
+        oss << words[i];
+    }
+
+    return oss.str();
+}
+
+/////// Main methods
+
+float calculate_sim(std::string sentence_1, std::string sentence_2) {
+    int n = sentence_1.size();
+    if (sentence_2.size() < n)
+        n = sentence_2.size();
+
+    float sims = 0.0;
+    for (int i = 0; i < n; i++) {
+        if (sentence_1[i] == sentence_2[i])
+            sims ++;
+    }
+
+    return sims / n;
+}
+
+
+std::string generate_template(std::deque<std::string> sentences) {
+    if (sentences.empty()) return "";
+
+    std::vector<std::vector<std::string>> splitSentences;
+    int m = 0;
+    for (size_t i = 0; i < sentences.size(); i++) {
+        splitSentences.push_back(split_sentence(sentences[i]));
+
+        if (i == 0 || splitSentences[i].size() < m){
+            m = splitSentences[i].size();
+        }
+    }
+
+    std::vector<std::string> words;
+    for (size_t j = 0; j < m; ++j) {
+        std::string aux = splitSentences[0][j];
+        for (size_t i = 1; i < splitSentences.size(); ++i) {
+            if (splitSentences[i][j] != aux) {
+                aux = "VAR";
+                break;
+            }
+        }
+        words.push_back(aux);
+    }
+
+    return join_sentence(words);
+}
+
+
+std::deque<std::string> generate_templates(
+    std::vector<std::deque<std::string>> sentences, float simSeq
+) {
+
+    std::deque<std::string> queueSent;
+    std::deque<std::string> queueSentCopy;
+
+    std::deque<std::string> templates;
+    std::deque<std::string> similar;
+    std::string template_ = "";
+
+    for (size_t i = 0; i < sentences.size(); i++) {
+        queueSent = sentences[i];
+
+        while (queueSent.size() > 0) {
+            template_ = queueSent[0];
+            queueSent.pop_front();
+
+            if (queueSent.size() == 0) {
+                templates.push_back(template_);
+
+            } else {
+
+                similar = {template_};
+                queueSentCopy = {};
+
+                for (size_t j = 0; j < queueSent.size(); j++) {
+                    if (calculate_sim(template_, queueSent[j]) > simSeq) {
+                        similar.push_back(queueSent[j]);
+                    } else {
+                        queueSentCopy.push_back(queueSent[j]);
+                    }
+
+                }
+
+                queueSent = queueSentCopy;
+                templates.push_back(generate_template(similar));
+            }
+        }
+    }
+
+    return templates;
+}
+
+
+std::deque<std::string> clean_templates(std::deque<std::string> templates) {
+
+    // Change "Hello VAR VAR" to "Hello VAR"
+    std::regex pattern("\\s*VAR\\s*VAR\\s*");
+    for (size_t i = 0; i < templates.size(); i++) {
+        templates[i] = std::regex_replace(templates[i], pattern, " VAR ");
+    }
+
+    // Erase templates "VAR"
+    auto it = std::find(templates.begin(), templates.end(), "VAR");
+    if (it != templates.end()) {
+        templates.erase(it);
+    }
+
+
+    // Remove duplicate templates
+    std::unordered_set<std::string> seen;
+    std::deque<std::string> uniqueTemplates;
+
+    for (const auto& word : templates) {
+        if (seen.find(word) == seen.end()) {
+            seen.insert(word);
+            uniqueTemplates.push_back(word);
+        }
+    }
+
+    return uniqueTemplates;
+}
+
+
+Templates drain_generator(
+    std::vector<std::deque<std::string>> sentences, float SimSeq
+) {
+
+    std::deque<std::string> templates = generate_templates(sentences, SimSeq);
+    templates = clean_templates(templates);
+
+    Templates temp_instance = Templates(templates);
+
+    return temp_instance;
+}

src/detectmateperformance/_core/parsers/drain.h

@@ -0,0 +1,27 @@
+#ifndef M_DRAIN_H
+#define M_DRAIN_H
+
+#include <vector>
+#include <utility>
+#include <string>
+#include <deque>
+
+#include "../_type/templates.h"
+
+
+float calculate_sim(std::string sentence_1, std::string sentence_2);
+
+std::string generate_template(std::deque<std::string> sentences);
+
+std::deque<std::string> generate_templates(
+    std::vector<std::deque<std::string>> sentences, float simSeq
+);
+
+std::deque<std::string> clean_templates(std::deque<std::string> templates);
+
+
+Templates drain_generator(
+    std::vector<std::deque<std::string>> sentences, float SimSeq
+);
+
+#endif