diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a9b5f8..9a2db1d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,6 +79,12 @@ add_library( src/detectmateperformance/_core/template_matcher/match_tree.cpp ) +add_library( + drain_op + src/detectmateperformance/_core/parsers/drain.h + src/detectmateperformance/_core/parsers/drain.cpp +) + # match_tree depends on variable and tree_op; make those transitive so # consumers of match_tree (executables/tests) don't need to worry about # static library link ordering. @@ -88,6 +94,7 @@ target_link_libraries( variable tree_op tree + drain_op ) add_executable( @@ -110,6 +117,11 @@ add_executable( tests/test_c/test_matcher_vars.cpp ) +add_executable( + test_drain_op + tests/test_c/test_drain_op.cpp +) + pybind11_add_module( bind_class src/detectmateperformance/_core/bind_class.cpp @@ -129,6 +141,8 @@ pybind11_add_module( src/detectmateperformance/_core/template_matcher/tree_op.cpp src/detectmateperformance/_core/template_matcher/match_tree.h src/detectmateperformance/_core/template_matcher/match_tree.cpp + src/detectmateperformance/_core/parsers/drain.h + src/detectmateperformance/_core/parsers/drain.cpp ) target_link_libraries( @@ -172,8 +186,23 @@ target_link_libraries( aux ) +target_link_libraries( + test_drain_op + GTest::gtest_main + drain_op + templates + parsedelement + parsed + tree + variable + tree_op + match_tree + aux +) + include(GoogleTest) gtest_discover_tests(test_c) gtest_discover_tests(test_type) gtest_discover_tests(test_matcher_tree) gtest_discover_tests(test_matcher_vars) +gtest_discover_tests(test_drain_op) diff --git a/docs/drain.md b/docs/drain.md new file mode 100644 index 0000000..c40b299 --- /dev/null +++ b/docs/drain.md @@ -0,0 +1,43 @@ + +# Drain + +This parsed implementation was based from this [publication](https://ieeexplore.ieee.org/document/8029742). + +```python +class Drain: + def __init__( + self, depth: int = 2, max_child: int = 10, sim: float = 0.5 + ) -> None: + pass + + def __len__(self) -> int: + """Buffer size""" + + def add(self, log: str) -> None: + """Add log to buffer""" + + def reset(self) -> None: + """Reset train buffer""" + + def generate(self) -> TreeMatcher: + """Generate Tree matcher""" + + def generate_from_df(self, df: pl.DataFrame) -> TreeMatcher: + """Generate Tree matcher from df""" +``` + + +## Usage + +```python +from detectmateperformanc.drain import Drain + + +rain_ = drain.Drain(depth=1, sim=0.3) + +drain_.add("hello tobias my man") +drain_.add("hello guillermo my man") +drain_.add("hello julie my woman") + +tree_match = drain_.generate() +``` diff --git a/docs/tree_matcher.md b/docs/tree_matcher.md index d39b5e1..4b45580 100644 --- a/docs/tree_matcher.md +++ b/docs/tree_matcher.md @@ -13,6 +13,9 @@ class TreeMatcher: def match_log(self, log: str, get_var: bool = False) -> ParsedLogs: """Match one preprocess log with the templates""" + def get_templates(self, do_print: bool = True) -> list[str]: + """Get the templates""" + def match_batch( self, logs: list[str], get_var: bool = False, n_workers: int = 1 ) -> ParsedLogs: diff --git a/docs/types.md b/docs/types.md index dab03cb..e4f0236 100644 --- a/docs/types.md +++ b/docs/types.md @@ -17,6 +17,8 @@ class LogTemplates: def get_next(self) -> list[str]: pass + def get_templates(self, do_print: bool = True) -> list[str]: pass + def __eq__(self, other: object) -> bool: pass def __str__(self) -> str: pass diff --git a/mkdocs.yml b/mkdocs.yml index 3292d3b..df5aa47 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,6 +8,7 @@ nav: - Home: index.md - Methods: - TreeMatcher: tree_matcher.md + - Drain: drain.md - Auxiliar: - Types: types.md - Metrics: metrics.md diff --git a/src/detectmateperformance/_core/bind_class.cpp b/src/detectmateperformance/_core/bind_class.cpp index 35c5b3e..bb23a63 100644 --- a/src/detectmateperformance/_core/bind_class.cpp +++ b/src/detectmateperformance/_core/bind_class.cpp @@ -1,6 +1,7 @@ #include #include +#include "parsers/drain.h" #include "template_matcher/match_tree.h" #include "_type/element.h" #include "_type/templates.h" @@ -11,15 +12,19 @@ namespace py = pybind11; PYBIND11_MODULE(bind_class, m) { + m.def("drain_generator", &drain_generator); + py::class_(m, "Templates") .def(py::init>()) .def("get_next_template", &Templates::getNext) .def("size", &Templates::size) .def("shape", &Templates::shape); + py::class_(m, "ParsedElement") .def_readonly("event_id", &ParsedElement::event_id) .def_readonly("log_template", &ParsedElement::log_template) .def_readonly("variables", &ParsedElement::variables); + py::class_(m, "Parsed") .def(py::init()) .def("get_elem_id", &ParsedMessages::getElemID) @@ -32,6 +37,7 @@ PYBIND11_MODULE(bind_class, m) { .def("get_all_var", &ParsedMessages::getAllVar) .def("size", &ParsedMessages::size) .def("shape", &ParsedMessages::shape); + py::class_(m, "MatchTree") .def(py::init()) .def("match_string", &MatchTree::match_string) diff --git a/src/detectmateperformance/_core/parsers/drain.cpp b/src/detectmateperformance/_core/parsers/drain.cpp new file mode 100644 index 0000000..699aeb5 --- /dev/null +++ b/src/detectmateperformance/_core/parsers/drain.cpp @@ -0,0 +1,170 @@ +#include "drain.h" + +#include +#include +#include +#include +#include +#include +#include + + +/////// Support methods + +std::vector split_sentence(std::string sentence) { + + std::istringstream iss(sentence); + std::vector words; + std::string word; + + while (iss >> word) { + words.push_back(word); + } + + return words; +} + + +std::string join_sentence(std::vector words) { + std::ostringstream oss; + + for (size_t i = 0; i < words.size(); ++i) { + if (i != 0) oss << " "; + oss << words[i]; + } + + return oss.str(); +} + +/////// Main methods + +float calculate_sim(std::string sentence_1, std::string sentence_2) { + int n = sentence_1.size(); + if (sentence_2.size() < n) + n = sentence_2.size(); + + float sims = 0.0; + for (int i = 0; i < n; i++) { + if (sentence_1[i] == sentence_2[i]) + sims ++; + } + + return sims / n; +} + + +std::string generate_template(std::deque sentences) { + if (sentences.empty()) return ""; + + std::vector> splitSentences; + int m = 0; + for (size_t i = 0; i < sentences.size(); i++) { + splitSentences.push_back(split_sentence(sentences[i])); + + if (i == 0 || splitSentences[i].size() < m){ + m = splitSentences[i].size(); + } + } + + std::vector words; + for (size_t j = 0; j < m; ++j) { + std::string aux = splitSentences[0][j]; + for (size_t i = 1; i < splitSentences.size(); ++i) { + if (splitSentences[i][j] != aux) { + aux = "VAR"; + break; + } + } + words.push_back(aux); + } + + return join_sentence(words); +} + + +std::deque generate_templates( + std::vector> sentences, float simSeq +) { + + std::deque queueSent; + std::deque queueSentCopy; + + std::deque templates; + std::deque similar; + std::string template_ = ""; + + for (size_t i = 0; i < sentences.size(); i++) { + queueSent = sentences[i]; + + while (queueSent.size() > 0) { + template_ = queueSent[0]; + queueSent.pop_front(); + + if (queueSent.size() == 0) { + templates.push_back(template_); + + } else { + + similar = {template_}; + queueSentCopy = {}; + + for (size_t j = 0; j < queueSent.size(); j++) { + if (calculate_sim(template_, queueSent[j]) > simSeq) { + similar.push_back(queueSent[j]); + } else { + queueSentCopy.push_back(queueSent[j]); + } + + } + + queueSent = queueSentCopy; + templates.push_back(generate_template(similar)); + } + } + } + + return templates; +} + + +std::deque clean_templates(std::deque templates) { + + // Change "Hello VAR VAR" to "Hello VAR" + std::regex pattern("\\s*VAR\\s*VAR\\s*"); + for (size_t i = 0; i < templates.size(); i++) { + templates[i] = std::regex_replace(templates[i], pattern, " VAR "); + } + + // Erase templates "VAR" + auto it = std::find(templates.begin(), templates.end(), "VAR"); + if (it != templates.end()) { + templates.erase(it); + } + + + // Remove duplicate templates + std::unordered_set seen; + std::deque uniqueTemplates; + + for (const auto& word : templates) { + if (seen.find(word) == seen.end()) { + seen.insert(word); + uniqueTemplates.push_back(word); + } + } + + return uniqueTemplates; +} + + +Templates drain_generator( + std::vector> sentences, float SimSeq +) { + + std::deque templates = generate_templates(sentences, SimSeq); + templates = clean_templates(templates); + + Templates temp_instance = Templates(templates); + + return temp_instance; +} diff --git a/src/detectmateperformance/_core/parsers/drain.h b/src/detectmateperformance/_core/parsers/drain.h new file mode 100644 index 0000000..3b05a0a --- /dev/null +++ b/src/detectmateperformance/_core/parsers/drain.h @@ -0,0 +1,27 @@ +#ifndef M_DRAIN_H +#define M_DRAIN_H + +#include +#include +#include +#include + +#include "../_type/templates.h" + + +float calculate_sim(std::string sentence_1, std::string sentence_2); + +std::string generate_template(std::deque sentences); + +std::deque generate_templates( + std::vector> sentences, float simSeq +); + +std::deque clean_templates(std::deque templates); + + +Templates drain_generator( + std::vector> sentences, float SimSeq +); + +#endif diff --git a/src/detectmateperformance/drain.py b/src/detectmateperformance/drain.py new file mode 100644 index 0000000..0320f54 --- /dev/null +++ b/src/detectmateperformance/drain.py @@ -0,0 +1,82 @@ +from detectmateperformance.match_tree import TreeMatcher +from detectmateperformance.types_ import LogTemplates + +from detectmateperformance.lib.bind_class import drain_generator + +import polars as pl +import warnings + + +def cluster_logs_df(df: pl.DataFrame, depth: int = 2, max_child: int = 10) -> dict[str, list[str]]: + df = df.with_columns(pl.col("Content").str.replace_all(r"[^a-zA-Z0-9\s]", " ")) + df = df.with_columns(pl.col("Content").str.replace_all(r"\d+", "VAR").str.replace_all(r"\s+", " ")) + df = df.unique() + + df = df.insert_column(-1, pl.col("Content").str.split(by=" ").list.len().alias("L")) + df = df.with_columns( + pl.when(pl.col("L") > max_child).then(max_child).otherwise(pl.col.L).alias("Length") + ).drop("L") + + cols = ["Length"] + for i in range(depth): + cols.append(f"Depth {i}") + df = df.insert_column( + -1, pl.col("Content").str.split(by=" ").list.get(i, null_on_oob=True).alias(cols[-1]) + ) + + df = df.fill_null("None") + df = df.with_columns(concat_list=pl.concat_list(cols).list.join("_")) + df = df.drop(cols) + + dict_ = df.group_by("concat_list").agg(pl.col("Content")).rows_by_key("concat_list") + return {k: v[0][0] for k, v in dict_.items()} + + +class Drain: + def __init__( + self, depth: int = 2, max_child: int = 10, sim: float = 0.5 + ) -> None: + + self.depth = depth + self.max_child = max_child + self.sim = sim + warnings.warn("This method is still a prototype!!") + + self.reset() + + def __str__(self) -> str: + msg = "\033[46m >>>> Generating templates \033[0m\n" + msg += "\033[46m" + "".join([" " for _ in range(100)]) + "\033[0m\n" + msg += f"\033[46m \033[0mDrain " + msg += f"\n\033[46m \033[0m -> (N. logs in buffer {len(self)})\n" + return msg + "\033[46m" + "".join([" " for _ in range(100)]) + "\033[0m\n" + + def __len__(self) -> int: + return len(self.buffer) + + def add(self, log: str) -> None: + self.buffer.append(log) + + def reset(self) -> None: + self.buffer: list[str] = [] + + def generate_from_df(self, df: pl.DataFrame) -> TreeMatcher: + + print("\033[46m >>>> Generating templates \033[0m") + print("\033[46m" + "".join([" " for _ in range(100)]) + "\033[0m") + + print("\033[46m \033[0m ⚙️ Clustering logs") + set_groups = cluster_logs_df(df=df[["Content"]], depth=self.depth, max_child=self.max_child) + + print("\033[46m \033[0m ⚙️ Generating templates") + templates = LogTemplates(drain_generator(list(set_groups.values()), self.sim)) + + print("\033[46m \033[0m 💻 Initializing tree matcher instance") + tree_matcher = TreeMatcher(templates) + + print("\033[46m \033[0m ✅ Process complete!") + print("\033[46m" + "".join([" " for _ in range(100)]) + "\033[0m") + return tree_matcher + + def generate(self) -> TreeMatcher: + return self.generate_from_df(pl.DataFrame({"Content": self.buffer})) diff --git a/src/detectmateperformance/lib/bind_class.cpython-312-x86_64-linux-gnu.so b/src/detectmateperformance/lib/bind_class.cpython-312-x86_64-linux-gnu.so index e4bf88d..bfa7a68 100755 Binary files a/src/detectmateperformance/lib/bind_class.cpython-312-x86_64-linux-gnu.so and b/src/detectmateperformance/lib/bind_class.cpython-312-x86_64-linux-gnu.so differ diff --git a/src/detectmateperformance/match_tree.py b/src/detectmateperformance/match_tree.py index 39302c4..591524a 100644 --- a/src/detectmateperformance/match_tree.py +++ b/src/detectmateperformance/match_tree.py @@ -24,6 +24,9 @@ def __str__(self) -> str: def __len__(self) -> int: return len(self.templates) + def get_templates(self, do_print: bool = True) -> list[str]: + return self.templates.get_templates(do_print=do_print) + def match_log(self, log: str, get_var: bool = False) -> ParsedLogs: if get_var: result = self.inst.match_string_with_var(log) diff --git a/src/detectmateperformance/types_.py b/src/detectmateperformance/types_.py index 7a2adf2..5728d75 100644 --- a/src/detectmateperformance/types_.py +++ b/src/detectmateperformance/types_.py @@ -8,8 +8,20 @@ def _load_file(path: str) -> list[str]: return [line.strip() for line in f if line.strip()] +def _templates_to_list(templates: Templates) -> list[str]: + temp_list: list[str] = [] + while (x := templates.get_next_template()) != []: + temp_list.append(" ".join(x).replace("VAR", "<*>")) + + return temp_list + + class LogTemplates: - def __init__(self, templates: list[str]): + def __init__(self, templates: list[str] | Templates): + if isinstance(templates, Templates): + templates = _templates_to_list(templates) + + self.temp_list = templates.copy() self.inst = Templates(list( map(lambda x: x.replace("<*>", "VAR"), templates) )) @@ -17,6 +29,13 @@ def __init__(self, templates: list[str]): def __len__(self) -> int: return self.inst.size() # type: ignore + def get_templates(self, do_print: bool = True) -> list[str]: + if do_print: + for temp in self.temp_list: + print(" -> ", temp) + + return self.temp_list + def shape(self) -> tuple[int, int]: return self.inst.shape() # type: ignore diff --git a/tests/test_c/test_drain_op.cpp b/tests/test_c/test_drain_op.cpp new file mode 100644 index 0000000..f1a70f4 --- /dev/null +++ b/tests/test_c/test_drain_op.cpp @@ -0,0 +1,118 @@ +#include + +#include +#include +#include "../../src/detectmateperformance/_core/parsers/drain.h" + + + +TEST(DrainTest, Similarity) { + std::string string_1 = "Hello, tomas world"; + std::string string_2 = "Hello, alice world"; + std::string string_3 = "Hello"; + std::string string_4 = "Ciao"; + + float result = calculate_sim(string_1, string_2); + EXPECT_TRUE(0.71 < result && result < 0.73); + + float result_2 = calculate_sim(string_1, string_1); + EXPECT_EQ(result_2, 1.0); + + float result_3 = calculate_sim(string_3, string_4); + EXPECT_EQ(result_3, 0.0); + + float result_4 = calculate_sim(string_3, string_1); + EXPECT_EQ(result_4, 1.0); +} + + +TEST(DrainTest, GenerateTemplate) { + + std::deque inp = { + "Hello tobias my man", + "Hello guillermo my man", + "Hello julie my woman", + }; + std::string expected = "Hello VAR my VAR"; + + EXPECT_EQ(generate_template(inp), expected); + + std::deque inp2 = { + "Hello tobias my man my bro", + "Hello guillermo my man", + "Hello julie my woman a", + }; + std::string expected2 = "Hello VAR my VAR"; + + EXPECT_EQ(generate_template(inp2), expected2); + +} + + +TEST(DrainTest, GenerateMultipleTemplate) { + + std::vector> inp = { + { + "Hello tobias my man", + "Hello guillermo my man", + "Hello julie my woman", + "Mamma mia here again", + "Mamma tia here again" + }, + { + "ciao bella" + } + }; + std::deque expected = {"Hello VAR my VAR"}; + + std::deque result = generate_templates(inp, 0.2); + + EXPECT_EQ(result.size(), 3); + EXPECT_EQ(result[0], "Hello VAR my VAR"); + EXPECT_EQ(result[1], "Mamma VAR here again"); + EXPECT_EQ(result[2], "ciao bella"); + +} + + +TEST(DrainTest, CleanTemplates) { + std::deque templates = { + "Hello there VAR VAR kenobi", + "Hello there VAR VAR", + "Hello there VAR VAR kenobi", + "VAR" + }; + std::deque expected = { + "Hello there VAR kenobi", + "Hello there VAR ", + }; + + EXPECT_EQ(clean_templates(templates), expected); +} + + +TEST(DrainTest, DrainGenerator) { + + std::vector> inp = { + { + "hello tobias my man number 2", + "hello guillermo my man number 2", + "hello julie my woman number 2," + }, + }; + + Templates result = drain_generator(inp, 0.2); + auto message1 = result.getNext(); + + for (size_t i= 0; i < message1.size(); i++) + std::cout << message1[i] << std::endl; + + EXPECT_EQ(message1.size(), 6); + EXPECT_EQ(message1[0], "hello"); + EXPECT_EQ(message1[1], "VAR"); + EXPECT_EQ(message1[2], "my"); + EXPECT_EQ(message1[3], "VAR"); + EXPECT_EQ(message1[4], "number"); + EXPECT_EQ(message1[5], "VAR"); + +} diff --git a/tests/test_python/test_drain.py b/tests/test_python/test_drain.py new file mode 100644 index 0000000..aa59c16 --- /dev/null +++ b/tests/test_python/test_drain.py @@ -0,0 +1,58 @@ +from detectmateperformance.lib.bind_class import Templates +from detectmateperformance import drain + +import polars as pl + + +class TestCommonMethods: + def test_cluster_logs_df(self): + df = pl.DataFrame({"Content": ["hello world, 12 ciao", "Ciao! ? bella bella"]}) + + expected = {'3_Ciao_bella': ['Ciao bella bella'], '4_hello_world': ['hello world VAR ciao']} + assert drain.cluster_logs_df(df, depth=2) == expected + + expected = {'3_Ciao_bella_bella': ['Ciao bella bella'], '4_hello_world_VAR': ['hello world VAR ciao']} + assert drain.cluster_logs_df(df, depth=3) == expected + + def test_cluster_logs_df_special_cases(self): + df = pl.DataFrame({"Content": ["hello world, 12 ciao", "Ciao bella bella"]}) + + expected = {'2_Ciao': ['Ciao bella bella'], '2_hello': ['hello world VAR ciao']} + assert drain.cluster_logs_df(df, depth=1, max_child=2) == expected + + expected = { + '3_Ciao_bella_bella_None_None_None_None_None_None_None': ['Ciao bella bella'], + '4_hello_world_VAR_ciao_None_None_None_None_None_None': ['hello world VAR ciao'] + } + assert drain.cluster_logs_df(df, depth=10) == expected + + def test_bind_method(self): + assert isinstance(drain.drain_generator( + [["hello there", "ciaooo bellaa ciaoo"], ["general kenobi"]], 0.4 + ), Templates) + + +class TestDrain: + def test_add_reset(self): + drain_ = drain.Drain() + assert len(drain_) == 0 + + drain_.add("Hello world") + drain_.add("Hello world") + drain_.add("Hello planet") + assert len(drain_) == 3 + + drain_.reset() + assert len(drain_) == 0 + + def test_drain(self): + drain_ = drain.Drain(depth=1, sim=0.3) + assert len(drain_) == 0 + + drain_.add("hello tobias my man") + drain_.add("hello guillermo my man") + drain_.add("hello julie my woman") + + tree_match = drain_.generate() + template = tree_match.match_log("hello tobias my man").get_all_templates()[0] + assert "hello VAR my VAR" == template