Development

README.md

@@ -53,13 +53,8 @@ dataset = {
 }
 
 
-def load_file(path_logs) -> list[str]:
-    with open(path_logs, "r") as f:
-        return f.readlines()
-
-
 results = matep.metrics.evaluate(
-    logs=load_file(dataset["path_logs"]),
+    logs=dataset["path_logs"],
     ground_templates=dataset["path_temp"],
     templates=dataset["path_temp"],
     regex=dataset["regex"]

src/detectmateperformance/match_tree.py

@@ -42,7 +42,7 @@ def match_batch(
 
     def __call__(
         self,
-        logs: list[str] | pl.DataFrame,
+        logs: list[str] | pl.DataFrame | str,
         get_var: bool = False,
         n_workers: int = 1,
         batch: int = int(3e+6),

src/detectmateperformance/pipeline_op.py

@@ -8,15 +8,20 @@
 import gc
 
 
-def preprocessing(logs: list[str], regex: str) -> pl.DataFrame:
+def load_logs(path: str) -> pl.DataFrame:
+    return pl.read_csv(
+        path,
+        has_header=False,
+        new_columns=['Message'],
+        separator='\n',
+        null_values=None,
+    )
+
+
+def preprocessing(logs: list[str] | str, regex: str) -> pl.DataFrame:
+    df = load_logs(logs) if isinstance(logs, str) else pl.DataFrame({"Message": logs})
     df = (
-        pl.DataFrame({"Message": logs})
-        .with_columns(
-            pl.col("Message")
-            .str.extract_groups(regex)
-            .alias("parts")
-        )
-        .unnest("parts")
+        df.with_columns(pl.col("Message").str.extract_groups(regex).alias("parts")).unnest("parts")
     ).drop("Message")
     df = df.drop_nulls()
 
@@ -75,7 +80,7 @@ def run_batches(
 
 def run_full_pipeline(
     func: Callable[[list[str], bool, int], ParsedLogs],
-    logs: list[str],
+    logs: list[str] | str,
     get_var: bool = False,
     n_workers: int = 1,
     batch: int = int(3e+6),

tests/test_python/test_match_tree.py

@@ -73,7 +73,8 @@ def test_big_batch_with_var(self):
     def test_call(self):
         logs = load_logs()
         tree_matcher = TreeMatcher.from_file(path_temp)
-        results = tree_matcher(logs, True, n_workers=3)
+        results = tree_matcher(path_logs, True, n_workers=3)
+        print(results)
 
         assert isinstance(results, pl.DataFrame)
         assert len(logs) == len(results)