From 5cd0b74ff66c0379ab9d29f94ea52d9ab14e0017 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Fri, 12 Jun 2026 10:44:23 +0200 Subject: [PATCH 1/2] add read logs --- src/detectmateperformance/match_tree.py | 2 +- src/detectmateperformance/pipeline_op.py | 23 ++++++++++++++--------- tests/test_python/test_match_tree.py | 3 ++- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/detectmateperformance/match_tree.py b/src/detectmateperformance/match_tree.py index b2a7183..39302c4 100644 --- a/src/detectmateperformance/match_tree.py +++ b/src/detectmateperformance/match_tree.py @@ -42,7 +42,7 @@ def match_batch( def __call__( self, - logs: list[str] | pl.DataFrame, + logs: list[str] | pl.DataFrame | str, get_var: bool = False, n_workers: int = 1, batch: int = int(3e+6), diff --git a/src/detectmateperformance/pipeline_op.py b/src/detectmateperformance/pipeline_op.py index 1626358..1a13db9 100644 --- a/src/detectmateperformance/pipeline_op.py +++ b/src/detectmateperformance/pipeline_op.py @@ -8,15 +8,20 @@ import gc -def preprocessing(logs: list[str], regex: str) -> pl.DataFrame: +def load_logs(path: str) -> pl.DataFrame: + return pl.read_csv( + path, + has_header=False, + new_columns=['Message'], + separator='\n', + null_values=None, + ) + + +def preprocessing(logs: list[str] | str, regex: str) -> pl.DataFrame: + df = load_logs(logs) if isinstance(logs, str) else pl.DataFrame({"Message": logs}) df = ( - pl.DataFrame({"Message": logs}) - .with_columns( - pl.col("Message") - .str.extract_groups(regex) - .alias("parts") - ) - .unnest("parts") + df.with_columns(pl.col("Message").str.extract_groups(regex).alias("parts")).unnest("parts") ).drop("Message") df = df.drop_nulls() @@ -75,7 +80,7 @@ def run_batches( def run_full_pipeline( func: Callable[[list[str], bool, int], ParsedLogs], - logs: list[str], + logs: list[str] | str, get_var: bool = False, n_workers: int = 1, batch: int = int(3e+6), diff --git a/tests/test_python/test_match_tree.py b/tests/test_python/test_match_tree.py index 012e470..1b2ab98 100644 --- a/tests/test_python/test_match_tree.py +++ b/tests/test_python/test_match_tree.py @@ -73,7 +73,8 @@ def test_big_batch_with_var(self): def test_call(self): logs = load_logs() tree_matcher = TreeMatcher.from_file(path_temp) - results = tree_matcher(logs, True, n_workers=3) + results = tree_matcher(path_logs, True, n_workers=3) + print(results) assert isinstance(results, pl.DataFrame) assert len(logs) == len(results) From db875c61f14366d3d22158a58eb8cb6a5d07745f Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Fri, 12 Jun 2026 10:51:01 +0200 Subject: [PATCH 2/2] update readme --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 956e62e..c47cede 100644 --- a/README.md +++ b/README.md @@ -53,13 +53,8 @@ dataset = { } -def load_file(path_logs) -> list[str]: - with open(path_logs, "r") as f: - return f.readlines() - - results = matep.metrics.evaluate( - logs=load_file(dataset["path_logs"]), + logs=dataset["path_logs"], ground_templates=dataset["path_temp"], templates=dataset["path_temp"], regex=dataset["regex"]