diff --git a/EasyChemML/Utilities/DataUtilities/BatchDatatypHolder.py b/EasyChemML/Utilities/DataUtilities/BatchDatatypHolder.py index 065e658..d3436b7 100644 --- a/EasyChemML/Utilities/DataUtilities/BatchDatatypHolder.py +++ b/EasyChemML/Utilities/DataUtilities/BatchDatatypHolder.py @@ -126,7 +126,7 @@ def checkAll_numbers(self) -> bool: return False return True else: - return True + return BatchDatatypClass.get_dtype_lvl(self[self.getColumns()[0]]) > 0 def check_containsObjects(self) -> bool: if len(self) > 1: @@ -134,7 +134,7 @@ def check_containsObjects(self) -> bool: for item in self: item: BatchDatatyp = self[item] - if BatchDatatypClass.get_dtype_lvl(item) < 0: + if BatchDatatypClass.get_dtype_lvl(item) < -1: return True return False else: diff --git a/EasyChemML/Utilities/DataUtilities/BatchTable.py b/EasyChemML/Utilities/DataUtilities/BatchTable.py index db73077..c5edea9 100644 --- a/EasyChemML/Utilities/DataUtilities/BatchTable.py +++ b/EasyChemML/Utilities/DataUtilities/BatchTable.py @@ -51,12 +51,12 @@ def __setitem__(self, key, value): if not (isinstance(value, np.ndarray) or isinstance(value, np.void)): raise Exception(f'the passed batch is not a numpy array or numpy array entry| {type(value)}') - if not self.getDatatypes().check_containsObjects() and not value.dtype == self[0].dtype: + my_datatypes: BatchDatatypHolder = self.getDatatypes() + + if my_datatypes.check_containsObjects() or not (value.dtype == self[0].dtype or (BatchDatatypClass.NUMPY_STRING in my_datatypes and len(value.shape)==1)): raise Exception(f'the dtype of the numpy array is different to the batchtable | {type(value)}') value: np.array = value - my_datatypes: BatchDatatypHolder = self.getDatatypes() - if BatchDatatypClass.PYTHON_OBJECT in my_datatypes: my_datatypes.removeAllnoneObject() # notObjects = [item for item in self.getDatatypes().getColumns() if item not in my_datatypes.getColumns()] @@ -270,7 +270,7 @@ def getWith(self, columns: List[str] = None): else: return 0 - def convert_2_ndarray(self, indicies: List[int] = None, columns: List[str] = None) -> np.ndarray: + def convert_2_ndarray(self, indicies: List[int] = None, columns: List[str] = None): if columns is None: columns = self.getColumns() @@ -292,7 +292,6 @@ def convert_2_ndarray(self, indicies: List[int] = None, columns: List[str] = Non else: raise Exception('different dTypes are not supported') - np_dtype = datatyp.toNUMPY() # if indicies is not None: # shape = (len(indicies), self.getWith(columns)) # else: @@ -301,8 +300,7 @@ def convert_2_ndarray(self, indicies: List[int] = None, columns: List[str] = Non if indicies is None: if datatyp == BatchDatatypClass.NUMPY_STRING: raw_data = self.getComplexSelection(columns, indicies) - data = rfn.structured_to_unstructured(raw_data, dtype=np_dtype) - return data + return [i[0].decode() for i in raw_data] else: raw_data = self.getComplexSelection(columns, indicies) if len(columns) == 1: @@ -313,8 +311,8 @@ def convert_2_ndarray(self, indicies: List[int] = None, columns: List[str] = Non else: if datatyp == BatchDatatypClass.NUMPY_STRING: raw_data = self.getComplexSelection(columns, indicies) - data = rfn.structured_to_unstructured(raw_data, dtype=np_dtype) - return data + return [i[0].decode() for i in raw_data] + else: raw_data: np.ndarray = self.getComplexSelection(columns, indicies) if len(columns) == 1: diff --git a/EasyChemML/Utilities/DataUtilities/BatchTableAlgorithms.py b/EasyChemML/Utilities/DataUtilities/BatchTableAlgorithms.py index d4f12ee..6ef5b9b 100644 --- a/EasyChemML/Utilities/DataUtilities/BatchTableAlgorithms.py +++ b/EasyChemML/Utilities/DataUtilities/BatchTableAlgorithms.py @@ -3,7 +3,7 @@ from EasyChemML.Environment import Environment from EasyChemML.Utilities.DataUtilities.BatchTable import BatchTable -from EasyChemML.Utilities.DataUtilities.TableAlgorithms.Sort.MergeSort import MergeSort +# from EasyChemML.Utilities.DataUtilities.TableAlgorithms.Sort.MergeSort import MergeSort from EasyChemML.Utilities.DataUtilities.TableAlgorithms.ReorderInplace import ReorderInplace @@ -15,7 +15,8 @@ def __init__(self, env:Environment): self.env = env def mergeSort(self, batchTable: BatchTable): - MergeSort.sort(batchTable, env=self.env) + # MergeSort.sort(batchTable, env=self.env) + print("no merge sort") def sort(self, batchTable: BatchTable, key_func: Callable[[Any, Any], int] = None): length = len(batchTable) diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/pyWrapper/RustBatchListFunctions_duplicates.py b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/pyWrapper/RustBatchListFunctions_duplicates.py index 0bdf9d9..3b26c3c 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/pyWrapper/RustBatchListFunctions_duplicates.py +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/pyWrapper/RustBatchListFunctions_duplicates.py @@ -49,7 +49,7 @@ class RustBatchListFunctions_duplicates: def __init__(self): self._batchlist_duplicates = BatchListFunctions_duplicates_py() - +# TODO: string support hinzufügen def count_duplicates(self, rustbatchholder: RustBatchholder, tableName: str, get_distibution_by_last_col: bool = False) -> RustBatchListFunctions_duplicates_result: bt = rustbatchholder.getRustBatchTable(tableName) @@ -76,7 +76,11 @@ def count_duplicates(self, rustbatchholder: RustBatchholder, tableName: str, result.duplicates_by_last_col) elif dtype == dtype.NUMPY_FLOAT32: raise Exception('float32 is not sortable at the moment') + elif dtype == dtype.NUMPY_STRING: + result = self._batchlist_duplicates.count_duplicates_on_sorted_list_string(bt, get_distibution_by_last_col) + return result elif dtype == dtype.NUMPY_FLOAT64: - raise Exception('float64 is not sortable at the moment') + result = self._batchlist_duplicates.count_duplicates_on_sorted_list_f64(bt, get_distibution_by_last_col) + return result else: raise Exception('datatype is not sortable at the moment') diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/pyWrapper/RustBatchSorter_Radix.py b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/pyWrapper/RustBatchSorter_Radix.py index f283ed6..544f172 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/pyWrapper/RustBatchSorter_Radix.py +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/pyWrapper/RustBatchSorter_Radix.py @@ -15,7 +15,7 @@ class RustBatchSorter_Radix: def __init__(self, tmp_path: str): self._sorter = BatchSorter_Radix_py(os.path.join(tmp_path, 'sort_tmp')) - +# TODO: string support hinzufügen def sort(self, rustbatchholder: RustBatchholder, tableName): bt = rustbatchholder.getRustBatchTable(tableName) dtype = rustbatchholder.rustBatchtable[tableName][1] diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/pyWrapper/RustBatchholder.py b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/pyWrapper/RustBatchholder.py index 545eda0..e8c68d7 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/pyWrapper/RustBatchholder.py +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/pyWrapper/RustBatchholder.py @@ -1,7 +1,6 @@ import os.path from typing import Dict, List, Tuple, Any from enum import Enum - import numpy as np from numpy.lib import recfunctions as rfn from tqdm import tqdm @@ -35,7 +34,7 @@ def clean(self): self.rustBatchtable = {} def transferToRust(self, bp: BatchPartition, table: str, columns: List[str] = None, memMode: MemoryMode = MemoryMode.InMemory): - bt = bp[table] + bt = bp[table] # bt = batchtable,bp = batchpartition if columns is None: columns = bt.getColumns() @@ -58,8 +57,10 @@ def transferToRust(self, bp: BatchPartition, table: str, columns: List[str] = No loaded_chunk = bt.convert_2_ndarray(indicies=list(range(i, len(bt))), columns=columns) else: loaded_chunk = bt.convert_2_ndarray(indicies=list(range(i, i + chunksize)), columns=columns) - - new_rbt.add_chunk(loaded_chunk) + if dataTypHolder[dataTypHolder.getColumns()[0]] == BatchDatatypClass.NUMPY_STRING and len(columns)==1: + new_rbt.add_chunk_wrapper(loaded_chunk) + elif len(columns)==1: + new_rbt.add_chunk(np.expand_dims(loaded_chunk, axis=1)) bar.update(len(loaded_chunk)) def _create_typed_BatchTable(self, dtype: BatchDatatypHolder, tableName:str): @@ -77,6 +78,8 @@ def _create_typed_BatchTable(self, dtype: BatchDatatypHolder, tableName:str): return self.rustBatchholder.get_batchtable_f32(tableName) elif dtype == dtype.NUMPY_FLOAT64: return self.rustBatchholder.get_batchtable_f64(tableName) + elif dtype == dtype.NUMPY_STRING: + return self.rustBatchholder.get_batchtable_string(tableName) def getRustBatchTable(self, tableName: str): if tableName in self.rustBatchtable: @@ -95,8 +98,12 @@ def transferToBatchtable(self, rustTableName: str, bp: BatchPartition, newBatchT bt = bp[newBatchTableName] for index, i in enumerate(range(0, len(bt), chunksize)): - loaded_chunk = rustbt.load_chunk(index) - loaded_chunk = rfn.unstructured_to_structured(loaded_chunk, dtype.toNUMPY_dtypes()) + if dtype in BatchDatatypClass.NUMPY_STRING and len(shape) == 1: + loaded_chunk = rustbt.get_loaded_string_chunk(index) + loaded_chunk = np.array(loaded_chunk) + else: + loaded_chunk = rustbt.load_chunk(index) + loaded_chunk = rfn.unstructured_to_structured(loaded_chunk, dtype.toNUMPY_dtypes()) if i + chunksize > len(bt): bt[i:len(bt)] = loaded_chunk else: diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/BatchListFunctions_duplicates.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/BatchListFunctions_duplicates.rs index 23b8c52..0e85d81 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/BatchListFunctions_duplicates.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/BatchListFunctions_duplicates.rs @@ -1,105 +1,115 @@ +use crate::BatchSystem::BatchTablesImplementation::BatchTable::{ + BatchTable, BatchTablesTypWrapper, +}; +use crate::Utilities::array_helper; +use ndarray::{Array, Axis, Ix1, Ix2}; +use ndarray_npy::ReadableElement; +use serde::de::DeserializeOwned; use std::borrow::Borrow; use std::collections::HashMap; use std::fmt::Display; use std::hash::Hash; use std::sync::{Arc, RwLock}; -use ndarray::{Array, Array2, ArrayView2, ArrayViewMut, Axis, Ix1, Ix2, IxDyn, s}; -use ndarray_npy::{ReadableElement}; -use serde::de::DeserializeOwned; -use serde::Deserialize; -use crate::BatchSystem::BatchTablesImplementation::BatchTable::{BatchTable, BatchTablesTypWrapper}; -use crate::Utilities::array_helper; -pub struct duplicat_result { - pub counted_entries:usize, - pub counted_duplicates:usize, - pub entry_most_duplicates:usize, +pub struct duplicat_result { + pub counted_entries: usize, + pub counted_duplicates: usize, + pub entry_most_duplicates: usize, pub duplicates_dist: HashMap, - pub duplicates_by_last_col: HashMap + pub duplicates_by_last_col: HashMap, } -pub fn count_duplicates_on_sorted_list<'de,T: ReadableElement + Ord + Clone + Hash + Display+ Copy + DeserializeOwned>(batchtable: Arc>, calc_duplicates_by_last_col:bool) -> duplicat_result { +pub fn count_duplicates_on_sorted_list< + 'de, + T: ReadableElement + Ord + Clone + Hash + Display + Copy + DeserializeOwned, +>( + batchtable: Arc>, + calc_duplicates_by_last_col: bool, +) -> duplicat_result { let mut batchtable = batchtable.read().unwrap(); let chunk_count = batchtable.get_table_chunk_count(); - let mut counted_entries:usize = 0; - let mut counted_duplicates:usize = 0; - let mut entry_most_duplicates:usize = 0; + let mut counted_entries: usize = 0; + let mut counted_duplicates: usize = 0; + let mut entry_most_duplicates: usize = 0; let mut duplicates_dist: HashMap = HashMap::new(); - let mut duplicates_by_last_col: HashMap = HashMap::new(); + let mut duplicates_by_last_col: HashMap = HashMap::new(); - let mut current_entry_duplicated:usize = 0; + let mut current_entry_duplicated: usize = 0; - let mut last_entry:Option> = None; - for chunk_index in 0..chunk_count{ + let mut last_entry: Option> = None; + for chunk_index in 0..chunk_count { let loaded_chunk: Array = batchtable.load_chunk(chunk_index); - - for entry in loaded_chunk.axis_iter(Axis(0)){ + for entry in loaded_chunk.axis_iter(Axis(0)) { counted_entries += 1; - if last_entry.is_none(){ + if last_entry.is_none() { last_entry = Option::from(entry.to_owned()); - }else{ + } else { let last_clone = last_entry.clone().unwrap(); - if array_helper::merge_compare_arrayView1(last_clone.view(), entry, calc_duplicates_by_last_col) == 0{ - if current_entry_duplicated == 0{ + if array_helper::merge_compare_arrayView1( + last_clone.view(), + entry, + calc_duplicates_by_last_col, + ) == 0 + { + if current_entry_duplicated == 0 { counted_duplicates += 2; current_entry_duplicated = 2; - }else { + } else { counted_duplicates += 1; current_entry_duplicated += 1; } - if calc_duplicates_by_last_col{ - if current_entry_duplicated == 2{ - + if calc_duplicates_by_last_col { + if current_entry_duplicated == 2 { let val_last = last_clone.last().unwrap(); let val_current = entry.view(); let val_current = val_current.last().unwrap(); - if duplicates_by_last_col.contains_key(val_last.borrow()){ - let hash_pointer = duplicates_by_last_col.get_mut(val_last.borrow()).unwrap(); + if duplicates_by_last_col.contains_key(val_last.borrow()) { + let hash_pointer = + duplicates_by_last_col.get_mut(val_last.borrow()).unwrap(); *hash_pointer += 1; - }else { + } else { duplicates_by_last_col.insert(*val_last, 1); } - if duplicates_by_last_col.contains_key(val_current.borrow()){ - let hash_pointer = duplicates_by_last_col.get_mut(val_current.borrow()).unwrap(); + if duplicates_by_last_col.contains_key(val_current.borrow()) { + let hash_pointer = duplicates_by_last_col + .get_mut(val_current.borrow()) + .unwrap(); *hash_pointer += 1; - }else { + } else { duplicates_by_last_col.insert(*val_current, 1); } - }else if current_entry_duplicated > 2 { + } else if current_entry_duplicated > 2 { let val_current = entry.view(); let val_current = val_current.last().unwrap(); - if duplicates_by_last_col.contains_key(val_current){ - let hash_pointer = duplicates_by_last_col.get_mut(val_current).unwrap(); + if duplicates_by_last_col.contains_key(val_current) { + let hash_pointer = + duplicates_by_last_col.get_mut(val_current).unwrap(); *hash_pointer += 1; - }else { + } else { duplicates_by_last_col.insert(*val_current, 1); } } } - - }else{ - + } else { last_entry = Option::from(entry.to_owned()); - if current_entry_duplicated >= 2{ - if duplicates_dist.contains_key(¤t_entry_duplicated){ + if current_entry_duplicated >= 2 { + if duplicates_dist.contains_key(¤t_entry_duplicated) { *duplicates_dist.get_mut(¤t_entry_duplicated).unwrap() += 1; - }else { + } else { duplicates_dist.insert(current_entry_duplicated, 1); } } - - - if current_entry_duplicated > entry_most_duplicates{ + if current_entry_duplicated > entry_most_duplicates { entry_most_duplicates = current_entry_duplicated; } @@ -109,13 +119,19 @@ pub fn count_duplicates_on_sorted_list<'de,T: ReadableElement + Ord + Clone + Ha } } - if current_entry_duplicated >= 2{ - if duplicates_dist.contains_key(¤t_entry_duplicated){ + if current_entry_duplicated >= 2 { + if duplicates_dist.contains_key(¤t_entry_duplicated) { *duplicates_dist.get_mut(¤t_entry_duplicated).unwrap() += 1; - }else { + } else { duplicates_dist.insert(current_entry_duplicated, 1); } } - duplicat_result{counted_entries, counted_duplicates, entry_most_duplicates, duplicates_dist, duplicates_by_last_col } -} \ No newline at end of file + duplicat_result { + counted_entries, + counted_duplicates, + entry_most_duplicates, + duplicates_dist, + duplicates_by_last_col, + } +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/BatchSort.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/BatchSort.rs index 6514e22..5e0de31 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/BatchSort.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/BatchSort.rs @@ -1,32 +1,42 @@ +use kdam::tqdm; +use kdam::BarExt; use std::collections::HashMap; +use std::collections::VecDeque; use std::fmt::Display; use std::hash::Hash; -use std::collections::VecDeque; -use std::time::Instant; use std::marker::Send; -use kdam::{tqdm, BarExt}; use std::ops::DerefMut; -use std::sync::{Arc, RwLock, RwLockWriteGuard}; +use std::sync::{Arc, RwLock}; +use std::time::Instant; -use ndarray::{Array, Array2, Axis, Dimension, Ix2, IxDyn, s}; -use ndarray_npy::{ReadableElement}; +use ndarray::{Array, Array2, Axis, Dimension, Ix2}; +use ndarray_npy::ReadableElement; -use crate::BatchSystem::BatchTablesImplementation::BatchTable::{BatchTable, BatchTableMemoryBased, BatchTablePathBased, BatchTablesTypWrapper}; +use crate::BatchSystem::BatchHolder::BatchHolder; +use crate::BatchSystem::BatchTablesImplementation::BatchTable::{ + BatchTable, BatchTableMemoryBased, BatchTablePathBased, BatchTablesTypWrapper, +}; +use crate::Utilities::array_helper; +use ndarray::s; use rayon::prelude::*; -use serde::{Deserialize, Serialize}; use serde::de::DeserializeOwned; -use crate::BatchSystem::BatchHolder::{BatchHolder}; -use crate::Utilities::array_helper; - -pub fn sort<'de,T: Ord + Clone + Hash + Display + Copy + Send + Serialize + DeserializeOwned + ReadableElement>(batchtable: Arc>, tmp_path:&str){ +use serde::Serialize; + +pub fn sort< + 'de, + T: Ord + Clone + Hash + Display + Copy + Send + Serialize + DeserializeOwned + ReadableElement, +>( + batchtable: Arc>, + tmp_path: &str, +) { let mut batchtable = batchtable.write().unwrap(); //sort batches println!("Sort batch files"); let mut bar = tqdm!(total = batchtable.get_table_chunk_count()); let start = Instant::now(); - for index in 0..batchtable.get_table_chunk_count(){ + for index in 0..batchtable.get_table_chunk_count() { bar.update(1); - let arr:&mut Array = &mut batchtable.load_chunk(index); + let arr: &mut Array = &mut batchtable.load_chunk(index); sort_batch(arr); batchtable.override_chunk_with_array(arr, index); } @@ -36,7 +46,8 @@ pub fn sort<'de,T: Ord + Clone + Hash + Display + Copy + Send + Serialize + Dese //merge batches let batch_count = batchtable.get_table_chunk_count(); - let merge_slices: VecDeque<((usize, usize), (usize, usize))> = merge_generate_merge_slices(batch_count); + let merge_slices: VecDeque<((usize, usize), (usize, usize))> = + merge_generate_merge_slices(batch_count); let mut tmp_batchholder: BatchHolder = BatchHolder::new(tmp_path); @@ -47,31 +58,47 @@ pub fn sort<'de,T: Ord + Clone + Hash + Display + Copy + Send + Serialize + Dese let (slice_left, slice_right) = slice; let (slice_left_start, slice_left_end) = slice_left; let (slice_right_start, slice_right_end) = slice_right; - merge_slice::(batchtable.deref_mut(), &mut tmp_batchholder, slice_left, slice_right); + merge_slice::( + batchtable.deref_mut(), + &mut tmp_batchholder, + slice_left, + slice_right, + ); if slice_left_start == slice_right_start && slice_left_end == slice_right_end { continue; } { - let current_tmp_table = tmp_batchholder.get_batchtable(batchtable.get_table_name() + "_sort"); + let current_tmp_table = + tmp_batchholder.get_batchtable(batchtable.get_table_name() + "_sort"); let mut current_tmp_table = current_tmp_table.write().unwrap(); - for (tmp_batchtable_index, slice_index) in (slice_left_start..slice_right_end + 1).enumerate() { - if batchtable.is_batch_table_pathbased() && current_tmp_table.is_batch_table_pathbased() { - let mut cast_currentTMPTable = current_tmp_table.to_batch_table_file_system().unwrap(); + for (tmp_batchtable_index, slice_index) in + (slice_left_start..slice_right_end + 1).enumerate() + { + if batchtable.is_batch_table_pathbased() + && current_tmp_table.is_batch_table_pathbased() + { + let mut cast_currentTMPTable = + current_tmp_table.to_batch_table_file_system().unwrap(); let cast_batchtable = batchtable.to_batch_table_file_system().unwrap(); let tmpTable_path = cast_currentTMPTable.get_chunk_path(tmp_batchtable_index); let slice_tmp_path = tmpTable_path.to_str().unwrap(); cast_batchtable.override_chunk_with_file(slice_tmp_path, slice_index); - } else if batchtable.is_batch_table_memorybased() && current_tmp_table.is_batch_table_memorybased() { - let cast_currentTMPTable = current_tmp_table.to_batch_table_in_memory().unwrap(); + } else if batchtable.is_batch_table_memorybased() + && current_tmp_table.is_batch_table_memorybased() + { + let cast_currentTMPTable = + current_tmp_table.to_batch_table_in_memory().unwrap(); let cast_batchtable = batchtable.to_batch_table_in_memory().unwrap(); - let vec_pointer = cast_currentTMPTable.get_raw_data_pointer(tmp_batchtable_index); + let vec_pointer = + cast_currentTMPTable.get_raw_data_pointer(tmp_batchtable_index); cast_batchtable.override_chunk_with_pointer(vec_pointer, slice_index); } else { - let arr: &mut Array = &mut current_tmp_table.load_chunk(tmp_batchtable_index); + let arr: &mut Array = + &mut current_tmp_table.load_chunk(tmp_batchtable_index); batchtable.override_chunk_with_array(arr, slice_index) } } @@ -81,18 +108,29 @@ pub fn sort<'de,T: Ord + Clone + Hash + Display + Copy + Send + Serialize + Dese let duration = start.elapsed(); println!("Rust batch merging takes : {:?}", duration); tmp_batchholder.clean(); - } -fn merge_slice<'de,T: ReadableElement + Ord + Clone + Hash + Display+ Copy+ Serialize+ DeserializeOwned>(mut batchtable: &mut BatchTablesTypWrapper, tmp_batchholder: &mut BatchHolder, slice_one: (usize, usize), slice_two: (usize, usize)){ +fn merge_slice< + 'de, + T: ReadableElement + Ord + Clone + Hash + Display + Copy + Serialize + DeserializeOwned, +>( + mut batchtable: &mut BatchTablesTypWrapper, + tmp_batchholder: &mut BatchHolder, + slice_one: (usize, usize), + slice_two: (usize, usize), +) { let (slice_one_start, slice_one_end) = slice_one; let (slice_two_start, slice_two_end) = slice_two; - if slice_one_start == slice_two_start && slice_one_end == slice_two_end{ + if slice_one_start == slice_two_start && slice_one_end == slice_two_end { return; } - tmp_batchholder.create_batchtable(batchtable.get_table_name() + "_sort", batchtable.get_shape(), batchtable.get_memory_mode()); + tmp_batchholder.create_batchtable( + batchtable.get_table_name() + "_sort", + batchtable.get_shape(), + batchtable.get_memory_mode(), + ); let mut tmp_batchtable = tmp_batchholder.get_batchtable(batchtable.get_table_name() + "_sort"); let mut tmp_batchtable = tmp_batchtable.write().unwrap(); @@ -101,9 +139,9 @@ fn merge_slice<'de,T: ReadableElement + Ord + Clone + Hash + Display+ Copy+ Seri let mut tmp_arr: Array = Array::clone(&left_arr); - let mut left_arr_rows_count:usize = left_arr.nrows(); - let mut right_arr_rows_count:usize = right_arr.nrows(); - let tmp_arr_rows_count:usize = tmp_arr.nrows(); + let mut left_arr_rows_count: usize = left_arr.nrows(); + let mut right_arr_rows_count: usize = right_arr.nrows(); + let tmp_arr_rows_count: usize = tmp_arr.nrows(); let mut current_slice_left = slice_one_start; let mut current_slice_right = slice_two_start; @@ -113,49 +151,77 @@ fn merge_slice<'de,T: ReadableElement + Ord + Clone + Hash + Display+ Copy+ Seri let mut current_left_arr_prt: usize = 0; loop { - - if current_tmp_vector_ptr >= tmp_arr_rows_count{ + if current_tmp_vector_ptr >= tmp_arr_rows_count { tmp_batchtable.add_chunk(&tmp_arr); current_tmp_vector_ptr = 0; } - if current_left_arr_prt >= left_arr_rows_count{ + if current_left_arr_prt >= left_arr_rows_count { current_slice_left += 1; - if current_slice_left <= slice_one_end{ + if current_slice_left <= slice_one_end { left_arr = batchtable.load_chunk(current_slice_left); left_arr_rows_count = left_arr.nrows(); current_left_arr_prt = 0; - }else{ - merge_copy_rest_of_array(&mut tmp_arr, &mut current_tmp_vector_ptr, batchtable, tmp_batchtable.deref_mut(), current_slice_right, slice_two_end, &mut current_right_arr_prt); + } else { + merge_copy_rest_of_array( + &mut tmp_arr, + &mut current_tmp_vector_ptr, + batchtable, + tmp_batchtable.deref_mut(), + current_slice_right, + slice_two_end, + &mut current_right_arr_prt, + ); break; } - } - if current_right_arr_prt >= right_arr_rows_count{ + if current_right_arr_prt >= right_arr_rows_count { current_slice_right += 1; - if current_slice_right <= slice_two_end{ + if current_slice_right <= slice_two_end { right_arr = batchtable.load_chunk(current_slice_right); right_arr_rows_count = right_arr.nrows(); current_right_arr_prt = 0; - }else{ - merge_copy_rest_of_array(&mut tmp_arr, &mut current_tmp_vector_ptr, batchtable, tmp_batchtable.deref_mut(), current_slice_left, slice_one_end, &mut current_left_arr_prt); + } else { + merge_copy_rest_of_array( + &mut tmp_arr, + &mut current_tmp_vector_ptr, + batchtable, + tmp_batchtable.deref_mut(), + current_slice_left, + slice_one_end, + &mut current_left_arr_prt, + ); break; } } - if current_slice_left >= slice_one_end && current_slice_right >= slice_two_end && current_left_arr_prt >= left_arr_rows_count && current_right_arr_prt >= right_arr_rows_count { + if current_slice_left >= slice_one_end + && current_slice_right >= slice_two_end + && current_left_arr_prt >= left_arr_rows_count + && current_right_arr_prt >= right_arr_rows_count + { break; } - let compare_result = array_helper::merge_compare_arrayView1(left_arr.row(current_left_arr_prt), right_arr.row(current_right_arr_prt), false); - - if compare_result >= 0{ - array_helper::merge_arrayView1_copy(&mut tmp_arr.row_mut(current_tmp_vector_ptr), &left_arr.row(current_left_arr_prt)); + let compare_result = array_helper::merge_compare_arrayView1( + left_arr.row(current_left_arr_prt), + right_arr.row(current_right_arr_prt), + false, + ); + + if compare_result >= 0 { + array_helper::merge_arrayView1_copy( + &mut tmp_arr.row_mut(current_tmp_vector_ptr), + &left_arr.row(current_left_arr_prt), + ); current_tmp_vector_ptr += 1; current_left_arr_prt += 1; - }else { - array_helper::merge_arrayView1_copy(&mut tmp_arr.row_mut(current_tmp_vector_ptr), &right_arr.row(current_right_arr_prt)); + } else { + array_helper::merge_arrayView1_copy( + &mut tmp_arr.row_mut(current_tmp_vector_ptr), + &right_arr.row(current_right_arr_prt), + ); current_tmp_vector_ptr += 1; current_right_arr_prt += 1; } @@ -164,106 +230,126 @@ fn merge_slice<'de,T: ReadableElement + Ord + Clone + Hash + Display+ Copy+ Seri tmp_batchtable.add_chunk(&tmp_arr.slice(s!(0..current_tmp_vector_ptr, ..)).to_owned()); } -fn merge_copy_rest_of_array<'de,T: ReadableElement + Ord + Clone + Hash + Display + Copy + Serialize+ DeserializeOwned> - (tmp_array: &mut Array2, mut current_tmp_vector_ptr:&mut usize, mut batchtable: &mut BatchTablesTypWrapper, tmp_batchtable: &mut BatchTablesTypWrapper, start_slice: usize, end_slice:usize, mut current_arr_ptr: &mut usize){ - - for i_slice in start_slice .. end_slice+1{ - let rest_array:Array2 = batchtable.load_chunk(i_slice); - for current_ptr in *current_arr_ptr .. rest_array.nrows(){ - if *current_tmp_vector_ptr >= tmp_array.nrows(){ +fn merge_copy_rest_of_array< + 'de, + T: ReadableElement + Ord + Clone + Hash + Display + Copy + Serialize + DeserializeOwned, +>( + tmp_array: &mut Array2, + mut current_tmp_vector_ptr: &mut usize, + mut batchtable: &mut BatchTablesTypWrapper, + tmp_batchtable: &mut BatchTablesTypWrapper, + start_slice: usize, + end_slice: usize, + mut current_arr_ptr: &mut usize, +) { + for i_slice in start_slice..end_slice + 1 { + let rest_array: Array2 = batchtable.load_chunk(i_slice); + for current_ptr in *current_arr_ptr..rest_array.nrows() { + if *current_tmp_vector_ptr >= tmp_array.nrows() { tmp_batchtable.add_chunk(&tmp_array); *current_tmp_vector_ptr = 0; } - array_helper::merge_arrayView1_copy(&mut tmp_array.row_mut(*current_tmp_vector_ptr), &rest_array.row(current_ptr)); + array_helper::merge_arrayView1_copy( + &mut tmp_array.row_mut(*current_tmp_vector_ptr), + &rest_array.row(current_ptr), + ); *current_tmp_vector_ptr += 1; } *current_arr_ptr = 0; } - *current_arr_ptr = tmp_array.nrows()+1; + *current_arr_ptr = tmp_array.nrows() + 1; } -fn merge_generate_merge_slices(batch_count:usize) -> VecDeque<((usize, usize), (usize, usize))> { - let mut merges: VecDeque<((usize,usize),(usize,usize))> = VecDeque::new(); - let mut last_merges: VecDeque<((usize,usize),(usize,usize))> = VecDeque::new(); - for batch_indexin in (0..batch_count).step_by(2){ +fn merge_generate_merge_slices(batch_count: usize) -> VecDeque<((usize, usize), (usize, usize))> { + let mut merges: VecDeque<((usize, usize), (usize, usize))> = VecDeque::new(); + let mut last_merges: VecDeque<((usize, usize), (usize, usize))> = VecDeque::new(); + for batch_indexin in (0..batch_count).step_by(2) { let first_entry = (batch_indexin, batch_indexin); - let mut second_entry = (batch_indexin+1, batch_indexin+1); + let mut second_entry = (batch_indexin + 1, batch_indexin + 1); - if batch_indexin+1 >= batch_count { + if batch_indexin + 1 >= batch_count { second_entry = (batch_indexin, batch_indexin); } - merges.push_back((first_entry,second_entry)); - last_merges.push_back((first_entry,second_entry)); + merges.push_back((first_entry, second_entry)); + last_merges.push_back((first_entry, second_entry)); } loop { let last_merge_len = last_merges.len(); let clone_last_merge = last_merges.clone(); last_merges.clear(); - for (i, (first_entry, second_entry)) in clone_last_merge. - iter().step_by(2 as usize).enumerate(){ - + for (i, (first_entry, second_entry)) in + clone_last_merge.iter().step_by(2 as usize).enumerate() + { let (first_left, first_right) = first_entry; let (second_left, second_right) = second_entry; - if (i*2)+1 >= last_merge_len{ - let new_first = (*first_left,*second_right); + if (i * 2) + 1 >= last_merge_len { + let new_first = (*first_left, *second_right); let new_second = (*first_left, *second_right); merges.push_back((new_first, new_second)); last_merges.push_back((new_first, new_second)); - }else{ - let ((next_first_left, next_first_right), (next_second_left, next_second_right)) = clone_last_merge[(i*2)+1]; - let new_first = (*first_left,*second_right); + } else { + let ((next_first_left, next_first_right), (next_second_left, next_second_right)) = + clone_last_merge[(i * 2) + 1]; + let new_first = (*first_left, *second_right); let new_second = (next_first_left, next_second_right); merges.push_back((new_first, new_second)); last_merges.push_back((new_first, new_second)); } - } - if last_merges.len() <= 1{ + if last_merges.len() <= 1 { return merges; } } } -pub fn sort_batch<'de,T: ReadableElement + Ord + Clone + Hash + Display + Copy + Send + DeserializeOwned>(mut unsorted_array: &mut Array2){ +pub fn sort_batch< + 'de, + T: ReadableElement + Ord + Clone + Hash + Display + Copy + Send + DeserializeOwned, +>( + mut unsorted_array: &mut Array2, +) { let input_shape = unsorted_array.shape().to_owned(); - let mut partitions:VecDeque<(usize, usize,usize)> = VecDeque::new(); + let mut partitions: VecDeque<(usize, usize, usize)> = VecDeque::new(); partitions.push_back((0, 0, input_shape[0])); - while !partitions.is_empty(){ - let (current_deep,start_index,end_index) = partitions.pop_front().unwrap(); + while !partitions.is_empty() { + let (current_deep, start_index, end_index) = partitions.pop_front().unwrap(); - let buckets = sort_batch_count_buckets(unsorted_array,¤t_deep, start_index, end_index); + let buckets = + sort_batch_count_buckets(unsorted_array, ¤t_deep, start_index, end_index); let mut v: Vec<_> = buckets.keys().cloned().collect(); - v.par_sort_by(|x,y| x.partial_cmp(&y).unwrap()); + v.par_sort_by(|x, y| x.partial_cmp(&y).unwrap()); let mut buckets_startpoint = sort_batch_start_pointer(buckets, v, start_index); - let mut current_pos:usize = start_index; + let mut current_pos: usize = start_index; while current_pos < end_index { - let current_val = unsorted_array.get((current_pos.clone(), current_deep)).unwrap(); - let (start_position, new_position, end_position) = buckets_startpoint.get_mut(current_val).unwrap(); + let current_val = unsorted_array + .get((current_pos.clone(), current_deep)) + .unwrap(); + let (start_position, new_position, end_position) = + buckets_startpoint.get_mut(current_val).unwrap(); - if current_pos < *end_position && current_pos >= *start_position{ + if current_pos < *end_position && current_pos >= *start_position { current_pos += 1; - *new_position+= 1; - }else { + *new_position += 1; + } else { sort_batch_swap(unsorted_array, current_pos, *new_position); - *new_position+= 1; + *new_position += 1; } } - - for (k,(start,_,end)) in buckets_startpoint{ - if start.abs_diff(end) > 1 && current_deep+1 < input_shape[1] { - partitions.push_back((current_deep+1,start,end)); + for (k, (start, _, end)) in buckets_startpoint { + if start.abs_diff(end) > 1 && current_deep + 1 < input_shape[1] { + partitions.push_back((current_deep + 1, start, end)); } } @@ -273,28 +359,34 @@ pub fn sort_batch<'de,T: ReadableElement + Ord + Clone + Hash + Display + Copy + } } -fn sort_batch_swap(mut unsorted_array: &mut Array2, index_a: usize, index_b:usize){ +fn sort_batch_swap( + mut unsorted_array: &mut Array2, + index_a: usize, + index_b: usize, +) { let tmp_a = unsorted_array.row(index_a).to_owned(); - for i in 0..tmp_a.shape()[0]{ + for i in 0..tmp_a.shape()[0] { unsorted_array[(index_a, i)] = unsorted_array[(index_b, i)]; unsorted_array[(index_b, i)] = tmp_a[i]; } - } -fn sort_batch_count_buckets(unsorted_array: &Array2, - col_index: &usize, start:usize, end:usize) -> HashMap{ - let mut bucket_map:HashMap = HashMap::new(); - for (current_pos, item) in unsorted_array.axis_iter(Axis(0)).enumerate().skip(start){ - - if current_pos >= end{ - return bucket_map +fn sort_batch_count_buckets( + unsorted_array: &Array2, + col_index: &usize, + start: usize, + end: usize, +) -> HashMap { + let mut bucket_map: HashMap = HashMap::new(); + for (current_pos, item) in unsorted_array.axis_iter(Axis(0)).enumerate().skip(start) { + if current_pos >= end { + return bucket_map; } - if !bucket_map.contains_key(&item[*col_index]){ - bucket_map.insert(item[*col_index].clone(),1); - }else { + if !bucket_map.contains_key(&item[*col_index]) { + bucket_map.insert(item[*col_index].clone(), 1); + } else { let entry = bucket_map.get_mut(&item[*col_index]).unwrap(); *entry += 1; } @@ -302,13 +394,24 @@ fn sort_batch_count_buckets(unsorted_ar bucket_map } -fn sort_batch_start_pointer(buckets: HashMap, keys: Vec, start:usize) -> HashMap { - let mut start_pointer_map:HashMap = HashMap::new(); +fn sort_batch_start_pointer( + buckets: HashMap, + keys: Vec, + start: usize, +) -> HashMap { + let mut start_pointer_map: HashMap = HashMap::new(); let mut current_pos: usize = start; for key in keys.into_iter() { - start_pointer_map.insert(key.clone(), (current_pos.clone(), current_pos.clone(), current_pos+buckets.get(&key).unwrap())); + start_pointer_map.insert( + key.clone(), + ( + current_pos.clone(), + current_pos.clone(), + current_pos + buckets.get(&key).unwrap(), + ), + ); current_pos += buckets.get(&key).unwrap(); } start_pointer_map -} \ No newline at end of file +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/mod.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/mod.rs index 45082d3..52a6f3b 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/mod.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/mod.rs @@ -1,4 +1,4 @@ -pub mod BatchSort; pub mod BatchListFunctions_duplicates; +pub mod BatchSort; pub mod py_BatchListFunctions_duplicates; -pub mod py_BatchSort; \ No newline at end of file +pub mod py_BatchSort; diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/py_BatchListFunctions_duplicates.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/py_BatchListFunctions_duplicates.rs index f1c9fcd..2d4df0a 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/py_BatchListFunctions_duplicates.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/py_BatchListFunctions_duplicates.rs @@ -1,12 +1,15 @@ +use ndarray::Array2; use std::collections::HashMap; use pyo3::prelude::*; -use pyo3::types::PyList; -use pyo3::Python; use crate::BatchFunctions::BatchListFunctions_duplicates; use crate::BatchFunctions::BatchListFunctions_duplicates::duplicat_result; -use crate::BatchSystem::PythonInterfaces::py_rustBatchTable::{BatchTableI8Py, BatchTableI16Py, BatchTableI32Py, BatchTableI64Py}; +use crate::BatchSystem::BatchTablesImplementation::BatchTable::BatchTable; +use crate::BatchSystem::PythonInterfaces::py_rustBatchTable::{ + BatchTableF64Py, BatchTableI16Py, BatchTableI32Py, BatchTableI64Py, BatchTableI8Py, + BatchTableStringPy, +}; #[pyclass] pub struct BatchListFunctions_duplicates_py {} @@ -14,123 +17,170 @@ pub struct BatchListFunctions_duplicates_py {} #[pyclass] pub struct Duplicat_result_i8 { #[pyo3(get)] - pub counted_entries:usize, + pub counted_entries: usize, #[pyo3(get)] - pub counted_duplicates:usize, + pub counted_duplicates: usize, #[pyo3(get)] - pub entry_most_duplicates:usize, + pub entry_most_duplicates: usize, #[pyo3(get)] pub duplicates_dist: HashMap, #[pyo3(get)] - pub duplicates_by_last_col: HashMap + pub duplicates_by_last_col: HashMap, } #[pyclass] pub struct Duplicat_result_i16 { #[pyo3(get)] - pub counted_entries:usize, + pub counted_entries: usize, #[pyo3(get)] - pub counted_duplicates:usize, + pub counted_duplicates: usize, #[pyo3(get)] - pub entry_most_duplicates:usize, + pub entry_most_duplicates: usize, #[pyo3(get)] pub duplicates_dist: HashMap, #[pyo3(get)] - pub duplicates_by_last_col: HashMap + pub duplicates_by_last_col: HashMap, } #[pyclass] pub struct Duplicat_result_i32 { #[pyo3(get)] - pub counted_entries:usize, + pub counted_entries: usize, #[pyo3(get)] - pub counted_duplicates:usize, + pub counted_duplicates: usize, #[pyo3(get)] - pub entry_most_duplicates:usize, + pub entry_most_duplicates: usize, #[pyo3(get)] pub duplicates_dist: HashMap, #[pyo3(get)] - pub duplicates_by_last_col: HashMap + pub duplicates_by_last_col: HashMap, } #[pyclass] pub struct Duplicat_result_i64 { #[pyo3(get)] - pub counted_entries:usize, + pub counted_entries: usize, #[pyo3(get)] - pub counted_duplicates:usize, + pub counted_duplicates: usize, #[pyo3(get)] - pub entry_most_duplicates:usize, + pub entry_most_duplicates: usize, #[pyo3(get)] pub duplicates_dist: HashMap, #[pyo3(get)] - pub duplicates_by_last_col: HashMap + pub duplicates_by_last_col: HashMap, } - - #[pymethods] impl BatchListFunctions_duplicates_py { - #[new] pub fn new() -> BatchListFunctions_duplicates_py { - BatchListFunctions_duplicates_py { } + BatchListFunctions_duplicates_py {} } #[pyo3(text_signature = "($self, batchtable, calc_duplicates_by_last_col)")] - pub fn count_duplicates_on_sorted_list_i8(&mut self, batchtable_i8: &BatchTableI8Py, calc_duplicates_by_last_col:bool) -> Duplicat_result_i8{ - let mut ground_batchtable = batchtable_i8.batchtable.clone(); - let duplicates_result: duplicat_result = BatchListFunctions_duplicates::count_duplicates_on_sorted_list(ground_batchtable, calc_duplicates_by_last_col); - - Duplicat_result_i8{ + pub fn count_duplicates_on_sorted_list_i8( + &mut self, + batchtable_i8: &BatchTableI8Py, + calc_duplicates_by_last_col: bool, + ) -> Duplicat_result_i8 { + let ground_batchtable = batchtable_i8.batchtable.clone(); + let duplicates_result: duplicat_result = + BatchListFunctions_duplicates::count_duplicates_on_sorted_list( + ground_batchtable, + calc_duplicates_by_last_col, + ); + + Duplicat_result_i8 { counted_entries: duplicates_result.counted_entries, counted_duplicates: duplicates_result.counted_duplicates, entry_most_duplicates: duplicates_result.entry_most_duplicates, duplicates_dist: duplicates_result.duplicates_dist, - duplicates_by_last_col: duplicates_result.duplicates_by_last_col + duplicates_by_last_col: duplicates_result.duplicates_by_last_col, } } #[pyo3(text_signature = "($self, batchtable, calc_duplicates_by_last_col)")] - pub fn count_duplicates_on_sorted_list_i16(&mut self, batchtable_i16: &BatchTableI16Py, calc_duplicates_by_last_col:bool) -> Duplicat_result_i16{ - let mut ground_batchtable = batchtable_i16.batchtable.clone(); - let duplicates_result: duplicat_result = BatchListFunctions_duplicates::count_duplicates_on_sorted_list(ground_batchtable, calc_duplicates_by_last_col); - - Duplicat_result_i16{ + pub fn count_duplicates_on_sorted_list_i16( + &mut self, + batchtable_i16: &BatchTableI16Py, + calc_duplicates_by_last_col: bool, + ) -> Duplicat_result_i16 { + let ground_batchtable = batchtable_i16.batchtable.clone(); + let duplicates_result: duplicat_result = + BatchListFunctions_duplicates::count_duplicates_on_sorted_list( + ground_batchtable, + calc_duplicates_by_last_col, + ); + + Duplicat_result_i16 { counted_entries: duplicates_result.counted_entries, counted_duplicates: duplicates_result.counted_duplicates, entry_most_duplicates: duplicates_result.entry_most_duplicates, duplicates_dist: duplicates_result.duplicates_dist, - duplicates_by_last_col: duplicates_result.duplicates_by_last_col + duplicates_by_last_col: duplicates_result.duplicates_by_last_col, } } #[pyo3(text_signature = "($self, batchtable, calc_duplicates_by_last_col)")] - pub fn count_duplicates_on_sorted_list_i32(&mut self, batchtable_i32: &BatchTableI32Py, calc_duplicates_by_last_col:bool) -> Duplicat_result_i32{ - let mut ground_batchtable = batchtable_i32.batchtable.clone(); - let duplicates_result: duplicat_result = BatchListFunctions_duplicates::count_duplicates_on_sorted_list(ground_batchtable, calc_duplicates_by_last_col); - - Duplicat_result_i32{ + pub fn count_duplicates_on_sorted_list_i32( + &mut self, + batchtable_i32: &BatchTableI32Py, + calc_duplicates_by_last_col: bool, + ) -> Duplicat_result_i32 { + let ground_batchtable = batchtable_i32.batchtable.clone(); + let duplicates_result: duplicat_result = + BatchListFunctions_duplicates::count_duplicates_on_sorted_list( + ground_batchtable, + calc_duplicates_by_last_col, + ); + + Duplicat_result_i32 { counted_entries: duplicates_result.counted_entries, counted_duplicates: duplicates_result.counted_duplicates, entry_most_duplicates: duplicates_result.entry_most_duplicates, duplicates_dist: duplicates_result.duplicates_dist, - duplicates_by_last_col: duplicates_result.duplicates_by_last_col + duplicates_by_last_col: duplicates_result.duplicates_by_last_col, } } #[pyo3(text_signature = "($self, batchtable, calc_duplicates_by_last_col)")] - pub fn count_duplicates_on_sorted_list_i64(&mut self, batchtable_i64: &BatchTableI64Py, calc_duplicates_by_last_col:bool) -> Duplicat_result_i64{ - let mut ground_batchtable = batchtable_i64.batchtable.clone(); - let duplicates_result: duplicat_result = BatchListFunctions_duplicates::count_duplicates_on_sorted_list(ground_batchtable, calc_duplicates_by_last_col); - - Duplicat_result_i64{ + pub fn count_duplicates_on_sorted_list_i64( + &mut self, + batchtable_i64: &BatchTableI64Py, + calc_duplicates_by_last_col: bool, + ) -> Duplicat_result_i64 { + let ground_batchtable = batchtable_i64.batchtable.clone(); + let duplicates_result: duplicat_result = + BatchListFunctions_duplicates::count_duplicates_on_sorted_list( + ground_batchtable, + calc_duplicates_by_last_col, + ); + + Duplicat_result_i64 { counted_entries: duplicates_result.counted_entries, counted_duplicates: duplicates_result.counted_duplicates, entry_most_duplicates: duplicates_result.entry_most_duplicates, duplicates_dist: duplicates_result.duplicates_dist, - duplicates_by_last_col: duplicates_result.duplicates_by_last_col + duplicates_by_last_col: duplicates_result.duplicates_by_last_col, } } -} + #[pyo3(text_signature = "($self, batchtable, calc_duplicates_by_last_col)")] + pub fn count_duplicates_on_sorted_list_f64( + &mut self, + batchtable_f64: &BatchTableF64Py, + calc_duplicates_by_last_col: bool, + ) { + let chunk: Array2 = batchtable_f64.batchtable.write().unwrap().load_chunk(0); + } +} +impl BatchListFunctions_duplicates_py { + pub fn count_duplicates_on_sorted_list_string( + &mut self, + batchtable_string: &BatchTableStringPy, + calc_duplicates_by_last_col: bool, + ) -> Array2 { + let chunk: Array2 = batchtable_string.batchtable.write().unwrap().load_chunk(0); + chunk + } +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/py_BatchSort.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/py_BatchSort.rs index ac9502a..b98e7ad 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/py_BatchSort.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchFunctions/py_BatchSort.rs @@ -1,45 +1,43 @@ use crate::BatchFunctions::BatchSort::sort; -use crate::BatchSystem::PythonInterfaces::py_rustBatchTable::{BatchTableF32Py, BatchTableF64Py, BatchTableI16Py, BatchTableI32Py, BatchTableI64Py, BatchTableI8Py}; +use crate::BatchSystem::PythonInterfaces::py_rustBatchTable::{ + BatchTableI16Py, BatchTableI32Py, BatchTableI64Py, BatchTableI8Py, +}; use pyo3::prelude::*; -use pyo3::types::PyList; -use pyo3::Python; - #[pyclass] -pub struct BatchSorter_Radix_py { - path: String +pub struct BatchSorter_Radix_py { + path: String, } #[pymethods] impl BatchSorter_Radix_py { - #[new] pub fn new(path: String) -> BatchSorter_Radix_py { BatchSorter_Radix_py { path } } #[pyo3(text_signature = "($self, batchtable, tmp_path)")] - pub fn sort_i64(&mut self, batchtable: &BatchTableI64Py){ - let mut original_bt = batchtable.batchtable.clone(); + pub fn sort_i64(&mut self, batchtable: &BatchTableI64Py) { + let original_bt = batchtable.batchtable.clone(); sort::(original_bt, self.path.as_str()) } #[pyo3(text_signature = "($self, batchtable, tmp_path)")] - pub fn sort_i32(&mut self, batchtable: &BatchTableI32Py){ - let mut original_bt = batchtable.batchtable.clone(); + pub fn sort_i32(&mut self, batchtable: &BatchTableI32Py) { + let original_bt = batchtable.batchtable.clone(); sort::(original_bt, self.path.as_str()) } #[pyo3(text_signature = "($self, batchtable, tmp_path)")] - pub fn sort_i16(&mut self, batchtable: &BatchTableI16Py){ - let mut original_bt = batchtable.batchtable.clone(); + pub fn sort_i16(&mut self, batchtable: &BatchTableI16Py) { + let original_bt = batchtable.batchtable.clone(); sort::(original_bt, self.path.as_str()) } #[pyo3(text_signature = "($self, batchtable, tmp_path)")] - pub fn sort_i8(&mut self, batchtable: &BatchTableI8Py){ - let mut original_bt = batchtable.batchtable.clone(); + pub fn sort_i8(&mut self, batchtable: &BatchTableI8Py) { + let original_bt = batchtable.batchtable.clone(); sort::(original_bt, self.path.as_str()) } -} \ No newline at end of file +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchHolder.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchHolder.rs index 077ec16..6738c9e 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchHolder.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchHolder.rs @@ -1,97 +1,142 @@ +use hex; use std::collections::HashMap; -use std::path::{Path, PathBuf}; use std::fs; -use std::rc::Rc; +use std::path::{Path, PathBuf}; use std::sync::{Arc, RwLock}; -use hex; -use digest::{Digest}; -use ndarray::{Data, Dimension, IxDyn}; -use ndarray_npy::{ReadableElement, WritableElement}; -use sha2::{Sha256}; +use digest::Digest; +use ndarray_npy::WritableElement; +use sha2::Sha256; -use crate::BatchSystem::BatchTablesImplementation::BatchTable::{BatchTable, BatchTablesTypWrapper}; +use crate::BatchSystem::BatchTablesImplementation::BatchTable::{ + BatchTable, BatchTablesTypWrapper, +}; use crate::BatchSystem::BatchTablesImplementation::BatchTable_FileSystem::BatchTable_FileSystem; use crate::BatchSystem::BatchTablesImplementation::BatchTable_InMemory::BatchTable_InMemory; #[derive(PartialEq, Clone)] -pub enum MemoryMode{ +pub enum MemoryMode { InMemory, - DirectIo + DirectIo, } - -pub struct BatchHolder{ +pub struct BatchHolder { pub stored_batchtables: HashMap>>, - pub root_folder: PathBuf, + pub root_folder: PathBuf, } -impl BatchHolder{ - +impl BatchHolder { pub fn new(path_str: &str) -> Self { - let stored_batch_tables: HashMap>> = HashMap::new(); + let stored_batch_tables: HashMap>> = + HashMap::new(); let path_obj = Path::new(path_str); if !path_obj.is_dir() && path_obj.exists() { - let message = format!("The given Path is a File {} ... failed by removing this file", path_str); + let message = format!( + "The given Path is a File {} ... failed by removing this file", + path_str + ); fs::remove_file(path_obj).expect(&*message); - }else if path_obj.is_dir() { - let message = format!("The given Path is not empty {} ... failed by removing dir", path_str); + } else if path_obj.is_dir() { + let message = format!( + "The given Path is not empty {} ... failed by removing dir", + path_str + ); fs::remove_dir_all(path_obj).expect(&*message); } let message = format!("Try to create given Path {} ... but its failed", path_str); fs::create_dir_all(Path::new(path_str)).expect(&*message); - BatchHolder { stored_batchtables: stored_batch_tables, root_folder: Path::new(path_str).to_path_buf()} + BatchHolder { + stored_batchtables: stored_batch_tables, + root_folder: Path::new(path_str).to_path_buf(), + } } - pub fn get_batchtable(&mut self, table_name:String) -> Arc> { + pub fn get_batchtable(&mut self, table_name: String) -> Arc> { let mut load_batchtable = self.stored_batchtables.get_mut(&table_name).unwrap(); return load_batchtable.clone(); } - pub fn get_mut_batchtable(&mut self, table_name:String) -> Arc> { + pub fn get_mut_batchtable(&mut self, table_name: String) -> Arc> { let mut load_batchtable = self.stored_batchtables.get(&table_name).unwrap(); return load_batchtable.clone(); } - pub fn create_batchtable(&mut self, table_name:String, chunk_shape:Vec, memory_mode: MemoryMode){ + pub fn create_batchtable( + &mut self, + table_name: String, + chunk_shape: Vec, + memory_mode: MemoryMode, + ) { let h = Self::create_hash(table_name.as_str(), Sha256::default()); - let h_str = hex::encode(h); + let h_str = hex::encode(h); let joined_path = self.root_folder.join(h_str); let new_dir_path_as_str = joined_path.into_os_string().into_string().unwrap(); let path = Path::new(new_dir_path_as_str.as_str()); - if path.exists() && path.is_dir(){ - let message = format!("The given Path is not empty {} ... failed by removing dir", new_dir_path_as_str.clone()); + if path.exists() && path.is_dir() { + let message = format!( + "The given Path is not empty {} ... failed by removing dir", + new_dir_path_as_str.clone() + ); fs::remove_dir_all(Path::new(path.to_str().unwrap())).expect(&*message); - }else if path.exists(){ - let message = format!("The given Path is a File {} ... failed by removing file", new_dir_path_as_str.clone()); + } else if path.exists() { + let message = format!( + "The given Path is a File {} ... failed by removing file", + new_dir_path_as_str.clone() + ); fs::remove_file(Path::new(path.to_str().unwrap())).expect(&*message); } - let message = format!("Try to create given Path {} ... but its failed", new_dir_path_as_str.clone()); + let message = format!( + "Try to create given Path {} ... but its failed", + new_dir_path_as_str.clone() + ); fs::create_dir(Path::new(path.to_str().unwrap())).expect(&*message); if memory_mode == MemoryMode::InMemory { - let batchtable = BatchTable_InMemory{table_folder : new_dir_path_as_str, table_name:table_name.clone(), table_chunk_count:0, chunk_shape, memory_mode, data_holder: vec![], first_chunk_size: 0 }; - self.stored_batchtables.insert(table_name, Arc::new(RwLock::new(BatchTablesTypWrapper::InMemory(batchtable)))); + let batchtable = BatchTable_InMemory { + table_folder: new_dir_path_as_str, + table_name: table_name.clone(), + table_chunk_count: 0, + chunk_shape, + memory_mode, + data_holder: vec![], + first_chunk_size: 0, + }; + self.stored_batchtables.insert( + table_name, + Arc::new(RwLock::new(BatchTablesTypWrapper::InMemory(batchtable))), + ); } else if memory_mode == MemoryMode::DirectIo { - let batchtable = BatchTable_FileSystem{table_folder : new_dir_path_as_str, table_name:table_name.clone(), table_chunk_count:0, chunk_shape, memory_mode, first_chunk_size: 0 }; - self.stored_batchtables.insert(table_name, Arc::new(RwLock::new(BatchTablesTypWrapper::DirectIo(batchtable)))); + let batchtable = BatchTable_FileSystem { + table_folder: new_dir_path_as_str, + table_name: table_name.clone(), + table_chunk_count: 0, + chunk_shape, + memory_mode, + first_chunk_size: 0, + }; + self.stored_batchtables.insert( + table_name, + Arc::new(RwLock::new(BatchTablesTypWrapper::DirectIo(batchtable))), + ); }; - } - pub fn remove_batchtable(&mut self, table_name:String){ - let remove_table = self.stored_batchtables.get_mut(table_name.as_str()).unwrap(); + pub fn remove_batchtable(&mut self, table_name: String) { + let remove_table = self + .stored_batchtables + .get_mut(table_name.as_str()) + .unwrap(); remove_table.write().unwrap().delete(); self.stored_batchtables.remove(table_name.as_str()); } - pub fn clean(&mut self){ + pub fn clean(&mut self) { let mut hashmap = &mut self.stored_batchtables; for (key, value) in hashmap.iter_mut() { value.write().unwrap().delete() @@ -106,5 +151,4 @@ impl BatchHolder{ hasher.update(msg); hasher.finalize().as_slice().to_vec() } - -} \ No newline at end of file +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/BatchTable.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/BatchTable.rs index c7a365c..2091b4d 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/BatchTable.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/BatchTable.rs @@ -1,18 +1,20 @@ -use std::ops::DerefMut; -use std::path::PathBuf; -use ndarray::{Array, ArrayBase, ArrayView, Data, Dimension, RawData}; -use ndarray_npy::{ReadableElement, WritableElement}; -use serde::{Deserialize, Serialize}; -use serde::de::DeserializeOwned; use crate::BatchSystem::BatchHolder::MemoryMode; use crate::BatchSystem::BatchTablesImplementation::BatchTable_FileSystem::BatchTable_FileSystem; use crate::BatchSystem::BatchTablesImplementation::BatchTable_InMemory::BatchTable_InMemory; +use ndarray::{Array, ArrayBase, Data, Dimension, RawData}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use std::path::PathBuf; -pub trait BatchTable{ - - fn add_chunk(&mut self, array: &ArrayBase) where ::Elem: serde::ser::Serialize; +pub trait BatchTable { + fn add_chunk(&mut self, array: &ArrayBase) + where + ::Elem: serde::ser::Serialize; - fn load_chunk(&self, index: usize) -> Array; + fn load_chunk( + &self, + index: usize, + ) -> Array; fn get_size(&self) -> usize; @@ -26,24 +28,24 @@ pub trait BatchTable{ fn get_memory_mode(&self) -> MemoryMode; - fn override_chunk_with_array(&mut self, array: &ArrayBase, index: usize) where ::Elem: serde::ser::Serialize; - + fn override_chunk_with_array( + &mut self, + array: &ArrayBase, + index: usize, + ) where + ::Elem: serde::ser::Serialize; } -pub trait BatchTablePathBased{ - - fn override_chunk_with_file(&mut self, file_path:&str, index:usize); - - fn get_chunk_path(&self, index:usize) -> PathBuf; +pub trait BatchTablePathBased { + fn override_chunk_with_file(&mut self, file_path: &str, index: usize); + fn get_chunk_path(&self, index: usize) -> PathBuf; } -pub trait BatchTableMemoryBased{ - - fn override_chunk_with_pointer(&mut self, vec:Vec, index:usize); - - fn get_raw_data_pointer(&self, index:usize) -> Vec; +pub trait BatchTableMemoryBased { + fn override_chunk_with_pointer(&mut self, vec: Vec, index: usize); + fn get_raw_data_pointer(&self, index: usize) -> Vec; } #[derive(Clone)] @@ -52,99 +54,113 @@ pub enum BatchTablesTypWrapper { DirectIo(BatchTable_FileSystem), } -impl BatchTable for BatchTablesTypWrapper{ - - fn add_chunk(&mut self, array: &ArrayBase) where ::Elem: serde::ser::Serialize{ - match self{ - BatchTablesTypWrapper::InMemory(batchtable) => {batchtable.add_chunk(array)} - BatchTablesTypWrapper::DirectIo(batchtable) => {batchtable.add_chunk(array)} +impl BatchTable for BatchTablesTypWrapper { + fn add_chunk(&mut self, array: &ArrayBase) + where + ::Elem: serde::ser::Serialize, + { + match self { + BatchTablesTypWrapper::InMemory(batchtable) => batchtable.add_chunk(array), + BatchTablesTypWrapper::DirectIo(batchtable) => batchtable.add_chunk(array), } } - fn load_chunk(&self, index: usize) -> Array{ - match self{ - BatchTablesTypWrapper::InMemory(batchtable) => {batchtable.load_chunk(index)} - BatchTablesTypWrapper::DirectIo(batchtable) => {batchtable.load_chunk(index)} + fn load_chunk( + &self, + index: usize, + ) -> Array { + match self { + BatchTablesTypWrapper::InMemory(batchtable) => batchtable.load_chunk(index), + BatchTablesTypWrapper::DirectIo(batchtable) => batchtable.load_chunk(index), } } fn get_size(&self) -> usize { - match self{ - BatchTablesTypWrapper::InMemory(batchtable) => {batchtable.get_size()} - BatchTablesTypWrapper::DirectIo(batchtable) => {batchtable.get_size()} + match self { + BatchTablesTypWrapper::InMemory(batchtable) => batchtable.get_size(), + BatchTablesTypWrapper::DirectIo(batchtable) => batchtable.get_size(), } } fn get_shape(&self) -> Vec { - match self{ - BatchTablesTypWrapper::InMemory(batchtable) => {batchtable.get_shape()} - BatchTablesTypWrapper::DirectIo(batchtable) => {batchtable.get_shape()} + match self { + BatchTablesTypWrapper::InMemory(batchtable) => batchtable.get_shape(), + BatchTablesTypWrapper::DirectIo(batchtable) => batchtable.get_shape(), } } fn get_table_chunk_count(&self) -> usize { - match self{ - BatchTablesTypWrapper::InMemory(batchtable) => {batchtable.get_table_chunk_count()} - BatchTablesTypWrapper::DirectIo(batchtable) => {batchtable.get_table_chunk_count()} + match self { + BatchTablesTypWrapper::InMemory(batchtable) => batchtable.get_table_chunk_count(), + BatchTablesTypWrapper::DirectIo(batchtable) => batchtable.get_table_chunk_count(), } } fn delete(&mut self) { - match self{ - BatchTablesTypWrapper::InMemory(batchtable) => {batchtable.delete()} - BatchTablesTypWrapper::DirectIo(batchtable) => {batchtable.delete()} + match self { + BatchTablesTypWrapper::InMemory(batchtable) => batchtable.delete(), + BatchTablesTypWrapper::DirectIo(batchtable) => batchtable.delete(), } } - fn get_table_name(&self) -> String{ - match self{ - BatchTablesTypWrapper::InMemory(batchtable) => {batchtable.get_table_name()} - BatchTablesTypWrapper::DirectIo(batchtable) => {batchtable.get_table_name()} + fn get_table_name(&self) -> String { + match self { + BatchTablesTypWrapper::InMemory(batchtable) => batchtable.get_table_name(), + BatchTablesTypWrapper::DirectIo(batchtable) => batchtable.get_table_name(), } } fn get_memory_mode(&self) -> MemoryMode { - match self{ - BatchTablesTypWrapper::InMemory(batchtable) => {batchtable.get_memory_mode()} - BatchTablesTypWrapper::DirectIo(batchtable) => {batchtable.get_memory_mode()} + match self { + BatchTablesTypWrapper::InMemory(batchtable) => batchtable.get_memory_mode(), + BatchTablesTypWrapper::DirectIo(batchtable) => batchtable.get_memory_mode(), } } - fn override_chunk_with_array(&mut self, array: &ArrayBase, index: usize) where ::Elem: serde::ser::Serialize{ - match self{ - BatchTablesTypWrapper::InMemory(batchtable) => {batchtable.override_chunk_with_array(array, index)} - BatchTablesTypWrapper::DirectIo(batchtable) => {batchtable.override_chunk_with_array(array, index)} + fn override_chunk_with_array( + &mut self, + array: &ArrayBase, + index: usize, + ) where + ::Elem: serde::ser::Serialize, + { + match self { + BatchTablesTypWrapper::InMemory(batchtable) => { + batchtable.override_chunk_with_array(array, index) + } + BatchTablesTypWrapper::DirectIo(batchtable) => { + batchtable.override_chunk_with_array(array, index) + } } } } -impl BatchTablesTypWrapper{ - - pub fn is_batch_table_pathbased(&self) -> bool{ +impl BatchTablesTypWrapper { + pub fn is_batch_table_pathbased(&self) -> bool { match self { - BatchTablesTypWrapper::InMemory(_) => { false } - BatchTablesTypWrapper::DirectIo(_) => { true } + BatchTablesTypWrapper::InMemory(_) => false, + BatchTablesTypWrapper::DirectIo(_) => true, } } - pub fn is_batch_table_memorybased(&self) -> bool{ + pub fn is_batch_table_memorybased(&self) -> bool { match self { - BatchTablesTypWrapper::InMemory(_) => { true } - BatchTablesTypWrapper::DirectIo(_) => { false } + BatchTablesTypWrapper::InMemory(_) => true, + BatchTablesTypWrapper::DirectIo(_) => false, } } - pub fn to_batch_table_file_system(&mut self) -> Option<&mut BatchTable_FileSystem>{ + pub fn to_batch_table_file_system(&mut self) -> Option<&mut BatchTable_FileSystem> { match self { - BatchTablesTypWrapper::InMemory(_) => { None } - BatchTablesTypWrapper::DirectIo(batchtable) => { Some(batchtable) } + BatchTablesTypWrapper::InMemory(_) => None, + BatchTablesTypWrapper::DirectIo(batchtable) => Some(batchtable), } } - pub fn to_batch_table_in_memory(&mut self) -> Option<&mut BatchTable_InMemory>{match self { - BatchTablesTypWrapper::InMemory(batchtable) => { Some(batchtable) } - BatchTablesTypWrapper::DirectIo(_) => { None } - }} + pub fn to_batch_table_in_memory(&mut self) -> Option<&mut BatchTable_InMemory> { + match self { + BatchTablesTypWrapper::InMemory(batchtable) => Some(batchtable), + BatchTablesTypWrapper::DirectIo(_) => None, + } } - - +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/BatchTable_FileSystem.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/BatchTable_FileSystem.rs index de36bcb..259eff9 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/BatchTable_FileSystem.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/BatchTable_FileSystem.rs @@ -1,60 +1,73 @@ -use std::fmt::format; -use std::fs::File; -use std::fs; -use std::ops::Add; -use std::path::{Path, PathBuf}; -use ndarray::{Array, Array2, ArrayBase, Data, Dimension, Ix2, IxDyn, RawData}; -use ndarray_npy::{NpzReader, NpzWriter, ReadableElement, WritableElement}; -use std::io::{Cursor, Seek, SeekFrom}; -use lz4::{Decoder, EncoderBuilder}; -use std::io::{self, Result}; -use serde::{Deserialize, Serialize}; -use serde::de::DeserializeOwned; use crate::BatchSystem::BatchHolder::MemoryMode; use crate::BatchSystem::BatchTablesImplementation::BatchTable::{BatchTable, BatchTablePathBased}; use crate::BatchSystem::BatchTablesImplementation::DataSerializing::Serializer; use crate::BatchSystem::BatchTablesImplementation::DataSerializing::Serializer::SerializerAlgo; - +use lz4::{Decoder, EncoderBuilder}; +use ndarray::{Array, ArrayBase, Data, Dimension, RawData}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use std::fs; +use std::fs::File; +use std::io::{self, Result}; +use std::io::{Cursor, Seek, SeekFrom}; +use std::ops::Add; +use std::path::{Path, PathBuf}; #[derive(Clone)] -pub struct BatchTable_FileSystem{ +pub struct BatchTable_FileSystem { pub table_folder: String, pub table_name: String, pub table_chunk_count: usize, pub chunk_shape: Vec, pub memory_mode: MemoryMode, - pub(crate) first_chunk_size: usize + pub(crate) first_chunk_size: usize, } -impl BatchTable_FileSystem{ - fn write_file(&mut self, array:&ArrayBase, path:PathBuf) where ::Elem: serde::ser::Serialize{ - let mut serialized_data_stream = Serializer::to_vec(array, SerializerAlgo::Postcard); - serialized_data_stream.seek(SeekFrom::Start(0)).unwrap( ); - - let output_file = File::create(path.clone()).expect(&*format!("Creating file failed: {}", path.to_str().unwrap())); - let mut encoder = EncoderBuilder::new().level(4).build(output_file).unwrap(); - io::copy(&mut serialized_data_stream, &mut encoder).unwrap(); - let (out, res) = encoder.finish(); +impl BatchTable_FileSystem { + fn write_file( + &mut self, + array: &ArrayBase, + path: PathBuf, + ) where + ::Elem: serde::ser::Serialize, + { + let mut serialized_data_stream = Serializer::to_vec(array, SerializerAlgo::SerdeJson); + serialized_data_stream.seek(SeekFrom::Start(0)).unwrap(); + + let output_file = File::create(path.clone()).expect(&*format!( + "Creating file failed: {}", + path.to_str().unwrap() + )); + let mut encoder = EncoderBuilder::new().level(4).build(output_file).unwrap(); + io::copy(&mut serialized_data_stream, &mut encoder).unwrap(); + let (out, res) = encoder.finish(); } - fn read_file<'de,T: DeserializeOwned, D: Dimension + DeserializeOwned>(&self, path: PathBuf) -> Array{ - let mut decoder = Decoder::new(File::open(path.clone()).unwrap_or_else(|_| panic!("Cannot find Path: {}", path.clone().to_str().unwrap()))).unwrap(); - let mut file = Cursor::new(Vec::new()); - io::copy(&mut decoder, &mut file).unwrap(); - - Serializer::from_vec(file, SerializerAlgo::Postcard) + fn read_file<'de, T: DeserializeOwned, D: Dimension + DeserializeOwned>( + &self, + path: PathBuf, + ) -> Array { + let mut decoder = + Decoder::new(File::open(path.clone()).unwrap_or_else(|_| { + panic!("Cannot find Path: {}", path.clone().to_str().unwrap()) + })) + .unwrap(); + let mut file = Cursor::new(Vec::new()); + io::copy(&mut decoder, &mut file).unwrap(); + + Serializer::from_vec(file, SerializerAlgo::SerdeJson) } } impl BatchTablePathBased for BatchTable_FileSystem { - fn override_chunk_with_file(&mut self, file_path:&str, index:usize){ + fn override_chunk_with_file(&mut self, file_path: &str, index: usize) { let chunk_path = self.get_chunk_path(index); fs::remove_file(chunk_path.clone()).unwrap(); fs::copy(file_path.to_string(), chunk_path).expect("copy of existing chunk fails"); } - fn get_chunk_path(&self, index: usize) -> PathBuf { + fn get_chunk_path(&self, index: usize) -> PathBuf { let mut filename = String::new(); filename = filename.add("chunk"); filename = filename.add(index.to_string().as_str()); @@ -65,15 +78,17 @@ impl BatchTablePathBased for BatchTable_FileSystem { } impl BatchTable for BatchTable_FileSystem { - - fn add_chunk(&mut self, array: &ArrayBase) where ::Elem: serde::ser::Serialize{ - if array.shape()[0] > self.chunk_shape[0]{ + fn add_chunk(&mut self, array: &ArrayBase) + where + ::Elem: serde::ser::Serialize, + { + if array.shape()[0] > self.chunk_shape[0] { panic!("The added chunk is larger than the chunk shape"); } - if self.table_chunk_count == 0{ + if self.table_chunk_count == 0 { self.first_chunk_size = array.shape()[0]; - }else if array.shape()[0] > self.first_chunk_size{ + } else if array.shape()[0] > self.first_chunk_size { panic!("The added chunk is larger than the chunk before"); } @@ -82,7 +97,10 @@ impl BatchTable for BatchTable_FileSystem { self.table_chunk_count += 1; } - fn load_chunk(&self, index: usize) -> Array{ + fn load_chunk( + &self, + index: usize, + ) -> Array { let chunk_path = self.get_chunk_path(index); self.read_file(chunk_path) } @@ -91,7 +109,7 @@ impl BatchTable for BatchTable_FileSystem { self.table_chunk_count } - fn get_shape(&self) -> Vec{ + fn get_shape(&self) -> Vec { return self.chunk_shape.clone(); } @@ -103,7 +121,7 @@ impl BatchTable for BatchTable_FileSystem { fs::remove_dir_all(self.table_folder.as_str()).unwrap_or(()); } - fn get_table_name(&self) -> String{ + fn get_table_name(&self) -> String { return self.table_name.clone(); } @@ -111,9 +129,15 @@ impl BatchTable for BatchTable_FileSystem { return self.memory_mode.clone(); } - fn override_chunk_with_array(&mut self, array: &ArrayBase, index: usize) where ::Elem: serde::ser::Serialize{ + fn override_chunk_with_array( + &mut self, + array: &ArrayBase, + index: usize, + ) where + ::Elem: serde::ser::Serialize, + { let chunk_path = self.get_chunk_path(index); fs::remove_file(chunk_path.clone()).unwrap(); self.write_file(array, chunk_path); } -} \ No newline at end of file +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/BatchTable_InMemory.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/BatchTable_InMemory.rs index c8ff80d..3650e99 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/BatchTable_InMemory.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/BatchTable_InMemory.rs @@ -1,19 +1,18 @@ -use std::borrow::BorrowMut; -use std::io; -use std::io::{Cursor, Seek, SeekFrom, Write}; -use digest::generic_array::arr; -use lz4::{Decoder, EncoderBuilder}; -use ndarray::{Array, ArrayBase, Data, Dimension, IxDyn, RawData}; -use ndarray_npy::{NpzReader, NpzWriter, ReadableElement, WritableElement}; -use serde::de::DeserializeOwned; -use serde::Serialize; use crate::BatchSystem::BatchHolder::MemoryMode; -use crate::BatchSystem::BatchTablesImplementation::BatchTable::{BatchTable, BatchTableMemoryBased}; +use crate::BatchSystem::BatchTablesImplementation::BatchTable::{ + BatchTable, BatchTableMemoryBased, +}; use crate::BatchSystem::BatchTablesImplementation::DataSerializing::Serializer; use crate::BatchSystem::BatchTablesImplementation::DataSerializing::Serializer::SerializerAlgo; +use lz4::{Decoder, EncoderBuilder}; +use ndarray::{Array, ArrayBase, Data, Dimension, RawData}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use std::io; +use std::io::{Cursor, Seek, SeekFrom}; #[derive(Clone)] -pub struct BatchTable_InMemory{ +pub struct BatchTable_InMemory { pub table_folder: String, pub table_name: String, pub table_chunk_count: usize, @@ -21,35 +20,46 @@ pub struct BatchTable_InMemory{ pub memory_mode: MemoryMode, pub(crate) first_chunk_size: usize, - pub(crate) data_holder: Vec> + pub(crate) data_holder: Vec>, } -impl BatchTable_InMemory{ - - fn generate_serial_data(&mut self, array: &ArrayBase) -> Vec where ::Elem: serde::ser::Serialize{ +impl BatchTable_InMemory { + fn generate_serial_data( + &mut self, + array: &ArrayBase, + ) -> Vec + where + ::Elem: serde::ser::Serialize, + { let compress_data_stream = Cursor::new(Vec::new()); - let mut serialized_data_stream = Serializer::to_vec(array, SerializerAlgo::Postcard); - serialized_data_stream.seek(SeekFrom::Start(0)).unwrap( ); + let mut serialized_data_stream = Serializer::to_vec(array, SerializerAlgo::SerdeJson); + serialized_data_stream.seek(SeekFrom::Start(0)).unwrap(); - let mut encoder = EncoderBuilder::new().level(4).build(compress_data_stream).unwrap(); + let mut encoder = EncoderBuilder::new() + .level(4) + .build(compress_data_stream) + .unwrap(); io::copy(&mut serialized_data_stream, &mut encoder).unwrap(); let (mut out, res) = encoder.finish(); - let out_vec_pointer= out.into_inner(); + let out_vec_pointer = out.into_inner(); out_vec_pointer } } impl BatchTable for BatchTable_InMemory { - fn add_chunk(&mut self, array: &ArrayBase) where ::Elem: serde::ser::Serialize{ - if array.shape()[0] > self.chunk_shape[0]{ + fn add_chunk(&mut self, array: &ArrayBase) + where + ::Elem: serde::ser::Serialize, + { + if array.shape()[0] > self.chunk_shape[0] { panic!("The added chunk is larger than the chunk shape"); } - if self.table_chunk_count == 0{ + if self.table_chunk_count == 0 { self.first_chunk_size = array.shape()[0]; - }else if array.shape()[0] > self.first_chunk_size{ + } else if array.shape()[0] > self.first_chunk_size { panic!("The added chunk is larger than the chunk before"); } @@ -58,21 +68,24 @@ impl BatchTable for BatchTable_InMemory { self.table_chunk_count += 1; } - fn load_chunk(&self, index: usize) -> Array{ + fn load_chunk( + &self, + index: usize, + ) -> Array { let compress_data_stream = Cursor::new(self.data_holder.get(index).unwrap()); let mut decoder = Decoder::new(compress_data_stream).unwrap(); let mut serde_data_stream = Cursor::new(Vec::new()); io::copy(&mut decoder, &mut serde_data_stream).unwrap(); - Serializer::from_vec(serde_data_stream, SerializerAlgo::Postcard) + Serializer::from_vec(serde_data_stream, SerializerAlgo::SerdeJson) } fn get_size(&self) -> usize { self.table_chunk_count } - fn get_shape(&self) -> Vec{ + fn get_shape(&self) -> Vec { self.chunk_shape.clone() } @@ -84,7 +97,7 @@ impl BatchTable for BatchTable_InMemory { self.data_holder = vec![]; } - fn get_table_name(&self) -> String{ + fn get_table_name(&self) -> String { self.table_name.clone() } @@ -92,20 +105,24 @@ impl BatchTable for BatchTable_InMemory { self.memory_mode.clone() } - fn override_chunk_with_array(&mut self, array: &ArrayBase, index: usize) where ::Elem: serde::ser::Serialize{ + fn override_chunk_with_array( + &mut self, + array: &ArrayBase, + index: usize, + ) where + ::Elem: serde::ser::Serialize, + { let processed_in_memory_data = self.generate_serial_data(array); self.data_holder[index] = processed_in_memory_data; } } impl BatchTableMemoryBased for BatchTable_InMemory { - fn override_chunk_with_pointer(&mut self, vec: Vec, index: usize) { self.data_holder[index] = vec } - fn get_raw_data_pointer(&self, index:usize) -> Vec{ + fn get_raw_data_pointer(&self, index: usize) -> Vec { self.data_holder[index].iter().cloned().collect() } - -} \ No newline at end of file +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/flexbuffers.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/flexbuffers.rs index 580e02d..f91713c 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/flexbuffers.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/flexbuffers.rs @@ -1,14 +1,19 @@ -use std::io::{Cursor, Seek, SeekFrom}; use ndarray::{Array, ArrayBase, Data, Dimension, RawData}; use serde::de::DeserializeOwned; use serde::Serialize; +use std::io::{Cursor, Seek, SeekFrom}; -pub(crate) fn to_vec(array:&ArrayBase) -> Cursor> where ::Elem: serde::ser::Serialize{ +pub(crate) fn to_vec(array: &ArrayBase) -> Cursor> +where + ::Elem: serde::ser::Serialize, +{ let mut serde_data_stream = Cursor::new(flexbuffers::to_vec(array).unwrap()); - serde_data_stream.seek(SeekFrom::Start(0)).unwrap( ); + serde_data_stream.seek(SeekFrom::Start(0)).unwrap(); serde_data_stream } -pub(crate) fn from_vec(mut input:Cursor>) -> Array{ +pub(crate) fn from_vec( + mut input: Cursor>, +) -> Array { flexbuffers::from_slice(input.get_mut()).unwrap() -} \ No newline at end of file +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/json_serde.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/json_serde.rs index c05c92b..72c38b4 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/json_serde.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/json_serde.rs @@ -1,14 +1,19 @@ -use std::io::{Cursor, Seek, SeekFrom}; use ndarray::{Array, ArrayBase, Data, Dimension, RawData}; use serde::de::DeserializeOwned; use serde::Serialize; +use std::io::{Cursor, Seek, SeekFrom}; -pub(crate) fn to_vec(array:&ArrayBase) -> Cursor> where ::Elem: serde::ser::Serialize{ +pub(crate) fn to_vec(array: &ArrayBase) -> Cursor> +where + ::Elem: serde::ser::Serialize, +{ let mut serde_data_stream = Cursor::new(serde_json::to_vec(array).unwrap()); - serde_data_stream.seek(SeekFrom::Start(0)).unwrap( ); + serde_data_stream.seek(SeekFrom::Start(0)).unwrap(); serde_data_stream } -pub(crate) fn from_vec(mut input:Cursor>) -> Array{ +pub(crate) fn from_vec( + mut input: Cursor>, +) -> Array { serde_json::from_slice(input.get_mut()).unwrap() -} \ No newline at end of file +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/mod.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/mod.rs index 16f0af3..9cb492f 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/mod.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/mod.rs @@ -1,3 +1,3 @@ +pub mod flexbuffers; pub mod json_serde; pub mod postcard; -pub mod flexbuffers; \ No newline at end of file diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/postcard.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/postcard.rs index a430663..b42b24e 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/postcard.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Modules/postcard.rs @@ -1,18 +1,20 @@ -use std::io::{Cursor, Seek, SeekFrom}; use ndarray::{Array, ArrayBase, Data, Dimension, RawData}; +use postcard; use serde::de::DeserializeOwned; use serde::Serialize; -use postcard; -use postcard::to_allocvec; - - +use std::io::{Cursor, Seek, SeekFrom}; -pub(crate) fn to_vec(array:&ArrayBase) -> Cursor> where ::Elem: serde::ser::Serialize{ +pub(crate) fn to_vec(array: &ArrayBase) -> Cursor> +where + ::Elem: serde::ser::Serialize, +{ let mut serde_data_stream = Cursor::new(postcard::to_allocvec(array).unwrap()); - serde_data_stream.seek(SeekFrom::Start(0)).unwrap( ); + serde_data_stream.seek(SeekFrom::Start(0)).unwrap(); serde_data_stream } -pub(crate) fn from_vec(mut input:Cursor>) -> Array{ +pub(crate) fn from_vec( + input: Cursor>, +) -> Array { postcard::from_bytes(input.get_ref()).unwrap() -} \ No newline at end of file +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Serializer.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Serializer.rs index a27cd2d..8412c82 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Serializer.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/Serializer.rs @@ -1,10 +1,10 @@ -use std::io::{Cursor, Seek, SeekFrom}; +use crate::BatchSystem::BatchTablesImplementation::DataSerializing::Modules::flexbuffers; +use crate::BatchSystem::BatchTablesImplementation::DataSerializing::Modules::json_serde; +use crate::BatchSystem::BatchTablesImplementation::DataSerializing::Modules::postcard; use ndarray::{Array, ArrayBase, Data, Dimension, RawData}; use serde::de::DeserializeOwned; use serde::Serialize; -use crate::BatchSystem::BatchTablesImplementation::DataSerializing::Modules::json_serde; -use crate::BatchSystem::BatchTablesImplementation::DataSerializing::Modules::postcard; -use crate::BatchSystem::BatchTablesImplementation::DataSerializing::Modules::flexbuffers; +use std::io::{Cursor, Seek, SeekFrom}; pub enum SerializerAlgo { SerdeJson, @@ -12,19 +12,28 @@ pub enum SerializerAlgo { Flexbuffers, } -pub fn to_vec(array:&ArrayBase, serializer:SerializerAlgo) -> Cursor> where ::Elem: serde::ser::Serialize{ +pub fn to_vec( + array: &ArrayBase, + serializer: SerializerAlgo, +) -> Cursor> +where + ::Elem: serde::ser::Serialize, +{ match serializer { - SerializerAlgo::SerdeJson => {json_serde::to_vec(array)} - SerializerAlgo::Postcard => {postcard::to_vec(array)} - SerializerAlgo::Flexbuffers => {flexbuffers::to_vec(array)} + SerializerAlgo::SerdeJson => json_serde::to_vec(array), + SerializerAlgo::Postcard => postcard::to_vec(array), + SerializerAlgo::Flexbuffers => flexbuffers::to_vec(array), } } -pub fn from_vec(mut input:Cursor>, serializer:SerializerAlgo) -> Array{ +pub fn from_vec( + mut input: Cursor>, + serializer: SerializerAlgo, +) -> Array { input.seek(SeekFrom::Start(0)).unwrap(); match serializer { - SerializerAlgo::SerdeJson => {json_serde::from_vec(input)} - SerializerAlgo::Postcard => {postcard::from_vec(input)} - SerializerAlgo::Flexbuffers => {flexbuffers::from_vec(input)} + SerializerAlgo::SerdeJson => json_serde::from_vec(input), + SerializerAlgo::Postcard => postcard::from_vec(input), + SerializerAlgo::Flexbuffers => flexbuffers::from_vec(input), } -} \ No newline at end of file +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/mod.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/mod.rs index a8d2eeb..e16d6e2 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/mod.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/DataSerializing/mod.rs @@ -1,2 +1,2 @@ pub mod Modules; -pub mod Serializer; \ No newline at end of file +pub mod Serializer; diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/mod.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/mod.rs index cf2b9c6..5e975b6 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/mod.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/BatchTablesImplementation/mod.rs @@ -1,5 +1,5 @@ +pub mod BatchTable; pub mod BatchTable_FileSystem; pub mod BatchTable_InMemory; -pub mod BatchTable; pub mod DataSerializing; diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/DType_Converter.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/DType_Converter.rs index 58f5654..98ef779 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/DType_Converter.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/DType_Converter.rs @@ -10,39 +10,39 @@ pub enum BatchDatatyp { NUMPY_FLOAT64, NUMPY_COMPLEX64, NUMPY_COMPLEX128, - UNKOWN + UNKOWN, } pub fn dtyp_int_in_enum(dtype_int: i32) -> BatchDatatyp { match dtype_int { -1 => BatchDatatyp::NUMPY_STRING, - -2 =>BatchDatatyp::PYTHON_OBJECT, - 1 => BatchDatatyp::NUMPY_INT8, - 2 => BatchDatatyp::NUMPY_INT16, - 3 => BatchDatatyp::NUMPY_FLOAT16, - 4 => BatchDatatyp::NUMPY_INT32, - 5 => BatchDatatyp::NUMPY_FLOAT32, - 6 => BatchDatatyp::NUMPY_INT64, - 7 => BatchDatatyp::NUMPY_FLOAT64, - 8 => BatchDatatyp::NUMPY_COMPLEX64, - 9 => BatchDatatyp::NUMPY_COMPLEX128, - _ => BatchDatatyp::UNKOWN + -2 => BatchDatatyp::PYTHON_OBJECT, + 1 => BatchDatatyp::NUMPY_INT8, + 2 => BatchDatatyp::NUMPY_INT16, + 3 => BatchDatatyp::NUMPY_FLOAT16, + 4 => BatchDatatyp::NUMPY_INT32, + 5 => BatchDatatyp::NUMPY_FLOAT32, + 6 => BatchDatatyp::NUMPY_INT64, + 7 => BatchDatatyp::NUMPY_FLOAT64, + 8 => BatchDatatyp::NUMPY_COMPLEX64, + 9 => BatchDatatyp::NUMPY_COMPLEX128, + _ => BatchDatatyp::UNKOWN, } } -pub fn enum_2_dtyp_int(bd_typ: BatchDatatyp) -> i32{ +pub fn enum_2_dtyp_int(bd_typ: BatchDatatyp) -> i32 { match bd_typ { BatchDatatyp::NUMPY_STRING => -1, BatchDatatyp::PYTHON_OBJECT => -2, - BatchDatatyp::NUMPY_INT8 => 1 , - BatchDatatyp::NUMPY_INT16 => 2 , - BatchDatatyp::NUMPY_FLOAT16 => 3, - BatchDatatyp::NUMPY_INT32 => 4 , - BatchDatatyp::NUMPY_FLOAT32 => 5 , - BatchDatatyp::NUMPY_INT64 => 6 , + BatchDatatyp::NUMPY_INT8 => 1, + BatchDatatyp::NUMPY_INT16 => 2, + BatchDatatyp::NUMPY_FLOAT16 => 3, + BatchDatatyp::NUMPY_INT32 => 4, + BatchDatatyp::NUMPY_FLOAT32 => 5, + BatchDatatyp::NUMPY_INT64 => 6, BatchDatatyp::NUMPY_FLOAT64 => 7, BatchDatatyp::NUMPY_COMPLEX64 => 8, - BatchDatatyp::NUMPY_COMPLEX128=> 9, - _ => -100 + BatchDatatyp::NUMPY_COMPLEX128 => 9, + _ => -100, } -} \ No newline at end of file +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/mod.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/mod.rs index 0b9d59f..051bf26 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/mod.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/mod.rs @@ -1,3 +1,3 @@ mod DType_Converter; pub mod py_rustBatchHolder; -pub mod py_rustBatchTable; \ No newline at end of file +pub mod py_rustBatchTable; diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/py_rustBatchHolder.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/py_rustBatchHolder.rs index c7de8ba..f34778f 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/py_rustBatchHolder.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/py_rustBatchHolder.rs @@ -1,89 +1,103 @@ -use std::rc::Rc; use std::env; -use ndarray::Ix2; -use numpy::{dtype, PyReadonlyArray1}; - use pyo3::prelude::*; -use pyo3::types::PyList; -use crate::BatchFunctions::py_BatchListFunctions_duplicates::{Duplicat_result_i16, Duplicat_result_i32, Duplicat_result_i64, Duplicat_result_i8, BatchListFunctions_duplicates_py}; -use crate::BatchSystem::BatchHolder::{BatchHolder, MemoryMode}; -use crate::BatchSystem::BatchTablesImplementation::BatchTable::{BatchTable}; -use crate::BatchFunctions::py_BatchSort::BatchSorter_Radix_py; -use crate::BatchSystem::PythonInterfaces::py_rustBatchTable::{BatchTableF32Py, BatchTableF64Py, BatchTableI16Py, BatchTableI32Py, BatchTableI64Py, BatchTableI8Py}; -use crate::BatchSystem::PythonInterfaces::DType_Converter::{BatchDatatyp}; +use crate::BatchFunctions::py_BatchListFunctions_duplicates::{ + BatchListFunctions_duplicates_py, Duplicat_result_i16, Duplicat_result_i32, + Duplicat_result_i64, Duplicat_result_i8, +}; +use crate::BatchFunctions::py_BatchSort::BatchSorter_Radix_py; +use crate::BatchSystem::BatchHolder::{BatchHolder, MemoryMode}; +use crate::BatchSystem::BatchTablesImplementation::BatchTable::BatchTable; +use crate::BatchSystem::PythonInterfaces::py_rustBatchTable::{ + BatchTableF32Py, BatchTableF64Py, BatchTableI16Py, BatchTableI32Py, BatchTableI64Py, + BatchTableI8Py, BatchTableStringPy, +}; #[pyclass] -pub struct BatchHolder_py { - pub batchholder: BatchHolder +pub struct BatchHolder_py { + pub batchholder: BatchHolder, } #[pymethods] impl BatchHolder_py { - #[new] pub fn new(path: String) -> BatchHolder_py { - BatchHolder_py { batchholder: BatchHolder::new(&path)} + BatchHolder_py { + batchholder: BatchHolder::new(&path), + } } #[pyo3(text_signature = "($self, table_name, chunk_shape)")] - pub fn create_new_table(&mut self, table_name: String, chunk_shape: Vec, memory_mode_string: String){ - if memory_mode_string == "InMemory"{ - self.batchholder.create_batchtable(table_name, chunk_shape, MemoryMode::InMemory); - }else if memory_mode_string == "DirectIO" { - self.batchholder.create_batchtable(table_name, chunk_shape, MemoryMode::DirectIo); - }else { + pub fn create_new_table( + &mut self, + table_name: String, + chunk_shape: Vec, + memory_mode_string: String, + ) { + if memory_mode_string == "InMemory" { + self.batchholder + .create_batchtable(table_name, chunk_shape, MemoryMode::InMemory); + } else if memory_mode_string == "DirectIO" { + self.batchholder + .create_batchtable(table_name, chunk_shape, MemoryMode::DirectIo); + } else { panic!("MemoryMode not found {}", memory_mode_string); } + } + #[pyo3(text_signature = "($self, table_name, chunk_shape)")] + pub fn get_batchtable_i64(&mut self, table_name: String) -> PyResult { + let batchtable = self.batchholder.get_batchtable(table_name); + let shape = batchtable.read().unwrap().get_shape(); + Ok(BatchTableI64Py { batchtable, shape }) } #[pyo3(text_signature = "($self, table_name, chunk_shape)")] - pub fn get_batchtable_i64(&mut self, table_name:String) -> PyResult{ + pub fn get_batchtable_i32(&mut self, table_name: String) -> PyResult { let batchtable = self.batchholder.get_batchtable(table_name); let shape = batchtable.read().unwrap().get_shape(); - Ok(BatchTableI64Py{ batchtable, shape }) + Ok(BatchTableI32Py { batchtable, shape }) } #[pyo3(text_signature = "($self, table_name, chunk_shape)")] - pub fn get_batchtable_i32(&mut self, table_name:String) -> PyResult{ + pub fn get_batchtable_i16(&mut self, table_name: String) -> PyResult { let batchtable = self.batchholder.get_batchtable(table_name); let shape = batchtable.read().unwrap().get_shape(); - Ok(BatchTableI32Py{ batchtable, shape }) + Ok(BatchTableI16Py { batchtable, shape }) } #[pyo3(text_signature = "($self, table_name, chunk_shape)")] - pub fn get_batchtable_i16(&mut self, table_name:String) -> PyResult{ + pub fn get_batchtable_i8(&mut self, table_name: String) -> PyResult { let batchtable = self.batchholder.get_batchtable(table_name); let shape = batchtable.read().unwrap().get_shape(); - Ok(BatchTableI16Py{ batchtable, shape }) + Ok(BatchTableI8Py { batchtable, shape }) } #[pyo3(text_signature = "($self, table_name, chunk_shape)")] - pub fn get_batchtable_i8(&mut self, table_name:String) -> PyResult{ + pub fn get_batchtable_f64(&mut self, table_name: String) -> PyResult { let batchtable = self.batchholder.get_batchtable(table_name); let shape = batchtable.read().unwrap().get_shape(); - Ok(BatchTableI8Py{ batchtable, shape }) + Ok(BatchTableF64Py { batchtable, shape }) } #[pyo3(text_signature = "($self, table_name, chunk_shape)")] - pub fn get_batchtable_f64(&mut self, table_name:String) -> PyResult{ + pub fn get_batchtable_f32(&mut self, table_name: String) -> PyResult { let batchtable = self.batchholder.get_batchtable(table_name); let shape = batchtable.read().unwrap().get_shape(); - Ok(BatchTableF64Py{ batchtable, shape }) + Ok(BatchTableF32Py { batchtable, shape }) } #[pyo3(text_signature = "($self, table_name, chunk_shape)")] - pub fn get_batchtable_f32(&mut self, table_name:String) -> PyResult{ + pub fn get_batchtable_string(&mut self, table_name: String) -> PyResult { let batchtable = self.batchholder.get_batchtable(table_name); let shape = batchtable.read().unwrap().get_shape(); - Ok(BatchTableF32Py{ batchtable, shape }) + Ok(BatchTableStringPy { batchtable, shape }) } #[pyo3(text_signature = "($self)")] - pub fn clean(&mut self){ + pub fn clean(&mut self) { self.batchholder.clean(); } } @@ -99,6 +113,7 @@ fn pyRustBatchsystem(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -108,4 +123,4 @@ fn pyRustBatchsystem(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; Ok(()) -} \ No newline at end of file +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/py_rustBatchTable.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/py_rustBatchTable.rs index a1b3554..257258c 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/py_rustBatchTable.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/BatchSystem/PythonInterfaces/py_rustBatchTable.rs @@ -1,53 +1,52 @@ -use std::sync::{Arc, RwLock}; -use ndarray::{Array, ArrayBase, Dimension, Ix0, Ix2, IxDyn}; +use ndarray::{Array, Array2, Ix2}; use numpy::{PyArray, PyReadonlyArray2, ToPyArray}; -use numpy::{PyReadonlyArrayDyn}; +use std::sync::{Arc, RwLock}; + use pyo3::prelude::*; use pyo3::Python; -use crate::BatchSystem::BatchTablesImplementation::BatchTable::{BatchTable, BatchTablesTypWrapper}; +use crate::BatchSystem::BatchTablesImplementation::BatchTable::{ + BatchTable, BatchTablesTypWrapper, +}; use crate::Utilities::array_helper; #[pyclass()] pub struct BatchTableI64Py { - pub batchtable: Arc>, + pub batchtable: Arc>, #[pyo3(get, set)] pub shape: Vec, - } #[pymethods] impl BatchTableI64Py { - + // wir kriegen ein Chunk von Python und adden ihn in Rust fn add_chunk(&mut self, arr: PyReadonlyArray2) { let arr_view = arr.as_array(); let mut batchtable = self.batchtable.write().unwrap(); batchtable.add_chunk(&arr_view); } - - fn override_chunk(&mut self, arr:PyReadonlyArray2, index: usize) { + // wir kriegen ein Chunk von Python und overwriten ihn in Rust + fn override_chunk(&mut self, arr: PyReadonlyArray2, index: usize) { let arr_view = arr.as_array(); let mut batchtable = self.batchtable.write().unwrap(); batchtable.override_chunk_with_array(&arr_view, index); } - fn print_arr(&mut self){ - let mut batchtable = self.batchtable.write().unwrap(); - for i in 0..batchtable.get_table_chunk_count(){ - let loaded_chunk:Array = batchtable.load_chunk(i); + fn print_arr(&mut self) { + let batchtable = self.batchtable.write().unwrap(); + for i in 0..batchtable.get_table_chunk_count() { + let loaded_chunk: Array = batchtable.load_chunk(i); array_helper::print_array_2D(&loaded_chunk); } } - - fn load_chunk<'py>(&mut self, py: Python<'py>, index:usize) -> &'py PyArray { - let mut batchtable = self.batchtable.write().unwrap(); - let mut loaded_chunk: Array = batchtable.load_chunk(index); + // von Rust zu Python + fn load_chunk<'py>(&mut self, py: Python<'py>, index: usize) -> &'py PyArray { + let batchtable = self.batchtable.write().unwrap(); + let loaded_chunk: Array = batchtable.load_chunk(index); let output_result = loaded_chunk.to_pyarray(py); return output_result; - } - } #[pyclass()] @@ -60,35 +59,32 @@ pub struct BatchTableI32Py { #[pymethods] impl BatchTableI32Py { - fn add_chunk(&mut self, arr: PyReadonlyArray2) { let arr_view = arr.as_array(); let mut batchtable = self.batchtable.write().unwrap(); batchtable.add_chunk(&arr_view); } - fn override_chunk(&mut self, arr:PyReadonlyArray2, index: usize) { + fn override_chunk(&mut self, arr: PyReadonlyArray2, index: usize) { let arr_view = arr.as_array(); let mut batchtable = self.batchtable.write().unwrap(); batchtable.override_chunk_with_array(&arr_view, index); } - fn print_arr(&mut self){ - let mut batchtable = self.batchtable.write().unwrap(); - for i in 0..batchtable.get_table_chunk_count(){ - let loaded_chunk:Array = batchtable.load_chunk(i); + fn print_arr(&mut self) { + let batchtable = self.batchtable.write().unwrap(); + for i in 0..batchtable.get_table_chunk_count() { + let loaded_chunk: Array = batchtable.load_chunk(i); array_helper::print_array_2D(&loaded_chunk); } } - fn load_chunk<'py>(&mut self, py: Python<'py>, index:usize) -> &'py PyArray { - let mut batchtable = self.batchtable.write().unwrap(); - let mut loaded_chunk: Array = batchtable.load_chunk(index); + fn load_chunk<'py>(&mut self, py: Python<'py>, index: usize) -> &'py PyArray { + let batchtable = self.batchtable.write().unwrap(); + let loaded_chunk: Array = batchtable.load_chunk(index); let output_result = loaded_chunk.to_pyarray(py); return output_result; - } - } #[pyclass()] @@ -101,34 +97,32 @@ pub struct BatchTableI16Py { #[pymethods] impl BatchTableI16Py { - fn add_chunk(&mut self, arr: PyReadonlyArray2) { let arr_view = arr.as_array(); let mut batchtable = self.batchtable.write().unwrap(); batchtable.add_chunk(&arr_view); } - fn override_chunk(&mut self, arr:PyReadonlyArray2, index: usize) { + fn override_chunk(&mut self, arr: PyReadonlyArray2, index: usize) { let arr_view = arr.as_array(); let mut batchtable = self.batchtable.write().unwrap(); batchtable.override_chunk_with_array(&arr_view, index); } - fn print_arr(&mut self){ - let mut batchtable = self.batchtable.write().unwrap(); - for i in 0..batchtable.get_table_chunk_count(){ - let loaded_chunk:Array = batchtable.load_chunk(i); + fn print_arr(&mut self) { + let batchtable = self.batchtable.write().unwrap(); + for i in 0..batchtable.get_table_chunk_count() { + let loaded_chunk: Array = batchtable.load_chunk(i); array_helper::print_array_2D(&loaded_chunk); } } - fn load_chunk<'py>(&mut self, py: Python<'py>, index:usize) -> &'py PyArray { - let mut batchtable = self.batchtable.write().unwrap(); - let mut loaded_chunk: Array = batchtable.load_chunk(index); + fn load_chunk<'py>(&mut self, py: Python<'py>, index: usize) -> &'py PyArray { + let batchtable = self.batchtable.write().unwrap(); + let loaded_chunk: Array = batchtable.load_chunk(index); let output_result = loaded_chunk.to_pyarray(py); return output_result; } - } #[pyclass()] @@ -141,34 +135,32 @@ pub struct BatchTableI8Py { #[pymethods] impl BatchTableI8Py { - fn add_chunk(&mut self, arr: PyReadonlyArray2) { let arr_view = arr.as_array(); let mut batchtable = self.batchtable.write().unwrap(); batchtable.add_chunk(&arr_view); } - fn override_chunk(&mut self, arr:PyReadonlyArray2, index: usize) { + fn override_chunk(&mut self, arr: PyReadonlyArray2, index: usize) { let arr_view = arr.as_array(); let mut batchtable = self.batchtable.write().unwrap(); batchtable.override_chunk_with_array(&arr_view, index); } - fn print_arr(&mut self){ - let mut batchtable = self.batchtable.write().unwrap(); - for i in 0..batchtable.get_table_chunk_count(){ - let loaded_chunk:Array = batchtable.load_chunk(i); + fn print_arr(&mut self) { + let batchtable = self.batchtable.write().unwrap(); + for i in 0..batchtable.get_table_chunk_count() { + let loaded_chunk: Array = batchtable.load_chunk(i); array_helper::print_array_2D(&loaded_chunk); } } - fn load_chunk<'py>(&mut self, py: Python<'py>, index:usize) -> &'py PyArray { - let mut batchtable = self.batchtable.write().unwrap(); - let mut loaded_chunk: Array = batchtable.load_chunk(index); + fn load_chunk<'py>(&mut self, py: Python<'py>, index: usize) -> &'py PyArray { + let batchtable = self.batchtable.write().unwrap(); + let loaded_chunk: Array = batchtable.load_chunk(index); let output_result = loaded_chunk.to_pyarray(py); return output_result; } - } #[pyclass()] @@ -181,34 +173,32 @@ pub struct BatchTableF32Py { #[pymethods] impl BatchTableF32Py { - fn add_chunk(&mut self, arr: PyReadonlyArray2) { let arr_view = arr.as_array(); let mut batchtable = self.batchtable.write().unwrap(); batchtable.add_chunk(&arr_view); } - fn override_chunk(&mut self, arr:PyReadonlyArray2, index: usize) { + fn override_chunk(&mut self, arr: PyReadonlyArray2, index: usize) { let arr_view = arr.as_array(); let mut batchtable = self.batchtable.write().unwrap(); batchtable.override_chunk_with_array(&arr_view, index); } - fn print_arr(&mut self){ - let mut batchtable = self.batchtable.write().unwrap(); - for i in 0..batchtable.get_table_chunk_count(){ - let loaded_chunk:Array = batchtable.load_chunk(i); + fn print_arr(&mut self) { + let batchtable = self.batchtable.write().unwrap(); + for i in 0..batchtable.get_table_chunk_count() { + let loaded_chunk: Array = batchtable.load_chunk(i); array_helper::print_array_2D(&loaded_chunk); } } - fn load_chunk<'py>(&mut self, py: Python<'py>, index:usize) -> &'py PyArray { - let mut batchtable = self.batchtable.write().unwrap(); - let mut loaded_chunk: Array = batchtable.load_chunk(index); + fn load_chunk<'py>(&mut self, py: Python<'py>, index: usize) -> &'py PyArray { + let batchtable = self.batchtable.write().unwrap(); + let loaded_chunk: Array = batchtable.load_chunk(index); let output_result = loaded_chunk.to_pyarray(py); return output_result; } - } #[pyclass()] @@ -221,32 +211,84 @@ pub struct BatchTableF64Py { #[pymethods] impl BatchTableF64Py { - fn add_chunk(&mut self, arr: PyReadonlyArray2) { let arr_view = arr.as_array(); let mut batchtable = self.batchtable.write().unwrap(); batchtable.add_chunk(&arr_view); } - fn override_chunk(&mut self, arr:PyReadonlyArray2, index: usize) { + fn override_chunk(&mut self, arr: PyReadonlyArray2, index: usize) { let arr_view = arr.as_array(); let mut batchtable = self.batchtable.write().unwrap(); batchtable.override_chunk_with_array(&arr_view, index); } - fn print_arr(&mut self){ - let mut batchtable = self.batchtable.write().unwrap(); - for i in 0..batchtable.get_table_chunk_count(){ - let loaded_chunk:Array = batchtable.load_chunk(i); + pub fn print_arr(&mut self) { + let batchtable = self.batchtable.write().unwrap(); + for i in 0..batchtable.get_table_chunk_count() { + let loaded_chunk: Array = batchtable.load_chunk(i); array_helper::print_array_2D(&loaded_chunk); } } - fn load_chunk<'py>(&mut self, py: Python<'py>, index:usize) -> &'py PyArray { - let mut batchtable = self.batchtable.write().unwrap(); - let mut loaded_chunk: Array = batchtable.load_chunk(index); + fn load_chunk<'py>(&mut self, py: Python<'py>, index: usize) -> &'py PyArray { + let batchtable = self.batchtable.write().unwrap(); + let loaded_chunk: Array = batchtable.load_chunk(index); let output_result = loaded_chunk.to_pyarray(py); return output_result; } +} + +#[pyclass()] +pub struct BatchTableStringPy { + pub batchtable: Arc>, + + #[pyo3(get)] + pub shape: Vec, +} + +#[pymethods] +impl BatchTableStringPy { + fn add_chunk_wrapper(&mut self, vec: Vec) { + let rust_arr = Array::from_shape_vec((vec.len(), 1), vec).unwrap(); + self.add_chunk(rust_arr) + } + + fn override_chunk_wrapper(&mut self, vec: Vec, index: usize) { + let rust_arr = Array::from_shape_vec((vec.len(), 1), vec).unwrap(); + self.override_chunk(rust_arr, index); + } + + pub fn print_arr(&mut self) { + let batchtable = self.batchtable.write().unwrap(); + for i in 0..batchtable.get_table_chunk_count() { + let loaded_chunk: Array = batchtable.load_chunk(i); + array_helper::print_array_2D(&loaded_chunk); + } + } + + fn get_loaded_string_chunk(&mut self, index: usize) -> Vec { + let loaded_chunk = self.load_chunk(index); + let pylist_chunk = loaded_chunk.into_raw_vec(); + return pylist_chunk; + } +} -} \ No newline at end of file +impl BatchTableStringPy { + fn add_chunk(&mut self, arr: Array2) { + let mut batchtable = self.batchtable.write().unwrap(); + batchtable.add_chunk(&arr); + } + + fn override_chunk(&mut self, arr: Array2, index: usize) { + let mut batchtable = self.batchtable.write().unwrap(); + batchtable.override_chunk_with_array(&arr, index); + } + + pub fn load_chunk(&mut self, index: usize) -> Array { + let batchtable = self.batchtable.write().unwrap(); + let loaded_chunk: Array = batchtable.load_chunk(index); + + return loaded_chunk; + } +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/lib.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/lib.rs index 9ff6d3d..a29f467 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/lib.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/src/lib.rs @@ -1,8 +1,8 @@ extern crate core; -mod BatchSystem; -mod Utilities; -mod BatchFunctions; +pub mod BatchFunctions; +pub mod BatchSystem; +pub mod Utilities; #[path = "../test/test_merge_sort.rs"] mod test_merge_sort; @@ -12,4 +12,3 @@ mod test_batchsystem; #[path = "../test/test_batchlistfunctions.rs"] mod test_batchlistfunctions; - diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/test/test_batchlistfunctions.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/test/test_batchlistfunctions.rs index abed275..27af365 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/test/test_batchlistfunctions.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/test/test_batchlistfunctions.rs @@ -1,21 +1,18 @@ #[cfg(test)] mod tests { use std::env; - use std::fs; - use std::fs::File; + use std::time::Instant; - use digest::generic_array::arr; - use ndarray_npy::{NpzReader, NpzWriter}; + use ndarray; - use ndarray::{Array2, ArrayBase, Axis, Ix2, OwnedRepr}; + use ndarray::{Array2, Axis}; use rand::Rng; - use rand::seq::index::IndexVec::USize; - use crate::Utilities::array_helper; - use rayon::prelude::*; - use crate::BatchFunctions::{BatchListFunctions_duplicates, BatchSort}; + use crate::BatchFunctions::BatchListFunctions_duplicates::duplicat_result; + use crate::BatchFunctions::{BatchListFunctions_duplicates, BatchSort}; use crate::BatchSystem::BatchHolder::{BatchHolder, MemoryMode}; use crate::BatchSystem::BatchTablesImplementation::BatchTable::BatchTable; + use rayon::prelude::*; #[test] fn test_small_array_to_file_find_duplicates() { @@ -30,20 +27,24 @@ mod tests { test.pop(); test.push(".."); test.push("test"); - let mut path = test.to_str().unwrap(); + let path = test.to_str().unwrap(); let mut batchholder: BatchHolder = BatchHolder::new(path); - batchholder.create_batchtable(String::from("test"), vec![height, width], MemoryMode::InMemory); + batchholder.create_batchtable( + String::from("test"), + vec![height, width], + MemoryMode::InMemory, + ); - let mut table = batchholder.get_batchtable(String::from("test")); + let table = batchholder.get_batchtable(String::from("test")); let mut test_vec: Vec> = Vec::new(); { let mut table = table.write().unwrap(); - for i in 0..chunk_count { - let ( mut arr, mut vec) = create_fingerprints_vector(width, height, true); + for _i in 0..chunk_count { + let (arr, vec) = create_fingerprints_vector(width, height, true); test_vec.extend(vec); - table.add_chunk( &arr); + table.add_chunk(&arr); } } @@ -58,7 +59,7 @@ mod tests { test.push(".."); test.push("test"); test.push("tmp"); - let mut path = test.to_str().unwrap(); + let path = test.to_str().unwrap(); let start = Instant::now(); BatchSort::sort::(table.clone(), path); @@ -66,18 +67,21 @@ mod tests { println!("Time elapsed in sort() is: {:?}", duration); let start = Instant::now(); - let result: duplicat_result = BatchListFunctions_duplicates::count_duplicates_on_sorted_list::(table.clone(), true); + let result: duplicat_result = + BatchListFunctions_duplicates::count_duplicates_on_sorted_list::( + table.clone(), + true, + ); let duration = start.elapsed(); println!("Time elapsed in count_duplicates is: {:?}", duration); println!("counted_entries: {:?}", result.counted_entries); println!("counted_duplicates: {:?}", result.counted_duplicates); println!("entry_most_duplicates: {:?}", result.entry_most_duplicates); - for x in 0..chunk_count { let arr: Array2 = table.read().unwrap().load_chunk(x); //array_helper::print_array_2D(&arr); - for (i, mut row) in arr.axis_iter(Axis(0)).enumerate() { + for (i, row) in arr.axis_iter(Axis(0)).enumerate() { for (j, col) in row.iter().enumerate() { assert_eq!(test_vec[(x * height) + i][j], *col); } @@ -91,7 +95,7 @@ mod tests { fn test_random_inplace() { let mut rng = rand::thread_rng(); - for i in 0..1000{ + for i in 0..1000 { println!("Test {}", i); let height = rng.gen_range(1..20); let width = rng.gen_range(1..100); @@ -104,18 +108,22 @@ mod tests { test.pop(); test.push(".."); test.push("test"); - let mut path = test.to_str().unwrap(); + let path = test.to_str().unwrap(); let mut batchholder: BatchHolder = BatchHolder::new(path); - batchholder.create_batchtable(String::from("test"), vec![height, width], MemoryMode::InMemory); + batchholder.create_batchtable( + String::from("test"), + vec![height, width], + MemoryMode::InMemory, + ); - let mut table = batchholder.get_batchtable(String::from("test")); + let table = batchholder.get_batchtable(String::from("test")); { let mut table = table.write().unwrap(); - for i in 0..chunk_count { - let ( mut arr, mut vec) = create_fingerprints_vector(width, height, true); - table.add_chunk( &arr); + for _i in 0..chunk_count { + let (arr, _vec) = create_fingerprints_vector(width, height, true); + table.add_chunk(&arr); } } @@ -127,7 +135,7 @@ mod tests { test.push(".."); test.push("test"); test.push("tmp"); - let mut path = test.to_str().unwrap(); + let path = test.to_str().unwrap(); let start = Instant::now(); BatchSort::sort::(table.clone(), path); @@ -135,7 +143,11 @@ mod tests { println!("Time elapsed in sort() is: {:?}", duration); let start = Instant::now(); - let result: duplicat_result = BatchListFunctions_duplicates::count_duplicates_on_sorted_list::(table.clone(), true); + let result: duplicat_result = + BatchListFunctions_duplicates::count_duplicates_on_sorted_list::( + table.clone(), + true, + ); let duration = start.elapsed(); println!("Time elapsed in count_duplicates is: {:?}", duration); println!("counted_entries: {:?}", result.counted_entries); @@ -145,30 +157,34 @@ mod tests { } } - fn create_fingerprints_vector(width:usize, height:usize, with_mw_last_col:bool) -> (Array2, Vec>) { + fn create_fingerprints_vector( + width: usize, + height: usize, + _with_mw_last_col: bool, + ) -> (Array2, Vec>) { let mut rng = rand::thread_rng(); let mut array: Array2 = Array2::zeros((height, width)); - let mut vec = vec![vec![0;width]; height]; + let mut vec = vec![vec![0; width]; height]; let mut col_pos = 0; - for (index_row, item) in array.iter_mut().enumerate() { - if col_pos < width-1{ - let random_number_bit:i32 = rng.gen_range(0..1); + for (_index_row, item) in array.iter_mut().enumerate() { + if col_pos < width - 1 { + let random_number_bit: i32 = rng.gen_range(0..1); *item = random_number_bit; - }else{ - let random_number_mw:i32 = rng.gen_range(30..60); + } else { + let random_number_mw: i32 = rng.gen_range(30..60); *item = random_number_mw; col_pos = 0; - continue + continue; } col_pos += 1; } - for (i, mut row) in array.axis_iter(Axis(0)).enumerate() { + for (i, row) in array.axis_iter(Axis(0)).enumerate() { for (j, col) in row.iter().enumerate() { vec[i][j] = *col; } } return (array, vec); } -} \ No newline at end of file +} diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/test/test_batchsystem.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/test/test_batchsystem.rs index 84f30b0..44eb5ab 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/test/test_batchsystem.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/test/test_batchsystem.rs @@ -1,15 +1,13 @@ #[cfg(test)] mod tests { use crate::BatchSystem::BatchHolder::{BatchHolder, MemoryMode}; - use crate::Utilities::array_helper; - use digest::generic_array::arr; + use ndarray; - use ndarray::{Array, Array2, ArrayBase, Dimension, Ix2, OwnedRepr}; + use ndarray::{arr2, Array, Array2, Dimension, Ix2}; use rand::Rng; - use rayon::prelude::*; use std::env; - use std::ops::Add; - use crate::BatchSystem::BatchTablesImplementation::BatchTable::{BatchTable, BatchTablesTypWrapper}; + + use crate::BatchSystem::BatchTablesImplementation::BatchTable::BatchTable; #[test] fn test_create_override_batch() { @@ -20,12 +18,12 @@ mod tests { test.pop(); test.push(".."); test.push("test"); - let mut path = test.to_str().unwrap(); + let path = test.to_str().unwrap(); let mut batchholder: BatchHolder = BatchHolder::new(path); - batchholder.create_batchtable(String::from("test"), vec![100,2048], MemoryMode::DirectIo); + batchholder.create_batchtable(String::from("test"), vec![100, 2048], MemoryMode::DirectIo); - let mut table = batchholder.get_batchtable(String::from("test")); + let table = batchholder.get_batchtable(String::from("test")); { let mut table = table.write().unwrap(); @@ -33,18 +31,50 @@ mod tests { table.add_chunk(&first_array_of_fp.clone().into_dyn()); - let mut loaded_array: Array = table.load_chunk(0); + let _loaded_array: Array = table.load_chunk(0); let second_array_of_fp = create_fingerprints_vector(); table.override_chunk_with_array(&second_array_of_fp, 0); - let mut loaded_array: Array = table.load_chunk(0); + let loaded_array: Array = table.load_chunk(0); assert_eq!(loaded_array, second_array_of_fp); assert_ne!(loaded_array, first_array_of_fp); } batchholder.clean(); } + #[test] + fn test_create_override_string_batch() { + let mut test = env::current_exe().unwrap(); + test.pop(); + test.pop(); + test.pop(); + test.pop(); + test.push(".."); + test.push("test1"); + let path = test.to_str().unwrap(); + + let mut batchholder: BatchHolder = BatchHolder::new(path); + batchholder.create_batchtable(String::from("test"), vec![11, 1], MemoryMode::DirectIo); + let table = batchholder.get_batchtable(String::from("test")); + { + let mut table = table.write().unwrap(); + + let first_array_of_strings = create_string_array(); + table.add_chunk(&first_array_of_strings.clone().into_dyn()); + + let _loaded_array: Array = table.load_chunk(0); + + let mut second_array_of_strings = first_array_of_strings.clone(); + second_array_of_strings[[0, 0]] = "O".to_string(); + table.override_chunk_with_array(&second_array_of_strings, 0); + + let loaded_array: Array = table.load_chunk(0); + assert_eq!(loaded_array, second_array_of_strings); + assert_ne!(loaded_array, first_array_of_strings); + } + batchholder.clean(); + } #[test] fn test_create_load_batchsystem() { let mut test = env::current_exe().unwrap(); @@ -53,28 +83,67 @@ mod tests { test.pop(); test.pop(); test.push(".."); - test.push("test"); - let mut path = test.to_str().unwrap(); + test.push("test2"); + let path = test.to_str().unwrap(); let mut batchholder: BatchHolder = BatchHolder::new(path); - batchholder.create_batchtable(String::from("test"), vec![100,2048], MemoryMode::DirectIo); + batchholder.create_batchtable(String::from("test"), vec![100, 2048], MemoryMode::DirectIo); { - let mut table = batchholder.get_batchtable(String::from("test")); - let mut table = table.write().unwrap(); + let table = batchholder.get_batchtable(String::from("test")); + let mut table = table.write().unwrap(); - let array_of_fp = create_fingerprints_vector(); + let array_of_fp = create_fingerprints_vector(); - table.add_chunk(&array_of_fp.to_owned()); + table.add_chunk(&array_of_fp.to_owned()); - let mut loaded_array: Array = table.load_chunk(0); - array_helper::print_array_2D(&loaded_array); + let loaded_array: Array = table.load_chunk(0); + // array_helper::print_array_2D(&loaded_array); - assert_eq!(loaded_array, array_of_fp); + assert_eq!(loaded_array, array_of_fp); } batchholder.clean() } + #[test] + fn test_create_load_string_batchsystem() { + let mut test = env::current_exe().unwrap(); + test.pop(); + test.pop(); + test.pop(); + test.pop(); + test.push(".."); + test.push("test3"); + let path = test.to_str().unwrap(); + + let mut batchholder: BatchHolder = BatchHolder::new(path); + batchholder.create_batchtable(String::from("test"), vec![11, 1], MemoryMode::DirectIo); + + { + let table = batchholder.get_batchtable(String::from("test")); + let mut table = table.write().unwrap(); + + let array_of_strings = create_string_array(); + table.add_chunk(&array_of_strings.clone().into_dyn()); + + let loaded_array: Array = table.load_chunk(0); + // array_helper::print_array_2D(&loaded_array); + + assert_eq!(loaded_array, array_of_strings); + } + batchholder.clean() + } + + fn create_string_array() -> Array2 { + arr2(&[ + ["Cn1c(CN2CCN(c3ccc(Cl)cc3)CC2)nc2ccccc21".to_string()], + ["COC(=O)[C@H](c1ccccc1Cl)N1CCc2sccc2C1".to_string()], + ["O=C(NC1Cc2ccccc2N(C[C@@H](O)CO)C1=O)c1cc2cc(Cl)sc2[nH]1".to_string()], + ["Cc1cccc(C[C@H](NC(=O)c2cc(C(C)(C)C)nn2C)C(=O)NCC#N)c1".to_string()], + ["OC1(C#Cc2ccc(-c3ccccc3)cc2)CN2CCC1CC2".to_string()], + ]) + } + fn create_fingerprints_vector() -> Array2 { let width = 2048; let height = 100; @@ -82,7 +151,7 @@ mod tests { let mut rng = rand::thread_rng(); let mut array: Array2 = Array2::zeros((height, width)); - for (index_row, item) in array.iter_mut().enumerate() { + for (_index_row, item) in array.iter_mut().enumerate() { let random_number: i32 = rng.gen_range(0..2); *item = random_number } diff --git a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/test/test_merge_sort.rs b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/test/test_merge_sort.rs index 69ab227..5a6174c 100644 --- a/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/test/test_merge_sort.rs +++ b/EasyChemML/Utilities/DataUtilities/RustBatchsystem/rustBatchsystem/test/test_merge_sort.rs @@ -1,25 +1,22 @@ #[cfg(test)] mod tests { use std::env; - use std::fs; - use std::fs::File; + use std::time::Instant; - use digest::generic_array::arr; - use ndarray_npy::{NpzReader, NpzWriter}; + use ndarray; - use ndarray::{Array2, ArrayBase, Axis, Ix2, OwnedRepr}; + use ndarray::{Array2, Axis}; use rand::Rng; - use rand::seq::index::IndexVec::USize; - use crate::Utilities::array_helper; + + use crate::BatchFunctions::BatchSort; use rayon::prelude::*; - use crate::BatchFunctions::{BatchListFunctions_duplicates, BatchSort}; - use crate::BatchFunctions::BatchListFunctions_duplicates::duplicat_result; + use crate::BatchSystem::BatchHolder::{BatchHolder, MemoryMode}; - use crate::BatchSystem::BatchTablesImplementation::BatchTable::{BatchTable, BatchTablesTypWrapper}; + use crate::BatchSystem::BatchTablesImplementation::BatchTable::BatchTable; #[test] - fn test_create_small_1Darray(){ - let (mut arr, mut vec) = create_fingerprints_vector(50,100); + fn test_create_small_1Darray() { + let (mut arr, mut vec) = create_fingerprints_vector(50, 100); println!("-----------------"); let start = Instant::now(); BatchSort::sort_batch(&mut arr); @@ -28,7 +25,7 @@ mod tests { println!("Time elapsed in expensive_function() is: {:?}", duration); vec.par_sort_unstable(); - for (i, mut row) in arr.axis_iter(Axis(0)).enumerate() { + for (i, row) in arr.axis_iter(Axis(0)).enumerate() { for (j, col) in row.iter().enumerate() { assert_eq!(vec[i][j], *col); } @@ -36,27 +33,34 @@ mod tests { } #[test] - fn test_medium_array_to_InMemory(){ + fn test_medium_array_to_InMemory() { let height = 10; let width = 10; let chunk_count = 10; let mut test = env::current_exe().unwrap(); - test.pop(); test.pop(); test.pop(); test.pop(); + test.pop(); + test.pop(); + test.pop(); + test.pop(); test.push("test"); test.push("tmp"); - let mut path = test.to_str().unwrap(); + let path = test.to_str().unwrap(); let mut batchholder: BatchHolder = BatchHolder::new(path); - batchholder.create_batchtable(String::from("test"), vec![height, width], MemoryMode::InMemory); + batchholder.create_batchtable( + String::from("test"), + vec![height, width], + MemoryMode::InMemory, + ); - let mut table = batchholder.get_batchtable(String::from("test")); - let mut test_vec:Vec> = Vec::new(); + let table = batchholder.get_batchtable(String::from("test")); + let mut test_vec: Vec> = Vec::new(); { let mut table = table.write().unwrap(); - for i in 0..chunk_count{ - let (mut arr, mut vec) = create_fingerprints_vector(width,height); + for _i in 0..chunk_count { + let (arr, vec) = create_fingerprints_vector(width, height); test_vec.extend(vec); table.add_chunk(&arr); } @@ -66,10 +70,13 @@ mod tests { //println!("--------------------------"); let mut test = env::current_exe().unwrap(); - test.pop(); test.pop(); test.pop(); test.pop(); + test.pop(); + test.pop(); + test.pop(); + test.pop(); test.push("test"); test.push("tmp_mem"); - let mut path = test.to_str().unwrap(); + let path = test.to_str().unwrap(); let start = Instant::now(); BatchSort::sort::(table.clone(), path); @@ -79,9 +86,9 @@ mod tests { println!("The following print is the sorted array from the batchsorter"); { let table = table.read().unwrap(); - for i in 0..chunk_count{ - let arr:Array2 = table.load_chunk(i); - //array_helper::print_array_2D(&arr); + for i in 0..chunk_count { + let _arr: Array2 = table.load_chunk(i); + //array_helper::print_array_2D(&arr); } } @@ -89,11 +96,11 @@ mod tests { println!("The following print is the sorted array from the par_sort_unstable"); println!("--------------------------"); - for x in 0..chunk_count{ - let arr:Array2 = table.read().unwrap().load_chunk(x); - for (i, mut row) in arr.axis_iter(Axis(0)).enumerate() { + for x in 0..chunk_count { + let arr: Array2 = table.read().unwrap().load_chunk(x); + for (i, row) in arr.axis_iter(Axis(0)).enumerate() { for (j, col) in row.iter().enumerate() { - assert_eq!(test_vec[(x*height)+i][j], *col); + assert_eq!(test_vec[(x * height) + i][j], *col); } } } @@ -101,48 +108,56 @@ mod tests { batchholder.clean() } - #[test] - fn test_medium_array_to_InMemory_differentSizes(){ + #[test] + fn test_medium_array_to_InMemory_differentSizes() { let height = 10; let width = 10; let chunk_count = 10; let mut test = env::current_exe().unwrap(); - test.pop(); test.pop(); test.pop(); test.pop(); + test.pop(); + test.pop(); + test.pop(); + test.pop(); test.push("test"); test.push("tmp"); - let mut path = test.to_str().unwrap(); + let path = test.to_str().unwrap(); let mut batchholder: BatchHolder = BatchHolder::new(path); - batchholder.create_batchtable(String::from("test"), vec![height, width], MemoryMode::InMemory); + batchholder.create_batchtable( + String::from("test"), + vec![height, width], + MemoryMode::InMemory, + ); - let mut table = batchholder.get_batchtable(String::from("test")); - let mut test_vec:Vec> = Vec::new(); + let table = batchholder.get_batchtable(String::from("test")); + let mut test_vec: Vec> = Vec::new(); { let mut table = table.write().unwrap(); - let (mut arr, mut vec) = create_fingerprints_vector(width,8); + let (arr, vec) = create_fingerprints_vector(width, 8); test_vec.extend(vec); table.add_chunk(&arr); - for i in 0..chunk_count{ - let (mut arr, mut vec) = create_fingerprints_vector(width,height); + for _i in 0..chunk_count { + let (arr, vec) = create_fingerprints_vector(width, height); test_vec.extend(vec); table.add_chunk(&arr); } - - } test_vec.par_sort_unstable(); //println!("--------------------------"); let mut test = env::current_exe().unwrap(); - test.pop(); test.pop(); test.pop(); test.pop(); + test.pop(); + test.pop(); + test.pop(); + test.pop(); test.push("test"); test.push("tmp_mem"); - let mut path = test.to_str().unwrap(); + let path = test.to_str().unwrap(); let start = Instant::now(); BatchSort::sort::(table.clone(), path); @@ -152,9 +167,9 @@ mod tests { println!("The following print is the sorted array from the batchsorter"); { let table = table.read().unwrap(); - for i in 0..chunk_count{ - let arr:Array2 = table.load_chunk(i); - //array_helper::print_array_2D(&arr); + for i in 0..chunk_count { + let _arr: Array2 = table.load_chunk(i); + //array_helper::print_array_2D(&arr); } } @@ -162,11 +177,11 @@ mod tests { println!("The following print is the sorted array from the par_sort_unstable"); println!("--------------------------"); - for x in 0..chunk_count{ - let arr:Array2 = table.read().unwrap().load_chunk(x); - for (i, mut row) in arr.axis_iter(Axis(0)).enumerate() { + for x in 0..chunk_count { + let arr: Array2 = table.read().unwrap().load_chunk(x); + for (i, row) in arr.axis_iter(Axis(0)).enumerate() { for (j, col) in row.iter().enumerate() { - assert_eq!(test_vec[(x*height)+i][j], *col); + assert_eq!(test_vec[(x * height) + i][j], *col); } } } @@ -175,27 +190,34 @@ mod tests { } #[test] - fn test_medium_array_to_DirectIO(){ + fn test_medium_array_to_DirectIO() { let height = 10; let width = 10; let chunk_count = 5000; let mut test = env::current_exe().unwrap(); - test.pop(); test.pop(); test.pop(); test.pop(); + test.pop(); + test.pop(); + test.pop(); + test.pop(); test.push("test"); test.push("tmp"); - let mut path = test.to_str().unwrap(); + let path = test.to_str().unwrap(); let mut batchholder: BatchHolder = BatchHolder::new(path); - batchholder.create_batchtable(String::from("test"), vec![height, width], MemoryMode::DirectIo); + batchholder.create_batchtable( + String::from("test"), + vec![height, width], + MemoryMode::DirectIo, + ); - let mut table = batchholder.get_batchtable(String::from("test")); - let mut test_vec:Vec> = Vec::new(); + let table = batchholder.get_batchtable(String::from("test")); + let mut test_vec: Vec> = Vec::new(); { let mut table = table.write().unwrap(); - for i in 0..chunk_count{ - let (mut arr, mut vec) = create_fingerprints_vector(width,height); + for _i in 0..chunk_count { + let (arr, vec) = create_fingerprints_vector(width, height); test_vec.extend(vec); table.add_chunk(&arr); } @@ -205,10 +227,13 @@ mod tests { //println!("--------------------------"); let mut test = env::current_exe().unwrap(); - test.pop(); test.pop(); test.pop(); test.pop(); + test.pop(); + test.pop(); + test.pop(); + test.pop(); test.push("test"); test.push("tmp_mem"); - let mut path = test.to_str().unwrap(); + let path = test.to_str().unwrap(); let start = Instant::now(); BatchSort::sort::(table.clone(), path); @@ -216,8 +241,8 @@ mod tests { println!("Time elapsed in expensive_function() is: {:?}", duration); println!("The following print is the sorted array from the batchsorter"); - for i in 0..chunk_count{ - let arr:Array2 = table.read().unwrap().load_chunk(i); + for i in 0..chunk_count { + let _arr: Array2 = table.read().unwrap().load_chunk(i); //array_helper::print_array_2D(&arr); } @@ -226,11 +251,11 @@ mod tests { //array_helper::print_vec_2D(&test_vec); println!("--------------------------"); - for x in 0..chunk_count{ - let arr:Array2 = table.read().unwrap().load_chunk(x); - for (i, mut row) in arr.axis_iter(Axis(0)).enumerate() { + for x in 0..chunk_count { + let arr: Array2 = table.read().unwrap().load_chunk(x); + for (i, row) in arr.axis_iter(Axis(0)).enumerate() { for (j, col) in row.iter().enumerate() { - assert_eq!(test_vec[(x*height)+i][j], *col); + assert_eq!(test_vec[(x * height) + i][j], *col); } } } @@ -238,21 +263,21 @@ mod tests { batchholder.clean() } - fn create_fingerprints_vector(width:usize, height:usize) -> (Array2, Vec>) { + fn create_fingerprints_vector(width: usize, height: usize) -> (Array2, Vec>) { let mut rng = rand::thread_rng(); let mut array: Array2 = Array2::zeros((height, width)); - let mut vec = vec![vec![0;width]; height]; + let mut vec = vec![vec![0; width]; height]; - for (index_row, item) in array.iter_mut().enumerate() { - let random_number:i32 = rng.gen_range(0..2); + for (_index_row, item) in array.iter_mut().enumerate() { + let random_number: i32 = rng.gen_range(0..2); *item = random_number; } - for (i, mut row) in array.axis_iter(Axis(0)).enumerate() { + for (i, row) in array.axis_iter(Axis(0)).enumerate() { for (j, col) in row.iter().enumerate() { vec[i][j] = *col; } } return (array, vec); } -} \ No newline at end of file +}