diff --git a/modules/trainer/GenericTrainer.py b/modules/trainer/GenericTrainer.py index dd17ad76e..a0b069267 100644 --- a/modules/trainer/GenericTrainer.py +++ b/modules/trainer/GenericTrainer.py @@ -193,7 +193,7 @@ def __prune_backups(self, backups_to_keep: int): try: shutil.rmtree(dirpath) except Exception: - print(f"Could not delete old rolling backup {dirpath}") + tqdm.write(f"Could not delete old rolling backup {dirpath}") return @@ -270,7 +270,7 @@ def on_sample_custom(sampler_output: ModelSamplerOutput): ) except Exception: traceback.print_exc() - print("Error during sampling, proceeding without sampling") + tqdm.write("Error during sampling, proceeding without sampling") torch_gc() @@ -303,7 +303,7 @@ def __sample_during_training( # We absolutely do not want to fail training just because the sample definition file becomes missing or broken right before sampling. except Exception: traceback.print_exc() - print("Error during loading the sample definition file, proceeding without sampling") + tqdm.write("Error during loading the sample definition file, proceeding without sampling") sample_params_list = [] if self.model.ema: @@ -428,7 +428,7 @@ def __save_backup_config(self, backup_path): if os.path.isfile(self.config.sample_definition_file_name): shutil.copy2(self.config.sample_definition_file_name, samples_path) - def __backup(self, train_progress: TrainProgress, print_msg: bool = True, print_cb: Callable[[str], None] = print): + def __backup(self, train_progress: TrainProgress, print_msg: bool = True): torch_gc() self.callbacks.on_update_status("Creating backup") @@ -443,7 +443,7 @@ def __backup(self, train_progress: TrainProgress, print_msg: bool = True, print_ try: if print_msg: - print_cb("Creating Backup " + backup_path) + tqdm.write("Creating Backup " + backup_path) self.model_saver.save( self.model, @@ -456,13 +456,13 @@ def __backup(self, train_progress: TrainProgress, print_msg: bool = True, print_ self.__save_backup_config(backup_path) except Exception: traceback.print_exc() - print("Could not save backup. Check your disk space!") + tqdm.write("Could not save backup. Check your disk space!") try: if os.path.isdir(backup_path): shutil.rmtree(backup_path) except Exception: traceback.print_exc() - print("Could not delete partial backup") + tqdm.write("Could not delete partial backup") finally: if self.config.rolling_backup: self.__prune_backups(self.config.rolling_backup_count) @@ -475,7 +475,7 @@ def __backup(self, train_progress: TrainProgress, print_msg: bool = True, print_ torch_gc() - def __save(self, train_progress: TrainProgress, print_msg: bool = True, print_cb: Callable[[str], None] = print): + def __save(self, train_progress: TrainProgress, print_msg: bool = True): torch_gc() self.callbacks.on_update_status("Saving") @@ -486,7 +486,7 @@ def __save(self, train_progress: TrainProgress, print_msg: bool = True, print_cb f"{self.config.save_filename_prefix}{get_string_timestamp()}-save-{train_progress.filename_string()}{self.config.output_model_format.file_extension()}" ) if print_msg: - print_cb("Saving " + save_path) + tqdm.write("Saving " + save_path) try: if self.model.ema: @@ -508,13 +508,13 @@ def __save(self, train_progress: TrainProgress, print_msg: bool = True, print_cb self.model.optimizer.train() except Exception: traceback.print_exc() - print("Could not save model. Check your disk space!") + tqdm.write("Could not save model. Check your disk space!") try: if os.path.isfile(save_path): shutil.rmtree(save_path) except Exception: traceback.print_exc() - print("Could not delete partial save") + tqdm.write("Could not delete partial save") finally: if self.model.ema: self.model.ema.copy_temp_to(self.parameters) @@ -719,9 +719,9 @@ def sample_commands_fun(): if multi.is_master() and (backup or save): self.model.to(self.temp_device) if backup: - self.__backup(train_progress, True, step_tqdm.write) + self.__backup(train_progress, True) if save: - self.__save(train_progress, True, step_tqdm.write) + self.__save(train_progress, True) self.model_setup.setup_train_device(self.model, self.config) self.callbacks.on_update_status("Training ...") diff --git a/modules/util/multi_gpu_util.py b/modules/util/multi_gpu_util.py index d983a2e8c..962434b65 100644 --- a/modules/util/multi_gpu_util.py +++ b/modules/util/multi_gpu_util.py @@ -6,6 +6,8 @@ import torch +from tqdm import tqdm + def is_enabled() -> bool: return torch.distributed.is_available() and torch.distributed.is_initialized() @@ -149,7 +151,7 @@ def parameter_divergence(params: list[torch.Tensor], train_device: torch.device) def warn_parameter_divergence(params: list[torch.Tensor], train_device: torch.device): divergence = parameter_divergence(params, train_device) if divergence is not None and divergence > 0: - print(f"\n\nWARNING: Parameter divergence between GPUs of {divergence}\n\n") + tqdm.write(f"WARNING: Parameter divergence between GPUs of {divergence}") @torch.no_grad()