diff --git a/dnora/executer/executer.py b/dnora/executer/executer.py index 43b4074d..8f3d9bdf 100644 --- a/dnora/executer/executer.py +++ b/dnora/executer/executer.py @@ -145,7 +145,7 @@ def _write( msg.info("No InputFileWriter defined. Won't do anything.") return - msg.header(input_file_writer, "Writing model input file...") + msg.header(input_file_writer, f"Writing model input file for '{self.model.grid().name}'...") # Controls generation of file names using the proper defaults etc. format = format or self._get_default_format() @@ -178,8 +178,8 @@ def _write( ) if not isinstance(output_files, list): output_files = [output_files] - - msg.to_multifile(output_files) + if any(fn for fn in output_files): + msg.to_multifile(output_files) self.model._input_file_exported_to[file_type] = output_files self.model._input_file_export_format[file_type] = self._get_default_format() @@ -196,6 +196,8 @@ def run_model( """Run the main model. Set post_process=False to disable any post-processing that might be defined.""" # Use the method generated by the decorateor, since that will automatically go through all nested grids if present + + self.run_input( model_runner=model_runner, model_folder=model_folder, @@ -214,16 +216,22 @@ def _run( dateformat: Optional[str] = None, post_process: bool = True, post_processors: Optional[list[PostProcessor]] = None, + parent_folder: Optional[str] = None, dry_run: bool = False, **kwargs, ) -> None: """Run the model.""" + + if self.model.parent() is not None: + parent_folder = parent_folder or str(Path(self.model.parent().input_file_exported_to(DnoraFileType.INPUT)[0]).parent) + else: + parent_folder = '' self._dry_run = dry_run file_type = file_type_from_string(file_type) model_runner = model_runner or self._model_runners.get(file_type) if model_runner is None: raise Exception("Define a ModelRunner!") - + msg.header(model_runner, f"Running model '{self.model.grid().name}'...") # Find location of model executable # E.g. For writing GRID and a preferred format of WW3 search for DNORA_GRID_WW3_PATH and DNORA_WW3_PATH model_folder = model_folder or read_environment_variable( @@ -246,12 +254,13 @@ def _run( edge_object=DnoraDataType.GRID, ) - msg.header(model_runner, "Running model...") + msg.plain(f"Using input file: {file_object.get_filepath()}") if not self.dry_run(): outfile = model_runner( file_object=file_object, model_folder=model_folder, + parent_folder=parent_folder, **kwargs, ) if outfile is not None: @@ -264,22 +273,43 @@ def _run( post_processors = post_processors or model_runner.post_processors() if post_processors and post_process: - self.post_process(post_processors, file_object, model_folder, **kwargs) + self.post_process(post_processors, file_object, model_folder, parent_folder,file_type, **kwargs) def post_process( self, post_processors: list[PostProcessor], file_object: FileNames, model_folder, + parent_folder: str, + file_type, **kwargs, ) -> None: """Post processes model run output, e.g. convert to netcdf or move files""" for post_processor in post_processors: msg.header(post_processor, "Post processing...") + if post_processor.for_nest is not None: + model = self.model.nest(get_dict=True)[post_processor.for_nest] + else: + model = self.model + file_objects = [] + for fn in model.input_file_exported_to(post_processor.for_file_type or file_type): + + exported_path = Path(fn) + primary_file = exported_path.stem + primary_folder = str(exported_path.parent) + file_objects.append(FileNames( + model=model, + filename=primary_file, + folder=primary_folder, + obj_type=post_processor.for_file_type or file_type, + format=self._get_default_format(), + edge_object=DnoraDataType.GRID, + )) + post_processor( - model=self.model, - file_object=file_object, + file_object=file_objects, model_folder=model_folder, + parent_folder=parent_folder, **kwargs, ) diff --git a/dnora/executer/inputfile/inputfile_writers.py b/dnora/executer/inputfile/inputfile_writers.py index 5c9f26e6..f4c98770 100644 --- a/dnora/executer/inputfile/inputfile_writers.py +++ b/dnora/executer/inputfile/inputfile_writers.py @@ -7,7 +7,7 @@ import os from pathlib import Path import json - +from dnora.grid.mask import LonLat # Import objects from typing import TYPE_CHECKING, Union, Optional @@ -993,9 +993,9 @@ def __call__( ) # else: # grid_exported_to = exported_files["grid"] - ww3_grid( grid, + model, filename, grid_exported_to, freq1, @@ -1031,26 +1031,34 @@ def __call__( else: filename = file_object.get_filepath() - wind_exported_to = apply_folder_on_server( - exported_files[self.file_type().name.lower()], folder_on_server - ) + __, forcing_filename = recuresively_find_parent_object_and_filename(model,self.file_type().name.lower()) + forcing_exported_to = [forcing_filename] + + if folder_on_server or model.parent() is None: + forcing_exported_to = apply_folder_on_server( + forcing_exported_to, folder_on_server) + elif model.parent() is not None: + parent_folder = Path('../')/Path(model.parent().input_file_exported_to('grid')[0]).parent + forcing_exported_to = [ parent_folder / Path(forcing_filename).name] + + if self.file_type() == DnoraFileType.ICE: if model[self.file_type().name].get("sic", strict=True) is not None: ww3_prnc( f"{filename}.sic", - wind_exported_to, + forcing_exported_to, forcing_type=self.file_type(), subtype="sic", ) if model[self.file_type().name].get("sit", strict=True) is not None: ww3_prnc( f"{filename}.sit", - wind_exported_to, + forcing_exported_to, forcing_type=self.file_type(), subtype="sit", ) else: - ww3_prnc(filename, wind_exported_to, forcing_type=self.file_type()) + ww3_prnc(filename, forcing_exported_to, forcing_type=self.file_type()) return filename @@ -1070,10 +1078,16 @@ def __call__( folder_on_server: str = "", **kwargs, ) -> str: + if model.parent() is not None: + msg.info(f"Assuming spectra will be available from the parent run '{model.parent().grid().name}'") + parent_folder = Path('../')/Path(model.parent().input_file_exported_to('grid')[0]).parent + spectra_exported_to = [ parent_folder / Path(model.parent().start_time().strftime('ww3.%Y%m_spec.nc'))] + else: + spectra_exported_to = apply_folder_on_server( + exported_files["spectra"], folder_on_server + ) msg.to_file(file_object.get_folder() + "/spectral_boundary_files.list") - spectra_exported_to = apply_folder_on_server( - exported_files["spectra"], folder_on_server - ) + ww3_specfile_list( file_object.get_folder() + "/spectral_boundary_files.list", spectra_exported_to, @@ -1097,6 +1111,22 @@ def __call__( return filename +def create_forcing_output_bool_dict(model)-> dict[str, bool]: + """Determines what forcing data will be available for the model, taking to account that a nested model can have used data loaded for the parent model""" + forcing = {} + for ftype in [DnoraDataType.WIND, DnoraDataType.WATERLEVEL, DnoraDataType.CURRENT, DnoraDataType.ICE]: + + obj, __ = recuresively_find_parent_object_and_filename(model,ftype) + forcing[ftype.name.lower()] = obj is not None + + if ftype == DnoraDataType.ICE: + forcing["sit"] = ( + obj is not None and obj.get("sit", strict=True) is not None + ) + forcing["sic"] = ( + obj is not None and obj.get("sic", strict=True) is not None + ) + return forcing class WW3(InputFileWriter): def __call__( @@ -1112,6 +1142,11 @@ def __call__( """To use homogeneous input, set all the variables in order as: homog = {'wind': [1,4]}""" if homog is None: homog = {} + if model.nest() is not None: + for __, nest in model.nest(get_dict=True).items(): + blon, blat = nest.grid().boundary_points() + model.grid().set_output_points(LonLat(lon=blon, lat=blat), append=True) + lons, lats = model.grid().output_points() spectral_output = len(lons) > 0 @@ -1123,29 +1158,26 @@ def __call__( start_time = model.start_time(crop_with="all").strftime("%Y%m%d %H0000") end_time = model.end_time(crop_with="all").strftime("%Y%m%d %H0000") + if file_object.get_filename() == "": filename = file_object.get_folder() + "/ww3_shel.nml" else: filename = file_object.get_filepath() - forcing = {} - forcing["wind"] = model.wind() is not None - forcing["waterlevel"] = model.waterlevel() is not None - forcing["current"] = model.current() is not None - forcing["sit"] = ( - model.ice() is not None and model.ice().get("sit", strict=True) is not None - ) - forcing["sic"] = ( - model.ice() is not None and model.ice().get("sic", strict=True) is not None - ) + forcing = create_forcing_output_bool_dict(model) + #output_nest = model.nest() is not None ww3_shel( - filename, start_time, end_time, stride, forcing, homog, spectral_output, output_vars + filename, start_time, end_time, stride, forcing, homog, spectral_output, output_vars#, output_nest ) # Make inputfiles for the post-processing ounf_filename = file_object.get_folder() + "/ww3_ounf.nml" - ounp_filename = file_object.get_folder() + "/ww3_ounp.nml" ww3_ounf(ounf_filename, start_time, len(model.time()), 3600, output_vars) - ww3_ounp(ounp_filename, start_time, len(model.time()), 3600) - return [filename, ounf_filename, ounp_filename] + if spectral_output > 0: + ounp_filename = file_object.get_folder() + "/ww3_ounp.nml" + ww3_ounp(ounp_filename, start_time, len(model.time()), 3600) + + return [filename, ounf_filename, ounp_filename] + else: + return [filename, ounf_filename] diff --git a/dnora/executer/inputfile/ww3_functions.py b/dnora/executer/inputfile/ww3_functions.py index dde12a5c..9e56ec14 100644 --- a/dnora/executer/inputfile/ww3_functions.py +++ b/dnora/executer/inputfile/ww3_functions.py @@ -12,6 +12,7 @@ def write_block(folder: str, fn: str, fout): def ww3_grid( grid, + model, filename: str, grid_exported_to: str, freq1: float, @@ -167,6 +168,38 @@ def write_inbnd(): fout.write(f" INBND_POINT({n+1:.0f}) = {block:.0f} 1 {flag}\n") fout.write("/\n\n") + # def write_nest_output(nested_grid, first_nest): + # fout.write('&OUTBND_COUNT_NML\n') + # fout.write(' OUTBND_COUNT%N_LINE = 4\n') + # fout.write('/\n') + # fout.write('&OUTBND_LINE_NML\n') + + # # North + # fout.write(f'OUTBND_LINE(1)%X0 = {nested_grid.edges("lon")[0]} ! x index start point\n') + # fout.write(f'OUTBND_LINE(1)%Y0 = {nested_grid.edges("lat")[1]} ! y index start point\n') + # fout.write(f'OUTBND_LINE(1)%DX = {nested_grid.dlon()} ! x-along increment\n') + # fout.write(f'OUTBND_LINE(1)%DY = 0. ! y-along increment\n') + # fout.write(f'OUTBND_LINE(1)%NP = -{nested_grid.nx()} ! number of points\n') + # # West + # fout.write(f'OUTBND_LINE(2)%X0 = {nested_grid.edges("lon")[0]} ! x index start point\n') + # fout.write(f'OUTBND_LINE(2)%Y0 = {nested_grid.edges("lat")[0]} ! y index start point\n') + # fout.write(f'OUTBND_LINE(2)%DX = 0. ! x-along increment\n') + # fout.write(f'OUTBND_LINE(2)%DY = {nested_grid.dlat()} ! y-along increment\n') + # fout.write(f'OUTBND_LINE(2)%NP = {nested_grid.ny()-1} ! number of points\n') + # # South + # fout.write(f'OUTBND_LINE(3)%X0 = {nested_grid.edges("lon")[0]} ! x index start point\n') + # fout.write(f'OUTBND_LINE(3)%Y0 = {nested_grid.edges("lat")[0]} ! y index start point\n') + # fout.write(f'OUTBND_LINE(3)%DX = {nested_grid.dlon()} ! x-along increment\n') + # fout.write(f'OUTBND_LINE(3)%DY = 0. ! y-along increment\n') + # fout.write(f'OUTBND_LINE(3)%NP = {nested_grid.nx()-1} ! number of points\n') + # # East + # fout.write(f'OUTBND_LINE(4)%X0 = {nested_grid.edges("lon")[1]} ! x index start point\n') + # fout.write(f'OUTBND_LINE(4)%Y0 = {nested_grid.edges("lat")[0]} ! y index start point\n') + # fout.write(f'OUTBND_LINE(4)%DX = 0. ! x-along increment\n') + # fout.write(f'OUTBND_LINE(4)%DY = {nested_grid.dlat()} ! y-along increment\n') + # fout.write(f'OUTBND_LINE(4)%NP = {nested_grid.ny()-1} ! number of points\n') + fout.write('/\n') + folder = __file__[:-17] + "/metadata/ww3_grid/" with open(filename, "w") as fout: @@ -197,6 +230,14 @@ def write_inbnd(): write_unst() write_block("inbnd.txt") write_inbnd() + + + + # if model.nest(): + # first_nest = True + # for __, nest in model.nest(get_dict=True).items(): + # write_nest_output(nest.grid(), first_nest) + # first_nest = False write_block("footer.txt") @@ -372,11 +413,13 @@ def write_date(): stride_str = f"{3600*stride:.0f}" restart_start = (pd.to_datetime(start_time)+pd.Timedelta(stride, 'h')).strftime('%Y%m%d %H%M00') - start_times = {"FIELD": start_time, "POINT": start_time, "RESTART": restart_start} - end_times = {"FIELD": end_time, "POINT": end_time, "RESTART": end_time} + start_times = {"FIELD": start_time, "POINT": start_time, "RESTART": restart_start, "BOUNDARY": start_time} + end_times = {"FIELD": end_time, "POINT": end_time, "RESTART": end_time, "BOUNDARY": end_time} - dt = {"FIELD": "3600", "POINT": "3600", "RESTART": stride_str} + dt = {"FIELD": "3600", "POINT": "3600", "RESTART": stride_str, "BOUNDARY": 3600} output_types = ["FIELD", "RESTART"] + # if output_nest: + # output_types.append('BOUNDARY') if spectral_output: output_types.append("POINT") for output_type in output_types: diff --git a/dnora/executer/model_runners.py b/dnora/executer/model_runners.py index c130e635..5e4c846e 100644 --- a/dnora/executer/model_runners.py +++ b/dnora/executer/model_runners.py @@ -11,7 +11,7 @@ import xarray as xr import numpy as np import pandas as pd - +from pathlib import Path from dnora.type_manager.dnora_objects import dnora_objects from dnora.type_manager.dnora_types import data_type_from_string @@ -118,25 +118,46 @@ def __call__(self, file_object: FileNames, model_folder: str) -> None: p.wait() -WW3_DEFAULT_INPUTFILE_NAMES = { - DnoraFileType.SPECTRA: "ww3_bounc.nml", - DnoraFileType.GRID: "ww3_grid.nml", - DnoraFileType.WIND: "ww3_prnc.nml", - DnoraFileType.INPUT: "ww3_shel.nml", - DnoraFileType.CURRENT: "ww3_prnc.nml", - DnoraFileType.WATERLEVEL: "ww3_prnc.nml", - DnoraFileType.ICE: "ww3_prnc.nml", -} +# WW3_DEFAULT_INPUTFILE_NAMES = { +# DnoraFileType.SPECTRA: "ww3_bounc.nml", +# DnoraFileType.GRID: "ww3_grid.nml", +# DnoraFileType.WIND: "ww3_prnc.nml", +# DnoraFileType.INPUT: "ww3_shel.nml", +# DnoraFileType.CURRENT: "ww3_prnc.nml", +# DnoraFileType.WATERLEVEL: "ww3_prnc.nml", +# DnoraFileType.ICE: "ww3_prnc.nml", +# } + +# WW3_DEFAULT_INPUTFILE_NAMES = { +# DnoraFileType.SPECTRA: "ww3_bounc.nml", +# DnoraFileType.GRID: "ww3_grid.nml", +# DnoraFileType.WIND: "ww3_prnc.nml", +# DnoraFileType.INPUT: "ww3_shel.nml", +# DnoraFileType.CURRENT: "ww3_prnc.nml", +# DnoraFileType.WATERLEVEL: "ww3_prnc.nml", +# DnoraFileType.ICE: "ww3_prnc.nml", +# } +def determine_right_file_object(file_object, program): + if not isinstance(file_object, list): + file_object = [file_object] + + file_obj = None + for fo in file_object: + if program in fo.get_filename(): + file_obj = fo + return file_obj class WW3(ModelRunner): - def __init__(self, program: str): + def __init__(self, program: str, for_nest: str = None, file_type: DnoraFileType = None): """E.g. program = 'grid' to run ww3_grid etc.""" self.program = program if program == "shel": self._post_processors = [WW3("ounf"), WW3("ounp")] else: self._post_processors = [] + self.for_nest = for_nest + self.for_file_type = file_type return def preferred_format(self) -> str: @@ -146,7 +167,15 @@ def preferred_format(self) -> str: def post_processors(self) -> list[PostProcessor]: return self._post_processors - def __call__(self, file_object, model_folder, nproc=4, **kwargs) -> None: + def __call__(self, file_object, model_folder, parent_folder: str, nproc=4, **kwargs) -> None: + file_object = determine_right_file_object(file_object, self.program) + + if file_object is None: + msg.plain(f"No inputfile present for {self.program}. Skipping.") + return + elif parent_folder and self.program=='bounc' and not self.for_nest: + msg.plain(f"Will have to wait to exectute {self.program} until parent run is done. Skipping for now.") + return # Copy model executables if needed if model_folder: @@ -156,16 +185,22 @@ def __call__(self, file_object, model_folder, nproc=4, **kwargs) -> None: shutil.copy(from_file, to_file) # Target input file, e.g. ww3_prnc.nml - to_file = WW3_DEFAULT_INPUTFILE_NAMES.get(file_object.obj_type) + #to_file = WW3_DEFAULT_INPUTFILE_NAMES.get(file_object.obj_type) + to_file = f"ww3_{self.program}.nml" + # if self.program == 'shel' and parent_folder: + # nest_output = Path(parent_folder) / Path('nest1.ww3') + # nest_input = Path(file_object.get_folder()) / Path('nest.ww3') + + # shutil.copy(nest_output, nest_input) # Written input files, e.g. 'ww3_prcn_wind.nml', or ['ww3_prnc_ice.nml.sic', 'ww3_prnc_ice.nml.sit'] - if self.program in ["shel", "grid", "bounc", "prnc"]: - from_files = [file_object.get_filename()] - out_files = [f"{file_object.get_folder()}/ww3_{self.program}"] - if file_object.obj_type == DnoraFileType.ICE: - from_files = [f"{from_files[0]}.sic", f"{from_files[0]}.sit"] - out_files = [f"{out_files[0]}_sic", f"{out_files[0]}_sit"] - out_files = [f"{of}.out" for of in out_files] + #if self.program in ["shel", "grid", "bounc", "prnc"]: + from_files = [file_object.get_filename()] + out_files = [f"{file_object.get_folder()}/ww3_{self.program}"] + if file_object.obj_type == DnoraFileType.ICE: + from_files = [f"{from_files[0]}.sic", f"{from_files[0]}.sit"] + out_files = [f"{out_files[0]}_sic", f"{out_files[0]}_sit"] + out_files = [f"{of}.out" for of in out_files] for from_file, out_file in zip(from_files, out_files): from_path = f"{file_object.get_folder()}/{from_file}" @@ -173,7 +208,6 @@ def __call__(self, file_object, model_folder, nproc=4, **kwargs) -> None: msg.plain(f"{from_path} not found. Skipping...") continue if from_file != to_file: - msg.copy_file(from_file, to_file) shutil.copy(from_path, f"{file_object.get_folder()}/{to_file}") @@ -189,11 +223,32 @@ def __call__(self, file_object, model_folder, nproc=4, **kwargs) -> None: p.wait() except FileNotFoundError: msg.advice( - f"ww3_{self.program} not found! 1) Set the environmental variable DNORA_WW3_PATH=/path/to/swashfolder or 2) provide keyword model_folder=/path/to/ww3folder" + f"ww3_{self.program} not found! 1) Set the environmental variable DNORA_WW3_PATH=/path/to/ww3folder or 2) provide keyword model_folder=/path/to/ww3folder" ) - + find_end_of_program(out_files[0], self.program) + if self.program == 'shel': + if file_object.model.nest(): + msg.info('Parent model run done! Setting up boundary pre-processing for nests:') + for key, __ in file_object.model.nest(get_dict=True).items(): + msg.plain(f"\t'{key}'") + self._post_processors.append(WW3('bounc', for_nest=key, file_type=DnoraFileType.SPECTRA)) + else: + self._post_processors = [WW3("ounf"), WW3("ounp")] return +def find_end_of_program(filename, program): + """Searches for 'End of program' in a WW3 output file and warns if it doesn't exist""" + + search_string = "End of program" + + # Open the file and search line by line + with open(filename, 'r') as file: + for line in file: + if search_string in line: + msg.plain(f'Execution of {program} successful!') + return + + msg.warning(f"Seems like {program} could not finish successfully!") class HOS_ocean(ModelRunner): def __init__(self): diff --git a/dnora/executer/post_processors.py b/dnora/executer/post_processors.py index 0a9e18fd..c4fe410d 100644 --- a/dnora/executer/post_processors.py +++ b/dnora/executer/post_processors.py @@ -29,6 +29,8 @@ class PostProcessor(ABC): + for_nest = None + for_file_type = None @abstractmethod def __call__(self, model: ModelRun, file_object: FileNames, **kwargs) -> None: pass diff --git a/dnora/grid/grid.py b/dnora/grid/grid.py index cfebb338..74bc6089 100644 --- a/dnora/grid/grid.py +++ b/dnora/grid/grid.py @@ -124,16 +124,22 @@ def process_grid( obj.set_topo(topo) - def set_boundary_points(self, mask_setter) -> None: + def set_boundary_points(self, mask_setter, append: bool=False) -> None: boundary_mask = mask_setter(self) + if append: + mask = np.logical_or(mask, self.boundary_mask()) self.set_boundary_mask(boundary_mask) - def set_output_points(self, mask_setter) -> None: + def set_output_points(self, mask_setter, append:bool = False) -> None: mask = mask_setter(self) + if append: + mask = np.logical_or(mask, self.output_mask()) self.set_output_mask(mask) - def set_waveseries_points(self, mask_setter) -> None: + def set_waveseries_points(self, mask_setter, append:bool=False) -> None: mask = mask_setter(self) + if append: + mask = np.logical_or(mask, self.waveseries_mask()) self.set_waveseries_mask(mask) def time(self) -> tuple: