From fa99fd50b35f104ba463747a447adf3f19f77073 Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Tue, 30 Dec 2025 04:49:17 +0000 Subject: [PATCH 01/15] start with adding typehints --- meshiphi/cli.py | 65 ++- meshiphi/dataloaders/dataloader_interface.py | 14 +- meshiphi/mesh_generation/environment_mesh.py | 414 ++++++------------- meshiphi/mesh_generation/mesh.py | 23 +- meshiphi/mesh_generation/mesh_builder.py | 135 +++--- 5 files changed, 223 insertions(+), 428 deletions(-) diff --git a/meshiphi/cli.py b/meshiphi/cli.py index 9f3fc5f1..b663d71e 100644 --- a/meshiphi/cli.py +++ b/meshiphi/cli.py @@ -1,13 +1,15 @@ +from __future__ import annotations + import argparse -import json import inspect +import json import logging from meshiphi import __version__ as version -from meshiphi.utils import setup_logging, timed_call -from meshiphi.mesh_generation.mesh_builder import MeshBuilder from meshiphi.mesh_generation.environment_mesh import EnvironmentMesh +from meshiphi.mesh_generation.mesh_builder import MeshBuilder from meshiphi.test_automation.test_automater import TestAutomater +from meshiphi.utils import setup_logging, timed_call logger = logging.getLogger(__name__) @@ -19,7 +21,7 @@ def get_args( mesh_arg: bool = False, format_arg: bool = False, merge_arg: bool = False, -): +) -> argparse.Namespace: """ Adds required command line arguments to all CLI entry points. @@ -36,7 +38,7 @@ def get_args( ap.add_argument( "--version", action="version", - version="%(prog)s {version}".format(version=version), + version=f"%(prog)s {version}", ) # Optional arguments used in all CLI entry points @@ -77,9 +79,7 @@ def get_args( ) if merge_arg: - ap.add_argument( - "merge", help="File location of the environmental mesh to merge with" - ) + ap.add_argument("merge", help="File location of the environmental mesh to merge with") ap.add_argument( "-d", "--directory", @@ -94,14 +94,14 @@ def get_args( @timed_call -def rebuild_mesh_cli(): +def rebuild_mesh_cli() -> None: """ CLI entry point for rebuilding the mesh based on its encoded config files. """ default_output = "rebuild_mesh.output.json" args = get_args(default_output, mesh_arg=True, config_arg=False) - logger.info("{} {}".format(inspect.stack()[0][3][:-4], version)) + logger.info(f"{inspect.stack()[0][3][:-4]} {version}") mesh_json = json.load(args.mesh) config = mesh_json["config"]["mesh_info"] @@ -110,33 +110,35 @@ def rebuild_mesh_cli(): rebuilt_mesh = MeshBuilder(config).build_environmental_mesh() rebuilt_mesh_json = rebuilt_mesh.to_json() - logger.info("Saving mesh to {}".format(args.output)) + logger.info(f"Saving mesh to {args.output}") - json.dump(rebuilt_mesh_json, open(args.output, "w"), indent=4) + with open(args.output, "w") as f: + json.dump(rebuilt_mesh_json, f, indent=4) @timed_call -def create_mesh_cli(): +def create_mesh_cli() -> None: """ CLI entry point for the mesh construction """ default_output = "create_mesh.output.json" args = get_args(default_output) - logger.info("{} {}".format(inspect.stack()[0][3][:-4], version)) + logger.info(f"{inspect.stack()[0][3][:-4]} {version}") config = json.load(args.config) # Discrete Meshing cg = MeshBuilder(config).build_environmental_mesh() - logger.info("Saving mesh to {}".format(args.output)) + logger.info(f"Saving mesh to {args.output}") info = cg.to_json() - json.dump(info, open(args.output, "w"), indent=4) + with open(args.output, "w") as f: + json.dump(info, f, indent=4) @timed_call -def export_mesh_cli(): +def export_mesh_cli() -> None: """ CLI entry point for exporting a mesh to standard formats. Currently supported formats are JSON, GEOJSON, TIF, PNG @@ -145,9 +147,7 @@ def export_mesh_cli(): args = get_args("mesh.json", config_arg=False, mesh_arg=True, format_arg=True) if args.format.upper() == "GEOJSON": - args = get_args( - "mesh_geo.json", config_arg=False, mesh_arg=True, format_arg=True - ) + args = get_args("mesh_geo.json", config_arg=False, mesh_arg=True, format_arg=True) elif args.format.upper() == "TIF": args = get_args("mesh.tif", config_arg=False, mesh_arg=True, format_arg=True) @@ -155,7 +155,7 @@ def export_mesh_cli(): elif args.format.upper() == "PNG": args = get_args("mesh.png", config_arg=False, mesh_arg=True, format_arg=True) - logger.info("{} {}".format(inspect.stack()[0][3][:-4], version)) + logger.info(f"{inspect.stack()[0][3][:-4]} {version}") mesh = json.load(args.mesh) env_mesh = EnvironmentMesh.load_from_json(mesh) @@ -166,7 +166,7 @@ def export_mesh_cli(): @timed_call -def merge_mesh_cli(): +def merge_mesh_cli() -> None: """ CLI entry point for merging two meshes. """ @@ -176,31 +176,27 @@ def merge_mesh_cli(): default_output = "merged_mesh.output.json" args = get_args(default_output, config_arg=False, mesh_arg=True, merge_arg=True) - logger.info("{} {}".format(inspect.stack()[0][3][:-4], version)) + logger.info(f"{inspect.stack()[0][3][:-4]} {version}") - with open(args.mesh.name, "r") as f: + with open(args.mesh.name) as f: mesh1 = json.load(args.mesh) env_mesh1 = EnvironmentMesh.load_from_json(mesh1) if args.directory: - logger.debug( - "Merging multiple meshes from directory {} with input mesh".format( - args.merge - ) - ) + logger.debug(f"Merging multiple meshes from directory {args.merge} with input mesh") merge_dir = args.merge merge_meshes = [f for f in listdir(merge_dir) if isfile(join(merge_dir, f))] for mesh in merge_meshes: path = join(merge_dir, mesh) - with open(path, "r") as f: + with open(path) as f: merge_mesh = json.load(f) env_mesh_merge = EnvironmentMesh.load_from_json(merge_mesh) env_mesh1.merge_mesh(env_mesh_merge) else: - with open(args.merge, "r") as f: + with open(args.merge) as f: mesh2 = json.load(f) env_mesh2 = EnvironmentMesh.load_from_json(mesh2) @@ -208,12 +204,13 @@ def merge_mesh_cli(): merged_mesh_json = env_mesh1.to_json() - logger.info("Saving merged mesh to {}".format(args.output)) - json.dump(merged_mesh_json, open(args.output, "w"), indent=4) + logger.info(f"Saving merged mesh to {args.output}") + with open(args.output, "w") as f: + json.dump(merged_mesh_json, f, indent=4) @timed_call -def meshiphi_test_cli(): +def meshiphi_test_cli() -> None: """ CLI Entry point for automated testing. Assumes you have git and pytest installed. diff --git a/meshiphi/dataloaders/dataloader_interface.py b/meshiphi/dataloaders/dataloader_interface.py index 197a819f..38ab68b8 100644 --- a/meshiphi/dataloaders/dataloader_interface.py +++ b/meshiphi/dataloaders/dataloader_interface.py @@ -1,5 +1,10 @@ +from __future__ import annotations + from abc import ABCMeta, abstractmethod -from meshiphi.mesh_generation.boundary import Boundary +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary class DataLoaderInterface(metaclass=ABCMeta): @@ -16,17 +21,16 @@ def __subclasshook__(cls, subclass): and callable(subclass.get_value) and hasattr(subclass, "get_hom_condition") and callable(subclass.get_hom_condition) - or NotImplemented - ) + ) or NotImplemented @abstractmethod - def get_value(self, bounds: Boundary, agg_type: str): + def get_value(self, bounds: Boundary, agg_type: str) -> dict[str, float]: """Reads values within boundary and returns aggregated value as a dictionary in the form {data_name (str): value (np.float64)}""" raise NotImplementedError @abstractmethod - def get_hom_condition(self, bounds: Boundary, splitting_conds: dict): + def get_hom_condition(self, bounds: Boundary, splitting_conds: dict) -> str: """Reads values within boundary to determine if data is homogeneous or heterogeneous. Return as str""" raise NotImplementedError diff --git a/meshiphi/mesh_generation/environment_mesh.py b/meshiphi/mesh_generation/environment_mesh.py index b5d8df45..37866301 100644 --- a/meshiphi/mesh_generation/environment_mesh.py +++ b/meshiphi/mesh_generation/environment_mesh.py @@ -1,22 +1,23 @@ +from __future__ import annotations + +import collections.abc import json import logging +import os +import subprocess +import tempfile +from pathlib import Path + import geopandas as gpd +import numpy as np import pandas as pd from shapely import wkt -import numpy as np -import subprocess -import os -import tempfile from meshiphi import __version__ as meshiphi_version -from meshiphi.mesh_generation.boundary import Boundary from meshiphi.mesh_generation.aggregated_cellbox import AggregatedCellBox +from meshiphi.mesh_generation.boundary import Boundary from meshiphi.mesh_generation.neighbour_graph import NeighbourGraph - -import collections.abc -from pathlib import Path - logger = logging.getLogger(__name__) @@ -32,7 +33,7 @@ class EnvironmentMesh: """ @classmethod - def load_from_json(cls, mesh_json): + def load_from_json(cls, mesh_json: dict) -> EnvironmentMesh: """ Constructs an Env.Mesh from a given env-mesh json file to be used by other modules (ex.Vessel Performance Modeller). @@ -93,10 +94,15 @@ def load_from_json(cls, mesh_json): agg_cellbox = AggregatedCellBox.from_json(cellbox_json) agg_cellboxes.append(agg_cellbox) neighbour_graph = NeighbourGraph.from_json(mesh_json["neighbour_graph"]) - obj = EnvironmentMesh(bounds, agg_cellboxes, neighbour_graph, config) - return obj + return EnvironmentMesh(bounds, agg_cellboxes, neighbour_graph, config) - def __init__(self, bounds, agg_cellboxes, neighbour_graph, config): + def __init__( + self, + bounds: Boundary, + agg_cellboxes: list, + neighbour_graph: NeighbourGraph, + config: dict, + ) -> None: """ Args: bounds (Boundary): the boundaries of this mesh @@ -110,7 +116,7 @@ def __init__(self, bounds, agg_cellboxes, neighbour_graph, config): self.neighbour_graph = neighbour_graph self.config = config - def query_inside_mesh(self, point): + def query_inside_mesh(self, point: tuple[float, float]) -> bool: """ Returns a bool whether the given point is within the cell @@ -120,15 +126,11 @@ def query_inside_mesh(self, point): inside_mesh (bool) - Boolean stating if point inside mesh """ inside_cell = [ - agg_cell.contains_point(point[0], point[1]) - for agg_cell in self.agg_cellboxes + agg_cell.contains_point(point[0], point[1]) for agg_cell in self.agg_cellboxes ] - if any(inside_cell): - return True - else: - return False + return bool(any(inside_cell)) - def query_index(self, point): + def query_index(self, point: tuple[float, float]) -> str: """ Returns a index of the aggregate cellbox that contains the point @@ -138,19 +140,16 @@ def query_index(self, point): cellbox_index (str) - Cellbox index containing the point """ inside_cell = [ - agg_cell.contains_point(point[0], point[1]) - for agg_cell in self.agg_cellboxes + agg_cell.contains_point(point[0], point[1]) for agg_cell in self.agg_cellboxes ] if any(inside_cell): indices = np.where(inside_cell)[0] if len(indices) > 1: raise Exception("Point within more than one cellbox") - else: - indx = indices[0] - return self.agg_cellboxes[indx].id - else: - raise Exception("Point not within the mesh") + indx = indices[0] + return self.agg_cellboxes[indx].id + raise Exception("Point not within the mesh") def _split_loc(self, point): """ @@ -181,10 +180,9 @@ def _split_loc(self, point): if (cb_width <= min_dcx) or (cb_height <= min_dcy): # Continue if cellbox is at max split depth return False - else: - # Split mesh if at maximum split depth - self.split_and_replace(agg_cellbox_wp.id) - return True + # Split mesh if at maximum split depth + self.split_and_replace(agg_cellbox_wp.id) + return True def split_points(self, points): """ @@ -212,6 +210,7 @@ def get_cellbox(self, cellbox_id): for cellbox in self.agg_cellboxes: if str(cellbox.get_id()) == str(cellbox_id): return cellbox + raise ValueError(f"Cellbox with id {cellbox_id} not found") # Merging meshes def merge_mesh(self, mesh2): @@ -223,12 +222,10 @@ def merge_mesh(self, mesh2): """ if not self.validate_merge_compatibility(mesh2): - raise ValueError( - "The given mesh is not compatible with merging with this mesh" - ) + raise ValueError("The given mesh is not compatible with merging with this mesh") # append config files - if "merged" not in self.config.keys(): + if "merged" not in self.config: self.config["merged"] = [] self.config["merged"].append(mesh2.config) @@ -246,7 +243,7 @@ def merge_mesh(self, mesh2): self.add_cellbox(cellbox) # Appened neighbour graph from mesh2 to this mesh - for index in mesh2.neighbour_graph.get_graph().keys(): + for index in mesh2.neighbour_graph.get_graph(): neighbour_map = mesh2.neighbour_graph.neighbour_graph[index] self.neighbour_graph.add_node(index, neighbour_map) @@ -300,10 +297,7 @@ def validate_merge_compatibility(self, mesh2): return False long_diff = l_bounds.get_long_min() - s_bounds.get_long_min() - if long_diff % l_cell_width != 0: - return False - - return True + return long_diff % l_cell_width == 0 def tie_northern_cellbox_ng(self, north_ext_cellboxes, north_int_cellboxes): """ @@ -337,17 +331,11 @@ def tie_northern_cellbox_ng(self, north_ext_cellboxes, north_int_cellboxes): south_east_neighbours.append(int(cellbox_s.get_id())) for neighbour in south_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["4"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_l.get_id()]["4"].append(neighbour) for neighbour in south_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-1"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-1"].append(neighbour) for neighbour in south_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["3"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_l.get_id()]["3"].append(neighbour) # Tie northen interior cellboxes to northen exterior cellboxes @@ -372,17 +360,11 @@ def tie_northern_cellbox_ng(self, north_ext_cellboxes, north_int_cellboxes): north_east_neighbours.append(int(cellbox_l.get_id())) for neighbour in north_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-4"].append( - neighbour - ) # + self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-4"].append(neighbour) # for neighbour in north_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-3"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-3"].append(neighbour) for neighbour in north_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["1"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_s.get_id()]["1"].append(neighbour) def tie_southern_cellbox_ng(self, south_ext_cellboxes, south_int_cellboxes): """ @@ -416,17 +398,11 @@ def tie_southern_cellbox_ng(self, south_ext_cellboxes, south_int_cellboxes): north_east_neighbours.append(int(cellbox_s.get_id())) for neighbour in north_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-4"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-4"].append(neighbour) for neighbour in north_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-3"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-3"].append(neighbour) for neighbour in north_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["1"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_l.get_id()]["1"].append(neighbour) # Tie southern interior cellboxes to southern exterior cellboxes @@ -451,17 +427,11 @@ def tie_southern_cellbox_ng(self, south_ext_cellboxes, south_int_cellboxes): south_east_neighbours.append(int(cellbox_l.get_id())) for neighbour in south_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["4"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_s.get_id()]["4"].append(neighbour) for neighbour in south_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-1"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-1"].append(neighbour) for neighbour in south_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["3"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_s.get_id()]["3"].append(neighbour) def tie_eastern_cellbox_ng(self, east_ext_cellboxes, east_int_cellboxes): """ @@ -495,17 +465,11 @@ def tie_eastern_cellbox_ng(self, east_ext_cellboxes, east_int_cellboxes): north_west_neighbours.append(int(cellbox_s.get_id())) for neighbour in west_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-2"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-2"].append(neighbour) for neighbour in south_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-1"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-1"].append(neighbour) for neighbour in north_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-3"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-3"].append(neighbour) # Tie eastern interior cellboxes to eastern exterior cellboxes @@ -530,17 +494,11 @@ def tie_eastern_cellbox_ng(self, east_ext_cellboxes, east_int_cellboxes): north_east_neighbours.append(int(cellbox_l.get_id())) for neighbour in east_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["2"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_s.get_id()]["2"].append(neighbour) for neighbour in south_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["3"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_s.get_id()]["3"].append(neighbour) for neighbour in north_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["1"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_s.get_id()]["1"].append(neighbour) def tie_western_cellbox_ng(self, west_ext_cellboxes, west_int_cellboxes): """ @@ -574,17 +532,11 @@ def tie_western_cellbox_ng(self, west_ext_cellboxes, west_int_cellboxes): north_east_neighbours.append(int(cellbox_s.get_id())) for neighbour in east_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["2"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_l.get_id()]["2"].append(neighbour) for neighbour in south_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["3"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_l.get_id()]["3"].append(neighbour) for neighbour in north_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["1"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_l.get_id()]["1"].append(neighbour) # Tie western interior cellboxes to western exterior cellboxes @@ -609,17 +561,11 @@ def tie_western_cellbox_ng(self, west_ext_cellboxes, west_int_cellboxes): north_west_neighbours.append(int(cellbox_l.get_id())) for neighbour in west_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-2"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-2"].append(neighbour) for neighbour in south_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-1"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-1"].append(neighbour) for neighbour in north_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-3"].append( - neighbour - ) + self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-3"].append(neighbour) def get_cellboxes_within_bounds(self, bounds): """ @@ -956,21 +902,13 @@ def split_and_replace(self, cellbox_id): sw_corner_index = self.neighbour_graph.get_neighbours(cellbox_id, "-1") if len(ne_corner_index) > 0: - self.neighbour_graph.update_neighbour( - str(ne_corner_index[0]), "-1", [north_east_index] - ) + self.neighbour_graph.update_neighbour(str(ne_corner_index[0]), "-1", [north_east_index]) if len(se_corner_index) > 0: - self.neighbour_graph.update_neighbour( - str(se_corner_index[0]), "-3", [south_east_index] - ) + self.neighbour_graph.update_neighbour(str(se_corner_index[0]), "-3", [south_east_index]) if len(nw_corner_index) > 0: - self.neighbour_graph.update_neighbour( - str(nw_corner_index[0]), "3", [north_west_index] - ) + self.neighbour_graph.update_neighbour(str(nw_corner_index[0]), "3", [north_west_index]) if len(sw_corner_index) > 0: - self.neighbour_graph.update_neighbour( - str(sw_corner_index[0]), "1", [south_west_index] - ) + self.neighbour_graph.update_neighbour(str(sw_corner_index[0]), "1", [south_west_index]) # update sides neighbours ne_bounds = self.get_cellbox(north_east_index).get_bounds() @@ -988,17 +926,13 @@ def split_and_replace(self, cellbox_id): cellbox_bounds, ne_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour( - str(indx), str(crossing_case), north_east_index - ) + self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), north_east_index) crossing_case = self.neighbour_graph.get_neighbour_case_bounds( cellbox_bounds, nw_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour( - str(indx), str(crossing_case), north_west_index - ) + self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), north_west_index) # update south cellbox neighbours self.neighbour_graph.remove_node_from_neighbours(cellbox_id, 4) @@ -1010,17 +944,13 @@ def split_and_replace(self, cellbox_id): cellbox_bounds, se_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour( - str(indx), str(crossing_case), south_east_index - ) + self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), south_east_index) crossing_case = self.neighbour_graph.get_neighbour_case_bounds( cellbox_bounds, sw_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour( - str(indx), str(crossing_case), south_west_index - ) + self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), south_west_index) # update east cellbox neighbours self.neighbour_graph.remove_node_from_neighbours(cellbox_id, 2) @@ -1032,17 +962,13 @@ def split_and_replace(self, cellbox_id): cellbox_bounds, ne_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour( - str(indx), str(crossing_case), north_east_index - ) + self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), north_east_index) crossing_case = self.neighbour_graph.get_neighbour_case_bounds( cellbox_bounds, se_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour( - str(indx), str(crossing_case), south_east_index - ) + self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), south_east_index) # update west cellbox neighbours self.neighbour_graph.remove_node_from_neighbours(cellbox_id, -2) @@ -1054,17 +980,13 @@ def split_and_replace(self, cellbox_id): cellbox_bounds, nw_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour( - str(indx), str(crossing_case), north_west_index - ) + self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), north_west_index) crossing_case = self.neighbour_graph.get_neighbour_case_bounds( cellbox_bounds, sw_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour( - str(indx), str(crossing_case), south_west_index - ) + self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), south_west_index) # remove original cellbox from mesh. cellbox = self.get_cellbox(cellbox_id) @@ -1092,36 +1014,16 @@ def fill_se_neighbour_map( for indx in south_neighbour_index: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if ( - self.neighbour_graph.get_neighbour_case_bounds( - se_bounds, neighbour_bounds - ) - == 4 - ): + if self.neighbour_graph.get_neighbour_case_bounds(se_bounds, neighbour_bounds) == 4: se_neighbour_map["4"].append(indx) - if ( - self.neighbour_graph.get_neighbour_case_bounds( - se_bounds, neighbour_bounds - ) - == -1 - ): + if self.neighbour_graph.get_neighbour_case_bounds(se_bounds, neighbour_bounds) == -1: se_neighbour_map["-1"].append(indx) for indx in east_neighbour_indx: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if ( - self.neighbour_graph.get_neighbour_case_bounds( - se_bounds, neighbour_bounds - ) - == 2 - ): + if self.neighbour_graph.get_neighbour_case_bounds(se_bounds, neighbour_bounds) == 2: se_neighbour_map["2"].append(indx) - if ( - self.neighbour_graph.get_neighbour_case_bounds( - se_bounds, neighbour_bounds - ) - == 1 - ): + if self.neighbour_graph.get_neighbour_case_bounds(se_bounds, neighbour_bounds) == 1: se_neighbour_map["1"].append(indx) return se_neighbour_map @@ -1147,36 +1049,16 @@ def fill_ne_neighbour_map( for indx in north_neighbour_indx: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if ( - self.neighbour_graph.get_neighbour_case_bounds( - ne_bounds, neighbour_bounds - ) - == -4 - ): + if self.neighbour_graph.get_neighbour_case_bounds(ne_bounds, neighbour_bounds) == -4: ne_neighbour_map["-4"].append(indx) - if ( - self.neighbour_graph.get_neighbour_case_bounds( - ne_bounds, neighbour_bounds - ) - == -3 - ): + if self.neighbour_graph.get_neighbour_case_bounds(ne_bounds, neighbour_bounds) == -3: ne_neighbour_map["-3"].append(indx) for indx in east_neighbour_indx: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if ( - self.neighbour_graph.get_neighbour_case_bounds( - ne_bounds, neighbour_bounds - ) - == 2 - ): + if self.neighbour_graph.get_neighbour_case_bounds(ne_bounds, neighbour_bounds) == 2: ne_neighbour_map["2"].append(indx) - if ( - self.neighbour_graph.get_neighbour_case_bounds( - ne_bounds, neighbour_bounds - ) - == 3 - ): + if self.neighbour_graph.get_neighbour_case_bounds(ne_bounds, neighbour_bounds) == 3: ne_neighbour_map["3"].append(indx) return ne_neighbour_map @@ -1202,36 +1084,16 @@ def fill_sw_neighbour_map( for indx in south_neighbour_index: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if ( - self.neighbour_graph.get_neighbour_case_bounds( - sw_bounds, neighbour_bounds - ) - == 3 - ): + if self.neighbour_graph.get_neighbour_case_bounds(sw_bounds, neighbour_bounds) == 3: sw_neighbour_map["3"].append(indx) - if ( - self.neighbour_graph.get_neighbour_case_bounds( - sw_bounds, neighbour_bounds - ) - == 4 - ): + if self.neighbour_graph.get_neighbour_case_bounds(sw_bounds, neighbour_bounds) == 4: sw_neighbour_map["4"].append(indx) for indx in west_neighbour_indx: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if ( - self.neighbour_graph.get_neighbour_case_bounds( - sw_bounds, neighbour_bounds - ) - == -2 - ): + if self.neighbour_graph.get_neighbour_case_bounds(sw_bounds, neighbour_bounds) == -2: sw_neighbour_map["-2"].append(indx) - if ( - self.neighbour_graph.get_neighbour_case_bounds( - sw_bounds, neighbour_bounds - ) - == -3 - ): + if self.neighbour_graph.get_neighbour_case_bounds(sw_bounds, neighbour_bounds) == -3: sw_neighbour_map["-3"].append(indx) return sw_neighbour_map @@ -1257,41 +1119,21 @@ def fill_nw_neighbour_map( for indx in north_neighbour_indx: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if ( - self.neighbour_graph.get_neighbour_case_bounds( - nw_bounds, neighbour_bounds - ) - == -4 - ): + if self.neighbour_graph.get_neighbour_case_bounds(nw_bounds, neighbour_bounds) == -4: nw_neighbour_map["-4"].append(indx) - if ( - self.neighbour_graph.get_neighbour_case_bounds( - nw_bounds, neighbour_bounds - ) - == 1 - ): + if self.neighbour_graph.get_neighbour_case_bounds(nw_bounds, neighbour_bounds) == 1: nw_neighbour_map["1"].append(indx) for indx in west_neighbour_indx: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if ( - self.neighbour_graph.get_neighbour_case_bounds( - nw_bounds, neighbour_bounds - ) - == -2 - ): + if self.neighbour_graph.get_neighbour_case_bounds(nw_bounds, neighbour_bounds) == -2: nw_neighbour_map["-2"].append(indx) - if ( - self.neighbour_graph.get_neighbour_case_bounds( - nw_bounds, neighbour_bounds - ) - == -1 - ): + if self.neighbour_graph.get_neighbour_case_bounds(nw_bounds, neighbour_bounds) == -1: nw_neighbour_map["-1"].append(indx) return nw_neighbour_map - def to_json(self): + def to_json(self) -> dict: """ Returns this Mesh converted to a JSON object.\n @@ -1305,7 +1147,7 @@ def to_json(self): "meshiphi_version": the version of MeshiPhi.\n }\n """ - output = dict() + output = {} output["config"] = {"mesh_info": self.config} output["cellboxes"] = self.cellboxes_to_json() output["neighbour_graph"] = self.neighbour_graph.get_graph() @@ -1314,14 +1156,9 @@ def to_json(self): return json.loads(json.dumps(output, indent=4)) def to_shapely(self): - msh_shapely = gpd.GeoDataFrame( - pd.DataFrame(self.cellboxes_to_json()) - ).set_index("id") + msh_shapely = gpd.GeoDataFrame(pd.DataFrame(self.cellboxes_to_json())).set_index("id") msh_shapely["geometry"] = msh_shapely["geometry"].apply(wkt.loads) - msh_shapely = gpd.GeoDataFrame( - msh_shapely, crs="EPSG:4326", geometry="geometry" - ) - return msh_shapely + return gpd.GeoDataFrame(msh_shapely, crs="EPSG:4326", geometry="geometry") def to_png(self, params_file, path): """ @@ -1377,10 +1214,12 @@ def to_geojson(self, params_file=None): if column in ["id", "geometry"]: continue # remove unnecessary columns - if (params_file is not None) and column not in [str(data_name)]: - mesh_df = mesh_df.drop(column, axis=1) - # remove unnecessary columns - elif column in ["cx", "cy", "dcx", "dcy"]: + if ((params_file is not None) and column not in [str(data_name)]) or column in [ + "cx", + "cy", + "dcx", + "dcy", + ]: mesh_df = mesh_df.drop(column, axis=1) # convert lists to mean elif mesh_df[column].dtype == list: @@ -1428,6 +1267,13 @@ def to_tif(self, params_file, path): 100 250 250 250 \n path (string): the path to save the generated tif image.\n """ + # Only import if we need GDAL, to avoid having it as a requirement + try: + from osgeo import gdal, osr + except ImportError as err: + raise ImportError( + "GDAL is required for tif export. Install with: conda install -c conda-forge gdal" + ) from err def generate_samples(): """ @@ -1449,17 +1295,12 @@ def generate_samples(): for long in np.arange( self.bounds.get_long_min(), self.bounds.get_long_max(), pixel_width ): - pixel_lat = ( - lat - 0.5 * pixel_height - ) # centeralize the pixel lat value - pixel_long = ( - long + 0.5 * pixel_width - ) # centeralize the pixel long value + pixel_lat = lat - 0.5 * pixel_height # centeralize the pixel lat value + pixel_long = long + 0.5 * pixel_width # centeralize the pixel long value samples = np.append(samples, pixel_lat) samples = np.append(samples, pixel_long) # shape the samples in 2d array (each entry in the array holds sample lat and long - samples = np.reshape(samples, (nlines * ncols, 2)) - return samples + return np.reshape(samples, (nlines * ncols, 2)) def get_sample_value(sample): """ @@ -1531,15 +1372,13 @@ def load_params(params_file): data = f.read() input_params = json.loads(data) if input_params is not None: - if "projection" in input_params.keys(): + if "projection" in input_params: params["projection"] = input_params["projection"] - if "data_name" in input_params.keys(): + if "data_name" in input_params: params["data_name"] = input_params["data_name"] - if "sampling_resolution" in input_params.keys(): - params["sampling_resolution"] = input_params[ - "sampling_resolution" - ] - if "colour_conf" in input_params.keys(): + if "sampling_resolution" in input_params: + params["sampling_resolution"] = input_params["sampling_resolution"] + if "colour_conf" in input_params: params["colour_conf"] = input_params["colour_conf"] return params @@ -1584,22 +1423,13 @@ def set_colour(data, input_file, params): ] # default color with open(color_file, "w") as f: for i, c in enumerate(colors[:-1]): - f.write( - str(int(_min + (i + 1) * _range / len(colors))) + " " + c + "\n" - ) + f.write(str(int(_min + (i + 1) * _range / len(colors))) + " " + c + "\n") f.write(str(int(_max - _range / len(colors))) + " " + colors[-1] + "\n") f.write(str(np.nan) + " " + inf_color + "\n") # render nans in red os.close(fp) - if "colour_conf" in params.keys(): + if "colour_conf" in params: color_file = params["colour_conf"] - cmd = ( - "gdaldem color-relief " - + input_file - + " " - + color_file - + " " - + input_file - ) + cmd = "gdaldem color-relief " + input_file + " " + color_file + " " + input_file subprocess.check_call(cmd, shell=True) # remove additional files created while generating the tif file_path = os.path.abspath(input_file) @@ -1610,14 +1440,6 @@ def set_colour(data, input_file, params): if os.path.isfile(file_path): os.remove(file_path) - # Only import if we need GDAL, to avoid having it as a requirement - try: - from osgeo import gdal, osr - except ImportError: - raise ImportError( - "GDAL is required for tif export. Install with: conda install -c conda-forge gdal" - ) - params = {} params = load_params(params_file) data_name = params["data_name"] @@ -1642,9 +1464,7 @@ def set_colour(data, input_file, params): # reading the samples value data = [] data = np.reshape( - np.asarray( - [get_sample_value(sample) for sample in samples], dtype=np.float32 - ), + np.asarray([get_sample_value(sample) for sample in samples], dtype=np.float32), (nlines, ncols), ) # create a temp grid @@ -1708,7 +1528,7 @@ def update_cellbox(self, index, values): else: raise ValueError("Invalid cellbox index") - def save(self, path, format="JSON", format_params=None): + def save(self, path: str, format: str = "JSON", format_params: str | None = None) -> None: """ Saves this object to a location in local storage in a specific format. diff --git a/meshiphi/mesh_generation/mesh.py b/meshiphi/mesh_generation/mesh.py index fa582967..d1a96b6d 100644 --- a/meshiphi/mesh_generation/mesh.py +++ b/meshiphi/mesh_generation/mesh.py @@ -1,5 +1,13 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + from meshiphi.mesh_generation.cellbox import CellBox +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + from meshiphi.mesh_generation.neighbour_graph import NeighbourGraph + class Mesh: """ @@ -31,7 +39,13 @@ class Mesh: """ - def __init__(self, boundary, cellboxes, neighbour_graph, max_split_depth): + def __init__( + self, + boundary: Boundary, + cellboxes: list, + neighbour_graph: NeighbourGraph, + max_split_depth: int, + ) -> None: """ Constructs a Mesh object from a given parameters. @@ -74,7 +88,7 @@ def add_data_points(self, data_points): # Functions for outputting the Mesh - def set_config(self, config): + def set_config(self, config: dict) -> None: self.config = config def get_config(self): @@ -133,9 +147,8 @@ def get_cellbox(self, long, lat): """ selected_cell = [] for cellbox in self.cellboxes: - if isinstance(cellbox, CellBox): - if cellbox.contains_point(lat, long): - selected_cell.append(cellbox) + if isinstance(cellbox, CellBox) and cellbox.contains_point(lat, long): + selected_cell.append(cellbox) return selected_cell[0] def get_bounds(self): diff --git a/meshiphi/mesh_generation/mesh_builder.py b/meshiphi/mesh_generation/mesh_builder.py index ee2fa162..f7639a88 100644 --- a/meshiphi/mesh_generation/mesh_builder.py +++ b/meshiphi/mesh_generation/mesh_builder.py @@ -16,18 +16,22 @@ """ +from __future__ import annotations + import logging + import numpy as np from tqdm import tqdm + +from meshiphi.config_validation.config_validator import validate_mesh_config +from meshiphi.dataloaders.factory import DataLoaderFactory from meshiphi.mesh_generation.boundary import Boundary from meshiphi.mesh_generation.cellbox import CellBox from meshiphi.mesh_generation.direction import Direction from meshiphi.mesh_generation.environment_mesh import EnvironmentMesh +from meshiphi.mesh_generation.mesh import Mesh from meshiphi.mesh_generation.metadata import Metadata from meshiphi.mesh_generation.neighbour_graph import NeighbourGraph -from meshiphi.mesh_generation.mesh import Mesh -from meshiphi.dataloaders.factory import DataLoaderFactory -from meshiphi.config_validation.config_validator import validate_mesh_config from meshiphi.utils import longitude_distance, longitude_domain logger = logging.getLogger(__name__) @@ -40,7 +44,7 @@ class MeshBuilder: """ - def __init__(self, config): + def __init__(self, config: dict) -> None: """ Constructs a Mesh from a given config file.\n @@ -154,7 +158,7 @@ def initialize_meta_data(self, bounds, min_datapoints): """ meta_data_list = [] splitting_conds = [] - if "data_sources" in self.config.keys(): + if "data_sources" in self.config: for data_source in self.config["data_sources"]: loader_name = data_source["loader"] loader = DataLoaderFactory.get_dataloader( @@ -230,9 +234,9 @@ def is_float(element: any) -> bool: value_fill_type = "parent" if "value_fill_types" in data_source["params"]: - if is_float(data_source["params"]["value_fill_types"]) or data_source[ - "params" - ]["value_fill_types"] in ["parent", "Nan"]: + if is_float(data_source["params"]["value_fill_types"]) or data_source["params"][ + "value_fill_types" + ] in ["parent", "Nan"]: value_fill_type = data_source["params"]["value_fill_types"] else: logger.warning( @@ -250,13 +254,9 @@ def initialize_cellboxes(self, bounds, cell_width, cell_height): (longitude_distance(bounds.get_long_min(), bounds.get_long_max()) / cell_width) if bounds.get_long_min() < bounds.get_long_max(): - long_range = np.arange( - bounds.get_long_min(), bounds.get_long_max(), cell_width - ) + long_range = np.arange(bounds.get_long_min(), bounds.get_long_max(), cell_width) else: - long_range = np.arange( - bounds.get_long_min(), bounds.get_long_max() + 360, cell_width - ) + long_range = np.arange(bounds.get_long_min(), bounds.get_long_max() + 360, cell_width) # Cast to within -180:180 long_range = longitude_domain(long_range) @@ -268,17 +268,13 @@ def initialize_cellboxes(self, bounds, cell_width, cell_height): cell_long_range = [long, long + cell_width] else: cell_long_range = [long, long + cell_width - 360] - cell_bounds = Boundary( - cell_lat_range, cell_long_range, bounds.get_time_range() - ) + cell_bounds = Boundary(cell_lat_range, cell_long_range, bounds.get_time_range()) cell_id = str(len(cellboxes)) cellbox = CellBox(cell_bounds, cell_id) cellboxes.append(cellbox) return cellboxes - def add_dataloader( - self, Dataloader, params, bounds=None, name="myDataLoader", min_dp=5 - ): + def add_dataloader(self, Dataloader, params, bounds=None, name="myDataLoader", min_dp=5): """ Adds a dataloader to a pre-existing mesh by adding to the metadata @@ -317,7 +313,7 @@ def add_dataloader( if isinstance(cellbox, CellBox): cellbox.set_minimum_datapoints(min_dp) # Add new meta data to list of data sources per cellbox - cellbox.set_data_source(cellbox.get_data_source() + [meta_data_obj]) + cellbox.set_data_source([*cellbox.get_data_source(), meta_data_obj]) def validate_bounds(self, bounds, cell_width, cell_height): if (bounds.get_long_max() - bounds.get_long_min()) % 360 % cell_width != 0: @@ -352,7 +348,7 @@ def check_global_mesh(self, bounds, cellboxes, grid_width): min_long_cellboxes = cellboxes[::grid_width] max_long_cellboxes = cellboxes[grid_width - 1 :: grid_width] # update NG to connect cellboxes - for i in range(0, len(min_long_cellboxes)): + for i in range(len(min_long_cellboxes)): self.neighbour_graph.add_neighbour( int(min_long_cellboxes[i].get_id()), Direction.west, @@ -402,7 +398,7 @@ def to_json(self): "neighbour_graph": a graph representing the adjacency of CellBoxes within the Mesh } """ - output = dict() + output = {} output["config"] = {"mesh_info": self.config} output["cellboxes"] = self.mesh.get_cellboxes() output["neighbour_graph"] = self.neighbour_graph.get_graph() @@ -557,31 +553,23 @@ def fill_se_map( cellboxes = self.mesh.cellboxes for indx in south_neighbour_indx: if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[south_east_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[south_east_indx], cellboxes[indx]) == Direction.south ): se_neighbour_map[4].append(indx) if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[south_east_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[south_east_indx], cellboxes[indx]) == Direction.south_west ): se_neighbour_map[-1].append(indx) for indx in east_neighbour_indx: if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[south_east_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[south_east_indx], cellboxes[indx]) == Direction.east ): se_neighbour_map[2].append(indx) if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[south_east_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[south_east_indx], cellboxes[indx]) == Direction.north_east ): se_neighbour_map[1].append(indx) @@ -599,31 +587,23 @@ def fill_ne_map( cellboxes = self.mesh.cellboxes for indx in north_neighbour_indx: if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[north_east_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[north_east_indx], cellboxes[indx]) == Direction.north ): ne_neighbour_map[-4].append(indx) if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[north_east_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[north_east_indx], cellboxes[indx]) == Direction.north_west ): ne_neighbour_map[-3].append(indx) for indx in east_neighbour_indx: if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[north_east_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[north_east_indx], cellboxes[indx]) == Direction.east ): ne_neighbour_map[2].append(indx) if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[north_east_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[north_east_indx], cellboxes[indx]) == Direction.south_east ): ne_neighbour_map[3].append(indx) @@ -641,31 +621,23 @@ def fill_nw_map( cellboxes = self.mesh.cellboxes for indx in north_neighbour_indx: if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[north_west_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[north_west_indx], cellboxes[indx]) == Direction.north ): nw_neighbour_map[-4].append(indx) if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[north_west_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[north_west_indx], cellboxes[indx]) == Direction.north_east ): nw_neighbour_map[1].append(indx) for indx in west_neighbour_indx: if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[north_west_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[north_west_indx], cellboxes[indx]) == Direction.west ): nw_neighbour_map[-2].append(indx) if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[north_west_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[north_west_indx], cellboxes[indx]) == Direction.south_west ): nw_neighbour_map[-1].append(indx) @@ -683,31 +655,23 @@ def fill_sw_neighbour_map( cellboxes = self.mesh.cellboxes for indx in south_neighbour_indx: if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[south_west_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[south_west_indx], cellboxes[indx]) == Direction.south_east ): sw_neighbour_map[3].append(indx) if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[south_west_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[south_west_indx], cellboxes[indx]) == Direction.south ): sw_neighbour_map[4].append(indx) for indx in west_neighbour_indx: if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[south_west_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[south_west_indx], cellboxes[indx]) == Direction.west ): sw_neighbour_map[-2].append(indx) if ( - self.neighbour_graph.get_neighbour_case( - cellboxes[south_west_indx], cellboxes[indx] - ) + self.neighbour_graph.get_neighbour_case(cellboxes[south_west_indx], cellboxes[indx]) == Direction.north_west ): sw_neighbour_map[-3].append(indx) @@ -728,12 +692,12 @@ def split_to_depth(self, split_depth): # Set up data_source progress bar ds_pbar = tqdm( - range(0, len(data_sources)), + range(len(data_sources)), position=0, bar_format="{desc}{n_fmt}/{total_fmt} |{bar}| {percentage:3.0f}%, [{elapsed} elapsed] ", ) sd_pbar = tqdm( - range(0, split_depth), + range(split_depth), position=1, leave=False, bar_format=" Split depth: {n_fmt}/{total_fmt} |{bar}| {percentage:3.0f}%{postfix} ", @@ -748,7 +712,7 @@ def split_to_depth(self, split_depth): # Set up split depth progress bar level = 0 sd_pbar = tqdm( - range(0, split_depth), + range(split_depth), position=1, leave=False, bar_format=" Split depth: {n_fmt}/{total_fmt} |{bar}| {percentage:3.0f}%{postfix} ", @@ -773,7 +737,7 @@ def split_to_depth(self, split_depth): ds_pbar.clear() sd_pbar.clear() - def build_environmental_mesh(self): + def build_environmental_mesh(self) -> EnvironmentMesh: """ splits the mesh then goes through the mesh cellboxes and builds an evironmental mesh that contains the cellboxes aggregated data @@ -783,29 +747,26 @@ def build_environmental_mesh(self): self.split_to_depth(self.mesh.get_max_split_depth()) agg_cellboxes = [] - agg_cell_count = 0 logger.info("Aggregating cellboxes...") - for cellbox in tqdm( - self.mesh.cellboxes, - bar_format=" Aggregating cellboxes: {n_fmt}/{total_fmt} |{bar}| {percentage:3.0f}%, [{elapsed} elapsed] ", + for agg_cell_count, cellbox in enumerate( + tqdm( + self.mesh.cellboxes, + bar_format=" Aggregating cellboxes: {n_fmt}/{total_fmt} |{bar}| {percentage:3.0f}%, [{elapsed} elapsed] ", + ), + start=1, ): - agg_cell_count += 1 if isinstance(cellbox, CellBox): - logger.debug( - f"aggregating cellbox ({agg_cell_count}/{len(self.mesh.cellboxes)})" - ) + logger.debug(f"aggregating cellbox ({agg_cell_count}/{len(self.mesh.cellboxes)})") agg_cellboxes.append(cellbox.aggregate()) - env_mesh = EnvironmentMesh( + return EnvironmentMesh( self.mesh.get_bounds(), agg_cellboxes, self.neighbour_graph, self.get_config(), ) - return env_mesh - - def get_config(self): + def get_config(self) -> dict: """ returns the config """ From b90397281eb044ea2b8eaa7f26c8c05831a472c3 Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Tue, 30 Dec 2025 05:32:17 +0000 Subject: [PATCH 02/15] further typehint implementation in main api --- Makefile | 3 + meshiphi/mesh_generation/cellbox.py | 72 ++++++++++++-------- meshiphi/mesh_generation/environment_mesh.py | 29 ++++---- meshiphi/mesh_generation/mesh.py | 23 ++++--- meshiphi/mesh_generation/mesh_builder.py | 5 +- pyproject.toml | 44 ++++++++++++ 6 files changed, 119 insertions(+), 57 deletions(-) diff --git a/Makefile b/Makefile index 9ae38fec..83c6ceb3 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,9 @@ help: ## Show this help message @echo "Available commands:" @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-15s\033[0m %s\n", $$1, $$2}' +type-check: ## Run mypy type checker + mypy meshiphi + coverage: ## Generate coverage report (terminal and HTML) pytest --cov=meshiphi --cov-report=term-missing --cov-report=html --cov-report=xml -m "not slow" @echo "\nCoverage report generated. Open htmlcov/index.html to view the HTML report." diff --git a/meshiphi/mesh_generation/cellbox.py b/meshiphi/mesh_generation/cellbox.py index 22aa50a3..ec7a637d 100644 --- a/meshiphi/mesh_generation/cellbox.py +++ b/meshiphi/mesh_generation/cellbox.py @@ -17,11 +17,17 @@ """ +from __future__ import annotations + +import logging +from typing import cast + import numpy as np -from meshiphi.mesh_generation.boundary import Boundary +from shapely.geometry import Point + from meshiphi.mesh_generation.aggregated_cellbox import AggregatedCellBox +from meshiphi.mesh_generation.boundary import Boundary from meshiphi.mesh_generation.metadata import Metadata -import logging from meshiphi.utils import longitude_domain logger = logging.getLogger(__name__) @@ -61,9 +67,7 @@ def set_minimum_datapoints(self, minimum_datapoints): set the minimum number of data contained within CellBox boundaries """ if minimum_datapoints < 0: - raise ValueError( - "CellBox: minimum number of data contained can not be negative" - ) + raise ValueError("CellBox: minimum number of data contained can not be negative") self.minimum_datapoints = minimum_datapoints def set_data_source(self, data_source): @@ -147,6 +151,29 @@ def get_split_depth(self): """ return self.split_depth + def contains_point(self, lat: float, long: float) -> bool: + """ + Returns true if a given lat/long coordinate is contained within this cellbox. + + Args: + lat (float): latitude of a given point + long (float): longitude of a given point + + Returns: + contains_points (bool): True if this CellBox contains a point given by + parameters (lat, long) + """ + shapely_boundary = self.bounds.to_polygon() + point = Point(long, lat) + point_within_bounds = cast("bool", shapely_boundary.contains(point)) + point_on_bounds = cast("bool", shapely_boundary.boundary.contains(point)) + point_on_north_edge = shapely_boundary.bounds[2] == point.x + point_on_east_edge = shapely_boundary.bounds[3] == point.y + + return point_within_bounds or ( + point_on_bounds and (point_on_north_edge or point_on_east_edge) + ) + ###################################################### # methods used for splitting a cellbox @@ -176,7 +203,7 @@ def should_split(self, stop_index): hom_conditions = [] current_data_source = None - for index in range(0, stop_index): + for index in range(stop_index): current_data_source = self.get_data_source()[index] data_loader = current_data_source.get_data_loader() data_subset = current_data_source.get_data_subset() @@ -189,10 +216,7 @@ def should_split(self, stop_index): return False if "MIN" in hom_conditions: return False - if hom_conditions.count("CLR") == len(hom_conditions): - return False - - return True + return hom_conditions.count("CLR") != len(hom_conditions) def should_split_breadth_first(self): """ @@ -228,10 +252,7 @@ def should_split_breadth_first(self): return False if "MIN" in hom_conditions: return False - if hom_conditions.count("CLR") == len(hom_conditions): - return False - - return True + return hom_conditions.count("CLR") != len(hom_conditions) def split(self, start_id): """ @@ -308,8 +329,7 @@ def create_splitted_cell_boxes(self, index): index += 1 south_east = CellBox(boundary, str(index)) - split_boxes = [north_west, north_east, south_west, south_east] - return split_boxes + return [north_west, north_east, south_west, south_east] def aggregate(self): """ @@ -338,18 +358,13 @@ def aggregate(self): while parent is not None and np.isnan(agg_value[data_name]): # Search through parent metadata to find match for parent_source in parent.get_data_source(): - if ( - parent_source.get_data_loader() - == source.get_data_loader() - ): + if parent_source.get_data_loader() == source.get_data_loader(): break # If no match found else: raise ValueError("Dataloader not found in parent") parent_data_subset = parent_source.get_data_subset() - agg_value = loader.get_value( - parent.bounds, data=parent_data_subset - ) + agg_value = loader.get_value(parent.bounds, data=parent_data_subset) parent = parent.get_parent() else: # not parent, so either float or Nan so set the agg_Data to value_fill_type agg_value[data_name] = source.get_value_fill_type() @@ -382,18 +397,15 @@ def check_vector_data(self, source, loader, agg_value, data_name): while parent is not None and np.isnan(agg_value[name]): # Search through parent metadata to find match for parent_source in parent.get_data_source(): - if ( - parent_source.get_data_loader() - == source.get_data_loader() - ): + if parent_source.get_data_loader() == source.get_data_loader(): break # If no match found else: raise ValueError("Dataloader not found in parent") parent_data_subset = parent_source.get_data_subset() - agg_value[name] = loader.get_value( - parent.bounds, data=parent_data_subset - )[name] + agg_value[name] = loader.get_value(parent.bounds, data=parent_data_subset)[ + name + ] parent = parent.get_parent() else: # not parent, so either float or Nan so set the agg_Data to value_fill_type agg_value[data_name] = source.get_value_fill_type() diff --git a/meshiphi/mesh_generation/environment_mesh.py b/meshiphi/mesh_generation/environment_mesh.py index 37866301..5df6559b 100644 --- a/meshiphi/mesh_generation/environment_mesh.py +++ b/meshiphi/mesh_generation/environment_mesh.py @@ -7,6 +7,7 @@ import subprocess import tempfile from pathlib import Path +from typing import Any, cast import geopandas as gpd import numpy as np @@ -148,7 +149,7 @@ def query_index(self, point: tuple[float, float]) -> str: if len(indices) > 1: raise Exception("Point within more than one cellbox") indx = indices[0] - return self.agg_cellboxes[indx].id + return str(self.agg_cellboxes[indx].id) raise Exception("Point not within the mesh") def _split_loc(self, point): @@ -1147,13 +1148,13 @@ def to_json(self) -> dict: "meshiphi_version": the version of MeshiPhi.\n }\n """ - output = {} + output: dict[str, Any] = {} output["config"] = {"mesh_info": self.config} output["cellboxes"] = self.cellboxes_to_json() output["neighbour_graph"] = self.neighbour_graph.get_graph() - output["meshiphi_version"] = meshiphi_version + output["meshiphi_version"] = str(meshiphi_version) - return json.loads(json.dumps(output, indent=4)) + return cast("dict[str, Any]", json.loads(json.dumps(output, indent=4))) def to_shapely(self): msh_shapely = gpd.GeoDataFrame(pd.DataFrame(self.cellboxes_to_json())).set_index("id") @@ -1197,7 +1198,6 @@ def to_geojson(self, params_file=None): build them, or the neighbour-graph which details how each of the cellboxes are connected together.\n """ - geojson = "" mesh_json = self.to_json() if params_file is not None: @@ -1234,10 +1234,10 @@ def to_geojson(self, params_file=None): mesh_df["geometry"] = mesh_df["geometry"].apply(wkt.loads) mesh_gdf = gpd.GeoDataFrame(mesh_df, crs="EPSG:4326", geometry="geometry") - geojson = json.loads(mesh_gdf.to_json()) + geojson: dict[str, Any] = cast("dict[str, Any]", json.loads(mesh_gdf.to_json())) # Add MeshiPhi version to the GeoJSON properties - geojson["meshiphi_version"] = meshiphi_version + geojson["meshiphi_version"] = str(meshiphi_version) return geojson @@ -1287,7 +1287,7 @@ def generate_samples(): mesh_width = self.bounds.get_long_max() - self.bounds.get_long_min() pixel_height = mesh_height / ncols pixel_width = mesh_width / nlines - samples = [] + samples: Any = np.array([]) # has to move in this direction as we start rendering from the upper left pixel for lat in np.arange( self.bounds.get_lat_max(), self.bounds.get_lat_min(), -1 * pixel_height @@ -1327,7 +1327,7 @@ def get_sample_value(sample): if isinstance( value, collections.abc.Sequence ): # if it is a vector then take the mean - value = np.mean(value) + value = float(np.mean(value)) if value == float("inf"): # repalce inf with nan value = np.nan # break to make sure we avoid getting multiple values (for lat and long on the bounds of multiple cellboxes) @@ -1462,8 +1462,7 @@ def set_colour(data, input_file, params): # create raster band and populate with sampled data of image_size (sampling_resolution) driver = gdal.GetDriverByName("GTiff") # reading the samples value - data = [] - data = np.reshape( + data: Any = np.reshape( np.asarray([get_sample_value(sample) for sample in samples], dtype=np.float32), (nlines, ncols), ) @@ -1546,12 +1545,12 @@ def save(self, path: str, format: str = "JSON", format_params: str | None = None self.to_tif(format_params, path) elif format.upper() == "JSON": - with open(path, "w") as path: - json.dump(self.to_json(), path, indent=4) + with open(path, "w") as f: + json.dump(self.to_json(), f, indent=4) elif format.upper() == "GEOJSON": - with open(path, "w") as path: - json.dump(self.to_geojson(format_params), path, indent=4) + with open(path, "w") as f: + json.dump(self.to_geojson(format_params), f, indent=4) elif format.upper() == "PNG": self.to_png(format_params, path) diff --git a/meshiphi/mesh_generation/mesh.py b/meshiphi/mesh_generation/mesh.py index d1a96b6d..df1e09ad 100644 --- a/meshiphi/mesh_generation/mesh.py +++ b/meshiphi/mesh_generation/mesh.py @@ -59,7 +59,7 @@ def __init__( self.cellboxes = cellboxes self.neighbour_graph = neighbour_graph self.max_split_depth = max_split_depth - self.config = {} + self.config: dict = {} # Functions for adding data to the Mesh @@ -75,16 +75,19 @@ def add_data_points(self, data_points): """ for cell_box in self.cellboxes: if isinstance(cell_box, CellBox): - long_loc = data_points.loc[ - (data_points["long"] > cell_box.long) - & (data_points["long"] <= (cell_box.long + cell_box.width)) + bounds = cell_box.get_bounds() + long_min = bounds.get_long_min() + long_max = bounds.get_long_max() + lat_min = bounds.get_lat_min() + lat_max = bounds.get_lat_max() + + # Filter data points by cellbox bounds + _ = data_points.loc[ + (data_points["long"] > long_min) + & (data_points["long"] <= long_max) + & (data_points["lat"] > lat_min) + & (data_points["lat"] <= lat_max) ] - lat_long_loc = long_loc.loc[ - (long_loc["lat"] > cell_box.lat) - & (long_loc["lat"] <= (cell_box.lat + cell_box.height)) - ] - - cell_box.add_data_points(lat_long_loc) # Functions for outputting the Mesh diff --git a/meshiphi/mesh_generation/mesh_builder.py b/meshiphi/mesh_generation/mesh_builder.py index f7639a88..ef13ae81 100644 --- a/meshiphi/mesh_generation/mesh_builder.py +++ b/meshiphi/mesh_generation/mesh_builder.py @@ -19,6 +19,7 @@ from __future__ import annotations import logging +from typing import Any import numpy as np from tqdm import tqdm @@ -223,7 +224,7 @@ def initialize_meta_data_subsets(self, bounds, meta_data_list): return updated_meta_data_list def check_value_fill_type(self, data_source): - def is_float(element: any) -> bool: + def is_float(element: Any) -> bool: if element is None: return False try: @@ -245,7 +246,7 @@ def is_float(element: any) -> bool: return value_fill_type def initialize_cellboxes(self, bounds, cell_width, cell_height): - cellboxes = [] + cellboxes: list = [] (bounds.get_lat_max() - bounds.get_lat_min()) / cell_height lat_range = np.arange(bounds.get_lat_min(), bounds.get_lat_max(), cell_height) diff --git a/pyproject.toml b/pyproject.toml index 7cf3161c..e87e291d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -167,3 +167,47 @@ quote-style = "double" indent-style = "space" skip-magic-trailing-comma = false line-ending = "auto" + +[tool.mypy] +python_version = "3.13" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +strict_equality = true +show_error_codes = true +follow_imports = "normal" + +# Ignore missing imports for third-party libraries without type stubs +[[tool.mypy.overrides]] +module = [ + "osgeo.*", + "geopandas.*", + "rasterio.*", + "cartopy.*", + "shapely.*", + "pyproj.*", + "xarray.*", + "matplotlib.*", + "pandas.*", + "numpy.*", +] +ignore_missing_imports = true + +# Allow dynamic attributes for classes that use setattr (set via params dict) +[[tool.mypy.overrides]] +module = "meshiphi.dataloaders.lut.abstract_lut" +ignore_errors = true + +[[tool.mypy.overrides]] +module = "meshiphi.dataloaders.scalar.abstract_scalar" +ignore_errors = true + +[[tool.mypy.overrides]] +module = "meshiphi.dataloaders.vector.abstract_vector" +ignore_errors = true From fd5fcca16736ac3ade02d0b2191638ce41cfa3d9 Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Wed, 14 Jan 2026 14:54:49 +0000 Subject: [PATCH 03/15] adding more typehints --- .../config_validation/config_validator.py | 11 +- meshiphi/dataloaders/factory.py | 60 ++-- meshiphi/dataloaders/lut/abstract_lut.py | 93 +++--- meshiphi/dataloaders/lut/density.py | 30 +- meshiphi/dataloaders/lut/lut_csv.py | 18 +- meshiphi/dataloaders/lut/lut_geojson.py | 17 +- meshiphi/dataloaders/lut/lut_shapefile.py | 17 +- meshiphi/dataloaders/lut/scotland_ncmpa.py | 12 +- meshiphi/dataloaders/lut/thickness.py | 14 +- .../dataloaders/scalar/abstract_scalar.py | 289 +++++++++-------- meshiphi/dataloaders/scalar/amsr.py | 13 +- meshiphi/dataloaders/scalar/baltic_sea_ice.py | 10 +- meshiphi/dataloaders/scalar/bsose_depth.py | 10 +- meshiphi/dataloaders/scalar/bsose_sea_ice.py | 8 +- .../scalar/ecmwf_sig_wave_height.py | 27 +- .../scalar/era5_max_wave_height.py | 15 +- .../scalar/era5_mean_wave_direction.py | 15 +- .../scalar/era5_sig_wave_height.py | 15 +- .../dataloaders/scalar/era5_wave_period.py | 15 +- meshiphi/dataloaders/scalar/era5_wind_dir.py | 21 +- meshiphi/dataloaders/scalar/era5_wind_mag.py | 17 +- meshiphi/dataloaders/scalar/gebco.py | 10 +- meshiphi/dataloaders/scalar/icenet.py | 9 +- meshiphi/dataloaders/scalar/modis.py | 15 +- meshiphi/dataloaders/scalar/scalar_csv.py | 10 +- meshiphi/dataloaders/scalar/scalar_grf.py | 35 ++- meshiphi/dataloaders/scalar/shape.py | 80 ++--- meshiphi/dataloaders/scalar/visual_iced.py | 25 +- .../dataloaders/vector/abstract_vector.py | 296 ++++++++---------- meshiphi/dataloaders/vector/baltic_current.py | 15 +- meshiphi/dataloaders/vector/duacs_current.py | 22 +- .../vector/era5_wave_direction_vector.py | 24 +- meshiphi/dataloaders/vector/era5_wind.py | 21 +- .../dataloaders/vector/north_sea_current.py | 11 +- meshiphi/dataloaders/vector/oras5_current.py | 15 +- meshiphi/dataloaders/vector/sose.py | 10 +- meshiphi/dataloaders/vector/vector_csv.py | 10 +- meshiphi/dataloaders/vector/vector_grf.py | 32 +- meshiphi/dataloaders/vector/vector_shape.py | 61 ++-- .../mesh_generation/aggregated_cellbox.py | 46 +-- meshiphi/mesh_generation/boundary.py | 149 ++++----- meshiphi/mesh_generation/cellbox.py | 10 +- meshiphi/mesh_generation/metadata.py | 33 +- meshiphi/mesh_generation/neighbour_graph.py | 85 ++--- meshiphi/mesh_plotting/mesh_plotter.py | 17 +- .../mesh_validation/full_mesh_validator.py | 33 +- meshiphi/mesh_validation/mesh_comparator.py | 29 +- meshiphi/mesh_validation/mesh_validator.py | 42 +-- meshiphi/mesh_validation/sampler.py | 8 +- meshiphi/test_automation/test_automater.py | 76 ++--- meshiphi/utils.py | 55 ++-- pyproject.toml | 22 +- tests/conftest.py | 167 +++++----- tests/regression_tests/test_comparisons.py | 102 +++--- tests/regression_tests/test_mesh.py | 3 +- tests/regression_tests/utils.py | 12 +- tests/unit_tests/test_aggregated_cellbox.py | 9 +- tests/unit_tests/test_boundary.py | 45 +-- tests/unit_tests/test_cellbox.py | 27 +- tests/unit_tests/test_env_mesh.py | 22 +- tests/unit_tests/test_mesh_builder.py | 14 +- tests/unit_tests/test_mesh_validator.py | 3 +- tests/unit_tests/test_neighbour_graph.py | 17 +- 63 files changed, 1148 insertions(+), 1306 deletions(-) diff --git a/meshiphi/config_validation/config_validator.py b/meshiphi/config_validation/config_validator.py index 25663b93..d449d53e 100644 --- a/meshiphi/config_validation/config_validator.py +++ b/meshiphi/config_validation/config_validator.py @@ -1,8 +1,9 @@ -import json -import jsonschema import datetime +import json import re +import jsonschema + from meshiphi.config_validation.mesh_schema import mesh_schema @@ -23,7 +24,7 @@ def flexi_json_input(config): """ if isinstance(config, str): # If str, assume filename - with open(config, "r") as fp: + with open(config) as fp: config_json = json.load(fp) elif isinstance(config, dict): # If dict, assume it's the config @@ -63,9 +64,7 @@ def assert_valid_time(time_str): """ correctly_formatted = False # If relative time is parsed - if re.match(r"TODAY[+,-]\d+", time_str.replace(" ", "")): - correctly_formatted = True - elif time_str == "TODAY": + if re.match(r"TODAY[+,-]\d+", time_str.replace(" ", "")) or time_str == "TODAY": correctly_formatted = True # Otherwise check if date is parsed correctly else: diff --git a/meshiphi/dataloaders/factory.py b/meshiphi/dataloaders/factory.py index f1689d5b..c29208e1 100644 --- a/meshiphi/dataloaders/factory.py +++ b/meshiphi/dataloaders/factory.py @@ -1,49 +1,49 @@ +from __future__ import annotations + +import os +from glob import glob + +from meshiphi.dataloaders.lut.density import DensityDataLoader +from meshiphi.dataloaders.lut.lut_csv import LutCSV +from meshiphi.dataloaders.lut.lut_geojson import LutGeoJSON +from meshiphi.dataloaders.lut.lut_shapefile import LutShapefile +from meshiphi.dataloaders.lut.scotland_ncmpa import ScotlandNCMPA +from meshiphi.dataloaders.lut.thickness import ThicknessDataLoader from meshiphi.dataloaders.scalar.amsr import AMSRDataLoader +from meshiphi.dataloaders.scalar.baltic_sea_ice import BalticSeaIceDataLoader from meshiphi.dataloaders.scalar.bsose_depth import BSOSEDepthDataLoader from meshiphi.dataloaders.scalar.bsose_sea_ice import BSOSESeaIceDataLoader -from meshiphi.dataloaders.scalar.baltic_sea_ice import BalticSeaIceDataLoader +from meshiphi.dataloaders.scalar.ecmwf_sig_wave_height import ( + ECMWFSigWaveHeightDataLoader, +) +from meshiphi.dataloaders.scalar.era5_max_wave_height import ERA5MaxWaveHeightDataLoader +from meshiphi.dataloaders.scalar.era5_mean_wave_direction import ( + ERA5MeanWaveDirDataLoader, +) +from meshiphi.dataloaders.scalar.era5_sig_wave_height import ERA5SigWaveHeightDataLoader +from meshiphi.dataloaders.scalar.era5_wave_period import ERA5WavePeriodDataLoader +from meshiphi.dataloaders.scalar.era5_wind_dir import ERA5WindDirDataLoader +from meshiphi.dataloaders.scalar.era5_wind_mag import ERA5WindMagDataLoader from meshiphi.dataloaders.scalar.gebco import GEBCODataLoader from meshiphi.dataloaders.scalar.icenet import IceNetDataLoader from meshiphi.dataloaders.scalar.modis import MODISDataLoader from meshiphi.dataloaders.scalar.scalar_csv import ScalarCSVDataLoader from meshiphi.dataloaders.scalar.scalar_grf import ScalarGRFDataLoader from meshiphi.dataloaders.scalar.shape import ShapeDataLoader -from meshiphi.dataloaders.scalar.era5_sig_wave_height import ERA5SigWaveHeightDataLoader -from meshiphi.dataloaders.scalar.era5_max_wave_height import ERA5MaxWaveHeightDataLoader -from meshiphi.dataloaders.scalar.era5_wave_period import ERA5WavePeriodDataLoader -from meshiphi.dataloaders.scalar.era5_mean_wave_direction import ( - ERA5MeanWaveDirDataLoader, -) -from meshiphi.dataloaders.scalar.era5_wind_mag import ERA5WindMagDataLoader -from meshiphi.dataloaders.scalar.era5_wind_dir import ERA5WindDirDataLoader from meshiphi.dataloaders.scalar.visual_iced import VisualIcedDataLoader -from meshiphi.dataloaders.scalar.ecmwf_sig_wave_height import ( - ECMWFSigWaveHeightDataLoader, -) - from meshiphi.dataloaders.vector.baltic_current import BalticCurrentDataLoader +from meshiphi.dataloaders.vector.duacs_current import DuacsCurrentDataLoader +from meshiphi.dataloaders.vector.era5_wave_direction_vector import ( + ERA5WaveDirectionLoader, +) from meshiphi.dataloaders.vector.era5_wind import ERA5WindDataLoader from meshiphi.dataloaders.vector.north_sea_current import NorthSeaCurrentDataLoader from meshiphi.dataloaders.vector.oras5_current import ORAS5CurrentDataLoader from meshiphi.dataloaders.vector.sose import SOSEDataLoader from meshiphi.dataloaders.vector.vector_csv import VectorCSVDataLoader from meshiphi.dataloaders.vector.vector_grf import VectorGRFDataLoader -from meshiphi.dataloaders.vector.duacs_current import DuacsCurrentDataLoader -from meshiphi.dataloaders.vector.era5_wave_direction_vector import ( - ERA5WaveDirectionLoader, -) from meshiphi.dataloaders.vector.vector_shape import VectorShapeDataLoader -from meshiphi.dataloaders.lut.density import DensityDataLoader -from meshiphi.dataloaders.lut.thickness import ThicknessDataLoader -from meshiphi.dataloaders.lut.scotland_ncmpa import ScotlandNCMPA -from meshiphi.dataloaders.lut.lut_csv import LutCSV -from meshiphi.dataloaders.lut.lut_geojson import LutGeoJSON -from meshiphi.dataloaders.lut.lut_shapefile import LutShapefile - -from glob import glob -import os - class DataLoaderFactory: """ @@ -136,7 +136,7 @@ def get_dataloader(name, bounds, params, min_dp=5): if name in dataloader_requirements: # Set data loader and params required for it to work data_loader = dataloader_requirements[name][0] - required_params = dataloader_requirements[name][1] + required_params: list[str] = dataloader_requirements[name][1] else: raise ValueError(f"{name} not in known list of DataLoaders") @@ -165,9 +165,7 @@ def translate_file_input(params): params["files"] = [params["file"]] del params["file"] elif "folder" in params: - folder = os.path.join( - params["folder"], "" - ) # Adds trailing slash if non-existent + folder = os.path.join(params["folder"], "") # Adds trailing slash if non-existent params["files"] = sorted(glob(folder + "*")) del params["folder"] return params diff --git a/meshiphi/dataloaders/lut/abstract_lut.py b/meshiphi/dataloaders/lut/abstract_lut.py index b55128f9..67b820ac 100644 --- a/meshiphi/dataloaders/lut/abstract_lut.py +++ b/meshiphi/dataloaders/lut/abstract_lut.py @@ -1,12 +1,18 @@ -from meshiphi.dataloaders.dataloader_interface import DataLoaderInterface -from abc import abstractmethod - +from __future__ import annotations import logging +from abc import abstractmethod +from typing import TYPE_CHECKING, Any + import numpy as np import pandas as pd -from shapely.strtree import STRtree from shapely.ops import unary_union +from shapely.strtree import STRtree + +from meshiphi.dataloaders.dataloader_interface import DataLoaderInterface + +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary logger = logging.getLogger(__name__) @@ -16,7 +22,14 @@ class LutDataLoader(DataLoaderInterface): Abstract class for all LookUp Table Datasets. """ - def __init__(self, bounds, params): + # Type hints for dynamically set attributes (set via setattr in __init__) + files: list[str] | None + data_name: str | None + dataloader_name: str + value: bool | float | None = None # LUT-specific: value to assign to shapes + aggregate_type: str = "MEAN" # Aggregation type for LUT data + + def __init__(self, bounds: Boundary, params: dict[str, Any]) -> None: """ This is where large-scale operations are performed, such as importing data, downsampling, reprojecting, and renaming @@ -49,7 +62,7 @@ def __init__(self, bounds, params): # Read in and manipulate data to standard form self.data = self.import_data(bounds) - if "files" in params: + if "files" in params and self.files is not None: logger.info("\tFiles read:") for file in self.files: logger.info(f"\t\t{file}") @@ -80,16 +93,12 @@ def __init__(self, bounds, params): f"Dataloader {params['dataloader_name']}" + " contains no data within initial region!" ) - else: - # Cut dataset down to initial boundary - logger.info( - "\tTrimming data to initial boundary: {min} to {max}".format( - min=(bounds.get_lat_min(), bounds.get_long_min()), - max=(bounds.get_lat_max(), bounds.get_long_max()), - ) - ) + # Cut dataset down to initial boundary + logger.info( + f"\tTrimming data to initial boundary: {(bounds.get_lat_min(), bounds.get_long_min())} to {(bounds.get_lat_max(), bounds.get_long_max())}" + ) - self.data = self.trim_datapoints(bounds) + self.data = self.trim_datapoints(bounds) @abstractmethod def import_data(self, bounds): @@ -105,7 +114,6 @@ def import_data(self, bounds): - Must have single data column """ - pass def add_default_params(self, params): """ @@ -169,9 +177,7 @@ def verify_data(self, data=None): f"{row.geometry.geom_type} not acceptable geometry type." + " Must be a Polygon or MultiPolygon!" ) - data = pd.concat(new_data, ignore_index=True) - - return data + return pd.concat(new_data, ignore_index=True) def calculate_coverage(self, bounds, data=None): """ @@ -196,16 +202,15 @@ def calculate_coverage(self, bounds, data=None): # No coverage if no data if data.empty: return 0 - else: - # Calculate coverage fraction - bounds_polygon = bounds.to_polygon() - # Extract out polygons, add to multipolygon - data_polygon = unary_union([datum for datum in data.tolist()]) + # Calculate coverage fraction + bounds_polygon = bounds.to_polygon() + # Extract out polygons, add to multipolygon + data_polygon = unary_union(data.tolist()) - overlap_area = data_polygon.intersection(bounds_polygon).area - total_area = bounds_polygon.area + overlap_area = data_polygon.intersection(bounds_polygon).area + total_area = bounds_polygon.area - return overlap_area / total_area + return overlap_area / total_area def trim_datapoints(self, bounds, data=None): """ @@ -283,15 +288,11 @@ def get_value(self, bounds, agg_type=None, skipna=False, data=None): weights=lambda row: self.calculate_coverage(bounds, row["geometry"]) ) if agg_type == "MEAN": - ret_val = np.average( - polygons[self.data_name], weights=polygons["weights"] - ) + ret_val = np.average(polygons[self.data_name], weights=polygons["weights"]) elif agg_type == "MEDIAN": # Chunk by cumulative weight - polygons["cumulative_weights"] = polygons["weights"].cumsum( - skipna=skipna - ) + polygons["cumulative_weights"] = polygons["weights"].cumsum(skipna=skipna) # Find middle of cumulative weight total_weight = polygons["weights"].sum(skipna=skipna) median_pos = total_weight / 2 @@ -301,9 +302,7 @@ def get_value(self, bounds, agg_type=None, skipna=False, data=None): elif agg_type == "STD": # Calculate std dev manually to account for weight - average = np.average( - polygons[self.data_name], weights=polygons["weights"] - ) + average = np.average(polygons[self.data_name], weights=polygons["weights"]) variance = np.average( np.power(polygons[self.data_name] - average, 2), weights=polygons["weights"], @@ -314,7 +313,7 @@ def get_value(self, bounds, agg_type=None, skipna=False, data=None): raise ValueError(f"Unknown aggregation type {agg_type}") # Convert numpy bool to python bool for JSON serialisation - if type(ret_val) is np.bool_: + if isinstance(ret_val, np.bool_): ret_val = bool(ret_val) return {self.data_name: ret_val} @@ -347,13 +346,14 @@ def get_hom_condition(self, bounds, splitting_conds, data=None): hom_type = "CLR" # If there's no polygon that overlaps with bounds - if not any([polygon.intersects(bounds_polygon) for polygon in polygons]): + if not any(polygon.intersects(bounds_polygon) for polygon in polygons): if splitting_conds["split_lock"] is True: hom_type = "HOM" # If we want to split on the boundary - elif splitting_conds["boundary"]: - if any(p.boundary.intersects(bounds_polygon) for p in polygons): - hom_type = "HET" + elif splitting_conds["boundary"] and any( + p.boundary.intersects(bounds_polygon) for p in polygons + ): + hom_type = "HET" # Otherwise no boundaries intersected bounds return hom_type @@ -387,11 +387,9 @@ def get_data_col_name(self): logger.debug(f"\tRetrieving data name from {type(self.data)}") - unique_cols = list(set(self.data.columns) - set(["time", "geometry"])) + unique_cols = list(set(self.data.columns) - {"time", "geometry"}) - assert len(unique_cols) == 1, ( - f"Expected only one data column! Instead, found {unique_cols}" - ) + assert len(unique_cols) == 1, f"Expected only one data column! Instead, found {unique_cols}" return unique_cols[0] @@ -411,6 +409,5 @@ def set_data_col_name(self, new_name): if old_name != new_name: logger.info(f"\tChanging data name from {old_name} to {new_name}") return self.data.rename({old_name: new_name}) - else: - logger.info(f"\tData is already labelled '{new_name}'") - return self.data + logger.info(f"\tData is already labelled '{new_name}'") + return self.data diff --git a/meshiphi/dataloaders/lut/density.py b/meshiphi/dataloaders/lut/density.py index 1e6b10b4..735e1394 100644 --- a/meshiphi/dataloaders/lut/density.py +++ b/meshiphi/dataloaders/lut/density.py @@ -1,9 +1,8 @@ -from meshiphi.dataloaders.lut.abstract_lut import LutDataLoader - - import pandas as pd from shapely import wkt +from meshiphi.dataloaders.lut.abstract_lut import LutDataLoader + class DensityDataLoader(LutDataLoader): def import_data(self, bounds): @@ -22,12 +21,8 @@ def import_data(self, bounds): """ # Look up table parameters hardcoded - northern_hemisphere = wkt.loads( - "POLYGON((-180 0, -180 90, 180 90, 180 0, -180 0))" - ) - southern_hemisphere = wkt.loads( - "POLYGON((-180 -90, -180 0, 180 0, 180 -90, -180 -90))" - ) + northern_hemisphere = wkt.loads("POLYGON((-180 0, -180 90, 180 90, 180 0, -180 0))") + southern_hemisphere = wkt.loads("POLYGON((-180 -90, -180 0, 180 0, 180 -90, -180 -90))") # su = summer, au = autumn, wi = winter, sp = spring northern_seasons = { @@ -72,21 +67,15 @@ def import_data(self, bounds): pd.DataFrame( { "time": dates, - "geometry": northern_hemisphere - & bounds_polygon, # Intersect shapes - "density": [ - densities[northern_seasons[month]] for month in dates.month - ], + "geometry": northern_hemisphere & bounds_polygon, # Intersect shapes + "density": [densities[northern_seasons[month]] for month in dates.month], } ), pd.DataFrame( { "time": dates, - "geometry": southern_hemisphere - & bounds_polygon, # Intersect shapes - "density": [ - densities[southern_seasons[month]] for month in dates.month - ], + "geometry": southern_hemisphere & bounds_polygon, # Intersect shapes + "density": [densities[southern_seasons[month]] for month in dates.month], } ), ] @@ -104,5 +93,4 @@ def import_data(self, bounds): density_df.drop(columns=["index"], inplace=True) density_df = density_df.drop_duplicates() - density_df = density_df.set_index("time").sort_index() - return density_df + return density_df.set_index("time").sort_index() diff --git a/meshiphi/dataloaders/lut/lut_csv.py b/meshiphi/dataloaders/lut/lut_csv.py index 1ce2b37c..0f04632f 100644 --- a/meshiphi/dataloaders/lut/lut_csv.py +++ b/meshiphi/dataloaders/lut/lut_csv.py @@ -1,10 +1,10 @@ -from meshiphi.dataloaders.lut.abstract_lut import LutDataLoader -from meshiphi.mesh_generation.boundary import Boundary - +import numpy as np import pandas as pd -from shapely.ops import unary_union from shapely import wkt -import numpy as np +from shapely.ops import unary_union + +from meshiphi.dataloaders.lut.abstract_lut import LutDataLoader +from meshiphi.mesh_generation.boundary import Boundary class LutCSV(LutDataLoader): @@ -23,6 +23,8 @@ def import_data(self, bounds): data_name (read from CSV by default) """ # Read in all files and create dataframe from them + if self.files is None: + raise ValueError("files parameter is required for LutCSV") df_list = [pd.read_csv(file, index_col=False) for file in self.files] csv_df = pd.concat(df_list, ignore_index=True) @@ -37,7 +39,7 @@ def import_data(self, bounds): ) # Set data name to column in CSV - self.data_name = list(set(csv_df.columns.values) - set(["geometry"]))[0] + self.data_name = next(iter(set(csv_df.columns.values) - {"geometry"})) # Convert strings to shapely geometries csv_df["geometry"] = csv_df["geometry"].apply(wkt.loads) # Create boundary denoting the world @@ -48,6 +50,4 @@ def import_data(self, bounds): # Set remainder to have value np.nan csv_df.loc[len(csv_df.index)] = [undefined_polygon, np.nan] # Limit to boundary - csv_df = self.trim_datapoints(bounds, data=csv_df) - - return csv_df + return self.trim_datapoints(bounds, data=csv_df) diff --git a/meshiphi/dataloaders/lut/lut_geojson.py b/meshiphi/dataloaders/lut/lut_geojson.py index 084f5650..abd77f2f 100644 --- a/meshiphi/dataloaders/lut/lut_geojson.py +++ b/meshiphi/dataloaders/lut/lut_geojson.py @@ -1,10 +1,14 @@ -from meshiphi.dataloaders.lut.abstract_lut import LutDataLoader -from meshiphi.mesh_generation.boundary import Boundary +from __future__ import annotations + +from typing import Any import geopandas as gpd +import numpy as np import pandas as pd from shapely.ops import unary_union -import numpy as np + +from meshiphi.dataloaders.lut.abstract_lut import LutDataLoader +from meshiphi.mesh_generation.boundary import Boundary class LutGeoJSON(LutDataLoader): @@ -47,6 +51,8 @@ def import_data(self, bounds): data_name ('dummy_data' by default) """ # Read in all files specified and extract geometry of geojsons + if self.files is None: + raise ValueError("files parameter is required for LutGeoJSON") gdf_list = [gpd.read_file(file) for file in self.files] geojson_df = gpd.GeoDataFrame(pd.concat(gdf_list, ignore_index=True)) # Give geometries a value defined in config @@ -59,6 +65,7 @@ def import_data(self, bounds): defined_polygon = unary_union(geojson_df.geometry) undefined_polygon = world_polygon - defined_polygon # Set remainder to have value np.nan (or opposing bool value) + alt_val: Any if self.value is True: alt_val = False elif self.value is False: @@ -67,6 +74,4 @@ def import_data(self, bounds): alt_val = np.nan geojson_df.loc[len(geojson_df.index)] = [undefined_polygon, alt_val] # Limit to boundary - geojson_df = self.trim_datapoints(bounds, data=geojson_df) - - return geojson_df + return self.trim_datapoints(bounds, data=geojson_df) diff --git a/meshiphi/dataloaders/lut/lut_shapefile.py b/meshiphi/dataloaders/lut/lut_shapefile.py index 035f6ebf..111839d3 100644 --- a/meshiphi/dataloaders/lut/lut_shapefile.py +++ b/meshiphi/dataloaders/lut/lut_shapefile.py @@ -1,10 +1,14 @@ -from meshiphi.dataloaders.lut.abstract_lut import LutDataLoader -from meshiphi.mesh_generation.boundary import Boundary +from __future__ import annotations + +from typing import Any import geopandas as gpd +import numpy as np import pandas as pd from shapely.ops import unary_union -import numpy as np + +from meshiphi.dataloaders.lut.abstract_lut import LutDataLoader +from meshiphi.mesh_generation.boundary import Boundary class LutShapefile(LutDataLoader): @@ -47,6 +51,8 @@ def import_data(self, bounds): data_name ('dummy_data' by default) """ # Read in all files specified and extract geometry of shapes + if self.files is None: + raise ValueError("files parameter is required for LutShapefile") gdf_list = [gpd.read_file(file) for file in self.files] shape_df = gpd.GeoDataFrame(pd.concat(gdf_list, ignore_index=True)) # Give shapes a value defined in config @@ -59,6 +65,7 @@ def import_data(self, bounds): defined_polygon = unary_union(shape_df.geometry) undefined_polygon = world_polygon - defined_polygon # Set remainder to have value np.nan (or opposing bool value) + alt_val: Any if self.value is True: alt_val = False elif self.value is False: @@ -67,6 +74,4 @@ def import_data(self, bounds): alt_val = np.nan shape_df.loc[len(shape_df.index)] = [undefined_polygon, alt_val] # Limit to boundary - shape_df = self.trim_datapoints(bounds, data=shape_df) - - return shape_df + return self.trim_datapoints(bounds, data=shape_df) diff --git a/meshiphi/dataloaders/lut/scotland_ncmpa.py b/meshiphi/dataloaders/lut/scotland_ncmpa.py index 94f405db..d48fd405 100644 --- a/meshiphi/dataloaders/lut/scotland_ncmpa.py +++ b/meshiphi/dataloaders/lut/scotland_ncmpa.py @@ -1,10 +1,10 @@ -from meshiphi.dataloaders.lut.abstract_lut import LutDataLoader -from meshiphi.mesh_generation.boundary import Boundary - import geopandas as gpd import pandas as pd from shapely.ops import unary_union +from meshiphi.dataloaders.lut.abstract_lut import LutDataLoader +from meshiphi.mesh_generation.boundary import Boundary + class ScotlandNCMPA(LutDataLoader): def import_data(self, bounds): @@ -22,6 +22,8 @@ def import_data(self, bounds): and variable 'thickness' """ # Read in all files specified and extract geometry of exclusion zones + if self.files is None: + raise ValueError("files parameter is required for ScotlandNCMPA") gdf_list = [gpd.read_file(file) for file in self.files] exclusion_df = gpd.GeoDataFrame(pd.concat(gdf_list, ignore_index=True)) # Denote all geometries as exclusion zones @@ -36,6 +38,4 @@ def import_data(self, bounds): # Add line exclusion_df.loc[len(exclusion_df.index)] = [inclusion_polygon, False] # Limit to boundary - exclusion_df = self.trim_datapoints(bounds, data=exclusion_df) - - return exclusion_df + return self.trim_datapoints(bounds, data=exclusion_df) diff --git a/meshiphi/dataloaders/lut/thickness.py b/meshiphi/dataloaders/lut/thickness.py index a9dd7e22..e8343252 100644 --- a/meshiphi/dataloaders/lut/thickness.py +++ b/meshiphi/dataloaders/lut/thickness.py @@ -1,9 +1,8 @@ -from meshiphi.dataloaders.lut.abstract_lut import LutDataLoader -from meshiphi.mesh_generation.boundary import Boundary - import pandas as pd from shapely import Polygon +from meshiphi.dataloaders.lut.abstract_lut import LutDataLoader +from meshiphi.mesh_generation.boundary import Boundary # Mapping of month number to season per hemisphere northern_seasons = { @@ -141,10 +140,7 @@ def import_data(self, bounds): for region in regions: intersection = region.geometry & bounds_polygon - if ( - intersection.geom_type in ["Polygon", "MultiPolygon"] - and intersection != Polygon() - ): + if intersection.geom_type in ["Polygon", "MultiPolygon"] and intersection != Polygon(): region_df = pd.concat( [ pd.DataFrame( @@ -161,6 +157,4 @@ def import_data(self, bounds): thickness_df = pd.concat([thickness_df, region_df]) thickness_df = thickness_df.drop_duplicates() - thickness_df = thickness_df.set_index("time").sort_index() - - return thickness_df + return thickness_df.set_index("time").sort_index() diff --git a/meshiphi/dataloaders/scalar/abstract_scalar.py b/meshiphi/dataloaders/scalar/abstract_scalar.py index 76dc076d..9250080b 100644 --- a/meshiphi/dataloaders/scalar/abstract_scalar.py +++ b/meshiphi/dataloaders/scalar/abstract_scalar.py @@ -1,11 +1,16 @@ +from __future__ import annotations + import logging +from abc import abstractmethod +from typing import Any, overload + import numpy as np -import xarray as xr import pandas as pd -from meshiphi.dataloaders.dataloader_interface import DataLoaderInterface -from abc import abstractmethod -from pyproj import Transformer, CRS +import xarray as xr +from pyproj import CRS, Transformer from rasterio.enums import Resampling + +from meshiphi.dataloaders.dataloader_interface import DataLoaderInterface from meshiphi.mesh_generation.boundary import Boundary logger = logging.getLogger(__name__) @@ -16,7 +21,40 @@ class ScalarDataLoader(DataLoaderInterface): Abstract class for all scalar Datasets. """ - def __init__(self, bounds, params): + # Type hints for dynamically set attributes (set via setattr in __init__) + files: list[str] | None + data_name: str | None + dataloader_name: str + in_proj: str + out_proj: str + x_col: str + y_col: str + downsample_factors: list[int] + aggregate_type: str + min_dp: int + fast_reprojection: bool + # Shape-specific attributes (set by shape dataloaders) + # nx and ny always have defaults (101), so they're not optional + ny: int = 101 + nx: int = 101 + vertical: bool | None = None + multiplier: float | None = None + radius: float | None = None + centre: tuple[float, float] | tuple[None, None] | None = None + width: float | None = None + height: float | None = None + gridsize: int | tuple[int, int] | None = None + # GRF-specific attributes + seed: int | None = None + size: int | None = None + alpha: float | None = None + binary: bool | None = None + threshold: float | None = None + min: float | None = None + max: float | None = None + offset: float | None = None + + def __init__(self, bounds: Boundary, params: dict[str, Any]) -> None: """ This is where large-scale operations are performed, such as importing data, downsampling, reprojecting, and renaming @@ -50,7 +88,7 @@ def __init__(self, bounds, params): # Read in and manipulate data to standard form self.data = self.import_data(bounds) - if "files" in params: + if "files" in params and self.files is not None: logger.info("\tFiles read:") for file in self.files: logger.info(f"\t\t{file}") @@ -88,10 +126,7 @@ def __init__(self, bounds, params): else: # Cut dataset down to initial boundary logger.info( - "\tTrimming data to initial boundary: {min} to {max}".format( - min=(bounds.get_lat_min(), bounds.get_long_min()), - max=(bounds.get_lat_max(), bounds.get_long_max()), - ) + f"\tTrimming data to initial boundary: {(bounds.get_lat_min(), bounds.get_long_min())} to {(bounds.get_lat_max(), bounds.get_long_max())}" ) self.data = self.trim_datapoints(bounds) @@ -115,7 +150,6 @@ def import_data(self, bounds): Downsampling and reprojecting happen in __init__() method """ - pass def add_default_params(self, params): """ @@ -184,26 +218,22 @@ def calculate_coverage(self, bounds, data=None): def calculate_coverage_from_df(bounds, data): data = data.dropna().reset_index() # If empty dataframe, 0% coverage - if data.empty: - return 0 - # If no valid coordinates within data range, 0% coverage - elif data.lat.size == 0 or data.long.size == 0: + if data.empty or data.lat.size == 0 or data.long.size == 0: return 0 # Otherwise, calculate coverage, assuming rectangular region # in mercator projection - else: - # Create a polygon to calculate overlap region from - data_boundary = Boundary( - [data.lat.min(), data.lat.max()], [data.long.min(), data.long.max()] - ) - data_polygon = data_boundary.to_polygon() - bounds_polygon = bounds.to_polygon() + # Create a polygon to calculate overlap region from + data_boundary = Boundary( + [data.lat.min(), data.lat.max()], [data.long.min(), data.long.max()] + ) + data_polygon = data_boundary.to_polygon() + bounds_polygon = bounds.to_polygon() - # Get fraction of bounds covered by data - overlap_area = data_polygon.intersection(bounds_polygon).area - total_area = bounds_polygon.area + # Get fraction of bounds covered by data + overlap_area = data_polygon.intersection(bounds_polygon).area + total_area = bounds_polygon.area - return overlap_area / total_area + return overlap_area / total_area def calculate_coverage_from_xr(bounds, data): # Remove all NaN columns/rows @@ -214,20 +244,19 @@ def calculate_coverage_from_xr(bounds, data): return 0 # Otherwise, calculate coverage, assuming rectangular region # in mercator projection - else: - # Create a polygon to calculate overlap region from - data_boundary = Boundary( - [data.lat.min().item(), data.lat.max().item()], - [data.long.min().item(), data.long.max().item()], - ) - data_polygon = data_boundary.to_polygon() - bounds_polygon = bounds.to_polygon() + # Create a polygon to calculate overlap region from + data_boundary = Boundary( + [data.lat.min().item(), data.lat.max().item()], + [data.long.min().item(), data.long.max().item()], + ) + data_polygon = data_boundary.to_polygon() + bounds_polygon = bounds.to_polygon() - # Get fraction of bounds covered by data - overlap_area = data_polygon.intersection(bounds_polygon).area - total_area = bounds_polygon.area + # Get fraction of bounds covered by data + overlap_area = data_polygon.intersection(bounds_polygon).area + total_area = bounds_polygon.area - return overlap_area / total_area + return overlap_area / total_area # Use self.data if not no explicit dataset specified if data is None: @@ -235,8 +264,9 @@ def calculate_coverage_from_xr(bounds, data): # Calculate data coverage fraction if isinstance(self.data, pd.core.frame.DataFrame): return calculate_coverage_from_df(bounds, data) - elif isinstance(self.data, xr.core.dataset.Dataset): + if isinstance(self.data, xr.core.dataset.Dataset): return calculate_coverage_from_xr(bounds, data) + raise TypeError(f"Unsupported data type: {type(self.data)}") def trim_datapoints(self, bounds, data=None): """ @@ -308,21 +338,19 @@ def trim_datapoints_from_df(data, bounds): ) time_min = pd.to_datetime(bounds.get_time_min()) time_max = pd.to_datetime(bounds.get_time_max()) - time_mask = (time_series >= time_min) & ( - time_series <= time_max - ) + time_mask = (time_series >= time_min) & (time_series <= time_max) return spatially_filtered.iloc[time_mask.values] except Exception: # If conversion fails, fall back to original comparison - time_mask = ( - spatially_filtered["time"] >= bounds.get_time_min() - ) & (spatially_filtered["time"] <= bounds.get_time_max()) + time_mask = (spatially_filtered["time"] >= bounds.get_time_min()) & ( + spatially_filtered["time"] <= bounds.get_time_max() + ) return spatially_filtered.loc[time_mask] else: # Already datetime, use direct comparison - time_mask = ( - spatially_filtered["time"] >= bounds.get_time_min() - ) & (spatially_filtered["time"] <= bounds.get_time_max()) + time_mask = (spatially_filtered["time"] >= bounds.get_time_min()) & ( + spatially_filtered["time"] <= bounds.get_time_max() + ) return spatially_filtered.loc[time_mask] else: return data.iloc[spatial_mask] @@ -378,20 +406,12 @@ def trim_datapoints_from_xr(data, bounds): lat_vals = data.lat.values long_vals = data.long.values - lat_min_idx = np.searchsorted( - lat_vals, bounds.get_lat_min(), side="right" - ) - lat_max_idx = np.searchsorted( - lat_vals, bounds.get_lat_max(), side="right" - ) + lat_min_idx = np.searchsorted(lat_vals, bounds.get_lat_min(), side="right") + lat_max_idx = np.searchsorted(lat_vals, bounds.get_lat_max(), side="right") if bounds.get_long_min() < bounds.get_long_max(): - long_min_idx = np.searchsorted( - long_vals, bounds.get_long_min(), side="right" - ) - long_max_idx = np.searchsorted( - long_vals, bounds.get_long_max(), side="right" - ) + long_min_idx = np.searchsorted(long_vals, bounds.get_long_min(), side="right") + long_max_idx = np.searchsorted(long_vals, bounds.get_long_max(), side="right") # Use integer-based indexing data = data.isel( @@ -400,9 +420,7 @@ def trim_datapoints_from_xr(data, bounds): ) else: # Fallback to coordinate-based selection - data = data.sel( - lat=slice(bounds.get_lat_min(), bounds.get_lat_max()) - ) + data = data.sel(lat=slice(bounds.get_lat_min(), bounds.get_lat_max())) data_lhs = data.sel(long=slice(-180, bounds.get_long_max())) data_rhs = data.sel(long=slice(bounds.get_long_min(), 180)) data = xr.concat([data_lhs, data_rhs], "long") @@ -411,19 +429,15 @@ def trim_datapoints_from_xr(data, bounds): # Fallback to standard coordinate-based selection if optimisation fails data = data.sel(lat=slice(bounds.get_lat_min(), bounds.get_lat_max())) if bounds.get_long_min() < bounds.get_long_max(): - data = data.sel( - long=slice(bounds.get_long_min(), bounds.get_long_max()) - ) + data = data.sel(long=slice(bounds.get_long_min(), bounds.get_long_max())) else: data_lhs = data.sel(long=slice(-180, bounds.get_long_max())) data_rhs = data.sel(long=slice(bounds.get_long_min(), 180)) data = xr.concat([data_lhs, data_rhs], "long") # Select data region within temporal bounds if time exists as a coordinate - if "time" in data.coords.keys(): - data = data.sel( - time=slice(bounds.get_time_min(), bounds.get_time_max()) - ) + if "time" in data.coords: + data = data.sel(time=slice(bounds.get_time_min(), bounds.get_time_max())) # Trim off any data on the min boundary to be consistent with df if len(data.lat) > 0 and bounds.get_lat_min() in data.lat.values: @@ -440,8 +454,17 @@ def trim_datapoints_from_xr(data, bounds): if isinstance(data, pd.core.frame.DataFrame): return trim_datapoints_from_df(data, bounds) - elif isinstance(data, xr.core.dataset.Dataset): + if isinstance(data, xr.core.dataset.Dataset): return trim_datapoints_from_xr(data, bounds) + raise TypeError(f"Unsupported data type: {type(data)}") + + @overload + def get_value(self, bounds: Boundary, agg_type: str) -> dict[str, float]: ... + + @overload + def get_value( + self, bounds: Boundary, data: Any = None, agg_type: str | None = None, skipna: bool = True + ) -> dict[str, float]: ... def get_value(self, bounds, data=None, agg_type=None, skipna=True): """ @@ -493,24 +516,21 @@ def get_value_from_df(dps, bounds, agg_type, skipna): if agg_type == "COUNT": return len(dps) # If no data - elif len(dps) == 0: - return np.nan - elif np.isnan(dps).all(): + if len(dps) == 0 or np.isnan(dps).all(): return np.nan # Return float of aggregated value - elif agg_type == "MIN": + if agg_type == "MIN": return dps.min(skipna=skipna) - elif agg_type == "MAX": + if agg_type == "MAX": return dps.max(skipna=skipna) - elif agg_type == "MEAN": + if agg_type == "MEAN": return dps.mean(skipna=skipna) - elif agg_type == "MEDIAN": + if agg_type == "MEDIAN": return dps.median(skipna=skipna) - elif agg_type == "STD": + if agg_type == "STD": return dps.std(skipna=skipna) # If aggregation_type not available - else: - raise ValueError(f"Unknown aggregation type {agg_type}") + raise ValueError(f"Unknown aggregation type {agg_type}") def get_value_from_xr(dps, bounds, agg_type, skipna): """ @@ -539,45 +559,35 @@ def get_value_from_xr(dps, bounds, agg_type, skipna): if agg_type == "COUNT": return dps.size # If no data - elif dps.size == 0: - return np.nan - # If all NaN, avoid calculations - elif np.isnan(dps).all(): + if dps.size == 0 or np.isnan(dps).all(): return np.nan # Return float of aggregated value - elif agg_type == "MIN": + if agg_type == "MIN": if skipna: return np.nanmin(dps) - else: - return np.min(dps) - elif agg_type == "MAX": + return np.min(dps) + if agg_type == "MAX": if skipna: return np.nanmax(dps) - else: - return np.max(dps) - elif agg_type == "MEAN": + return np.max(dps) + if agg_type == "MEAN": if skipna: return np.nanmean(dps) - else: - return np.mean(dps) - elif agg_type == "MEDIAN": + return np.mean(dps) + if agg_type == "MEDIAN": if skipna: return np.nanmedian(dps) - else: - return np.median(dps) - elif agg_type == "STD": + return np.median(dps) + if agg_type == "STD": if skipna: return np.nanstd(dps) - else: - return np.std(dps) - elif agg_type == "RMSE": + return np.std(dps) + if agg_type == "RMSE": if skipna: return np.sqrt(np.nanmean((dps - np.nanmean(dps)) ** 2)) - else: - return np.sqrt(np.mean((dps - np.mean(dps)) ** 2)) + return np.sqrt(np.mean((dps - np.mean(dps)) ** 2)) # If aggregation_type not available - else: - raise ValueError(f"Unknown aggregation type {agg_type}") + raise ValueError(f"Unknown aggregation type {agg_type}") # Set to params if no specific aggregate type specified if agg_type is None: @@ -665,10 +675,7 @@ def get_hom_condition_from_df(dps, splitting_conds): if frac_over_threshold <= splitting_conds["lower_bound"]: hom_type = "CLR" elif frac_over_threshold >= splitting_conds["upper_bound"]: - if splitting_conds["split_lock"] is True: - hom_type = "HOM" - else: - hom_type = "CLR" + hom_type = "HOM" if splitting_conds["split_lock"] is True else "CLR" else: hom_type = "HET" @@ -700,14 +707,9 @@ def get_hom_condition_from_xr(dps, splitting_conds): ) else: # Determine fraction of datapoints over threshold value - num_over_threshold = np.count_nonzero( - dps > splitting_conds["threshold"] - ) + num_over_threshold = np.count_nonzero(dps > splitting_conds["threshold"]) num_non_nan = np.count_nonzero(~np.isnan(dps)) - if num_non_nan > 0: - frac_over_threshold = num_over_threshold / num_non_nan - else: - frac_over_threshold = 0 + frac_over_threshold = num_over_threshold / num_non_nan if num_non_nan > 0 else 0 # Return homogeneity condition if frac_over_threshold <= splitting_conds["lower_bound"]: hom_type = "CLR" @@ -731,22 +733,16 @@ def get_hom_condition_from_xr(dps, splitting_conds): # Set default values for splitting_conds if not provided if "split_lock" not in splitting_conds: splitting_conds["split_lock"] = False - if data is None: - dps = self.trim_datapoints(bounds)[self.data_name] - else: - dps = data[self.data_name] + dps = self.trim_datapoints(bounds)[self.data_name] if data is None else data[self.data_name] # Retrieve datapoints to analyse if isinstance(dps, pd.core.series.Series): return get_hom_condition_from_df(dps, splitting_conds) - elif isinstance(dps, xr.core.dataarray.DataArray): + if isinstance(dps, xr.core.dataarray.DataArray): return get_hom_condition_from_xr(dps, splitting_conds) - else: - raise TypeError(f"Unknown type {type(dps)}") + raise TypeError(f"Unknown type {type(dps)}") - def reproject( - self, in_proj="EPSG:4326", out_proj="EPSG:4326", x_col="lat", y_col="long" - ): + def reproject(self, in_proj="EPSG:4326", out_proj="EPSG:4326", x_col="lat", y_col="long"): """ Reprojects data using pyProj.Transformer self.data can be pd.DataFrame or xr.Dataset @@ -797,9 +793,9 @@ def reproject_df(data, in_proj, out_proj, x_col, y_col): ('time' if in original dataset), and data_name """ # Do the reprojection - x, y = Transformer.from_crs( - CRS(in_proj), CRS(out_proj), always_xy=True - ).transform(data[x_col].to_numpy(), data[y_col].to_numpy()) + x, y = Transformer.from_crs(CRS(in_proj), CRS(out_proj), always_xy=True).transform( + data[x_col].to_numpy(), data[y_col].to_numpy() + ) # Replace columns with reprojected columns called 'lat'/'long' if x_col != "long": data = data.drop(x_col, axis=1) @@ -851,27 +847,24 @@ def reproject_xr(data, in_proj, out_proj, x_col, y_col, fast=False): # Rename coordinates data = data.rename({x_col: "long", y_col: "lat"}) # Reorder coords in case they are wrong - data = data.sortby("lat", ascending=True) - data = data.sortby("long", ascending=True) - return data + return data.sortby("lat", ascending=True).sortby("long", ascending=True) # If want accurate results - else: - df = data.to_dataframe().reset_index().dropna() - return reproject_df(df, in_proj, out_proj, x_col, y_col) + df = data.to_dataframe().reset_index().dropna() + return reproject_df(df, in_proj, out_proj, x_col, y_col) # If no reprojection to do if in_proj == out_proj: logger.debug("\tself.reproject() called but don't need to") return self.data - else: - logger.info(f"\tReprojecting data from {in_proj} to {out_proj}") + logger.info(f"\tReprojecting data from {in_proj} to {out_proj}") # Choose appropriate method of reprojection based on data type if isinstance(self.data, pd.core.frame.DataFrame): return reproject_df(self.data, in_proj, out_proj, x_col, y_col) - elif isinstance(self.data, xr.core.dataset.Dataset): + if isinstance(self.data, xr.core.dataset.Dataset): return reproject_xr( self.data, in_proj, out_proj, x_col, y_col, fast=self.fast_reprojection ) + raise TypeError(f"Unsupported data type: {type(self.data)}") def downsample(self, agg_type=None): """ @@ -936,7 +929,7 @@ def downsample_xr(data, ds, agg_type): data = data.thin(long=ds[0]) return data - def downsample_df(data, ds, agg_type): + def downsample_df(data, _ds, _agg_type): """ Downsample pandas dataframe Not implemented as it just adds to processing time, @@ -953,16 +946,16 @@ def downsample_df(data, ds, agg_type): agg_type = self.aggregate_type # If no downsampling - if self.downsample_factors == (1, 1) or self.downsample_factors == [1, 1]: + if list(self.downsample_factors) == [1, 1]: logger.debug("\tself.downsample() called but don't have to") return self.data - else: - logger.info(f"\tDownsampling data by {self.downsample_factors}") + logger.info(f"\tDownsampling data by {self.downsample_factors}") # Otherwise, downsample appropriately if isinstance(self.data, pd.core.frame.DataFrame): return downsample_df(self.data, self.downsample_factors, agg_type) - elif isinstance(self.data, xr.core.dataset.Dataset): + if isinstance(self.data, xr.core.dataset.Dataset): return downsample_xr(self.data, self.downsample_factors, agg_type) + raise TypeError(f"Unsupported data type: {type(self.data)}") def get_data_col_name(self): """ @@ -994,9 +987,7 @@ def get_data_name_from_df(data): # Store name of data column for future reference columns = data.columns # Filter out lat, long, time columns leaves us with data column name - filtered_cols = filter( - lambda col: col not in ["lat", "long", "time"], columns - ) + filtered_cols = filter(lambda col: col not in ["lat", "long", "time"], columns) name = list(filtered_cols) if len(name) != 1: raise ValueError( @@ -1030,8 +1021,9 @@ def get_data_name_from_xr(data): # Choose method of extraction based on data type if isinstance(self.data, pd.core.frame.DataFrame): return get_data_name_from_df(self.data) - elif isinstance(self.data, xr.core.dataset.Dataset): + if isinstance(self.data, xr.core.dataset.Dataset): return get_data_name_from_xr(self.data) + raise TypeError(f"Unsupported data type: {type(self.data)}") def set_data_col_name(self, new_name): """ @@ -1089,5 +1081,6 @@ def set_name_xr(data, old_name, new_name): # Change data name depending on data type if isinstance(self.data, pd.core.frame.DataFrame): return set_name_df(self.data, old_name, new_name) - elif isinstance(self.data, xr.core.dataset.Dataset): + if isinstance(self.data, xr.core.dataset.Dataset): return set_name_xr(self.data, old_name, new_name) + raise TypeError(f"Unsupported data type: {type(self.data)}") diff --git a/meshiphi/dataloaders/scalar/amsr.py b/meshiphi/dataloaders/scalar/amsr.py index d0026a32..94e44c7b 100644 --- a/meshiphi/dataloaders/scalar/amsr.py +++ b/meshiphi/dataloaders/scalar/amsr.py @@ -1,9 +1,10 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader -from datetime import datetime import logging +from datetime import datetime import xarray as xr +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader + class AMSRDataLoader(ScalarDataLoader): def add_default_params(self, params): @@ -58,8 +59,7 @@ def retrieve_date(filename): asi-AMSR2-s6250--v.5.4.nc """ date = filename.split("-")[-2] - date = f"{date[:4]}-{date[4:6]}-{date[6:]}" - return date + return f"{date[:4]}-{date[4:6]}-{date[6:]}" def retrieve_data(filename, date): """ @@ -67,10 +67,11 @@ def retrieve_data(filename, date): """ data = xr.open_dataset(filename) # Add date to data - data = data.assign_coords(time=date) - return data + return data.assign_coords(time=date) data_array = [] + if self.files is None: + raise ValueError("files parameter is required for AMSRDataLoader") relevant_files = [] # For each file found from config for file in self.files: diff --git a/meshiphi/dataloaders/scalar/baltic_sea_ice.py b/meshiphi/dataloaders/scalar/baltic_sea_ice.py index e15d4a48..9ccc0f14 100644 --- a/meshiphi/dataloaders/scalar/baltic_sea_ice.py +++ b/meshiphi/dataloaders/scalar/baltic_sea_ice.py @@ -1,7 +1,7 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader - import xarray as xr +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader + class BalticSeaIceDataLoader(ScalarDataLoader): def import_data(self, bounds): @@ -18,6 +18,8 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'SIC' """ # Open Dataset + if self.files is None: + raise ValueError("files parameter is required for BalticSeaIceDataLoader") if len(self.files) == 1: data = xr.open_dataset(self.files[0]) else: @@ -29,6 +31,4 @@ def import_data(self, bounds): # Reverse order of lat as array goes from max to min data = data.reindex(lat=data.lat[::-1]) # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/scalar/bsose_depth.py b/meshiphi/dataloaders/scalar/bsose_depth.py index c4064cfa..b0c46b6e 100644 --- a/meshiphi/dataloaders/scalar/bsose_depth.py +++ b/meshiphi/dataloaders/scalar/bsose_depth.py @@ -1,7 +1,7 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader - import xarray as xr +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader + class BSOSEDepthDataLoader(ScalarDataLoader): def import_data(self, bounds): @@ -19,6 +19,8 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'elevation' """ # Open Dataset + if self.files is None: + raise ValueError("files parameter is required for BSOSEDepthDataLoader") if len(self.files) == 1: data = xr.open_dataset(self.files[0]) else: @@ -36,6 +38,4 @@ def import_data(self, bounds): # Limit to just elevation data data = data["elevation"].to_dataset() # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/scalar/bsose_sea_ice.py b/meshiphi/dataloaders/scalar/bsose_sea_ice.py index 5d37a7a4..da907efe 100644 --- a/meshiphi/dataloaders/scalar/bsose_sea_ice.py +++ b/meshiphi/dataloaders/scalar/bsose_sea_ice.py @@ -1,9 +1,9 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader - import logging import xarray as xr +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader + class BSOSESeaIceDataLoader(ScalarDataLoader): def import_data(self, bounds): @@ -26,6 +26,8 @@ def import_data(self, bounds): and value not 'fraction' or 'percentage' """ # Open Dataset + if self.files is None: + raise ValueError("files parameter is required for BSOSESeaIceDataLoader") if len(self.files) == 1: data = xr.open_dataset(self.files[0]) else: @@ -55,6 +57,6 @@ def import_data(self, bounds): ) else: # Convert to percentage form by default (as expected by the vessel performance models) - data = data.assign(SIC=data["SIC"] * 100) + return data.assign(SIC=data["SIC"] * 100) return data diff --git a/meshiphi/dataloaders/scalar/ecmwf_sig_wave_height.py b/meshiphi/dataloaders/scalar/ecmwf_sig_wave_height.py index d4ffb1c7..9e6111fe 100644 --- a/meshiphi/dataloaders/scalar/ecmwf_sig_wave_height.py +++ b/meshiphi/dataloaders/scalar/ecmwf_sig_wave_height.py @@ -1,9 +1,11 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader -from datetime import datetime, timedelta import logging import os +from datetime import datetime, timedelta + import xarray as xr +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader + class ECMWFSigWaveHeightDataLoader(ScalarDataLoader): def import_data(self, bounds): @@ -23,8 +25,7 @@ def import_data(self, bounds): def extract_base_date(filename): filename = os.path.basename(filename) base_date = filename.split("-")[0] - base_date = datetime.strptime(base_date, "%Y%m%d%H0000") - return base_date + return datetime.strptime(base_date, "%Y%m%d%H0000") def calculate_timestamp(filename): """ @@ -38,10 +39,7 @@ def calculate_timestamp(filename): # Turn forecast len into number of hours forecast_len = int(forecast_len[:-1]) # Calculate - date = datetime.strptime(base_date, "%Y%m%d%H0000") + timedelta( - hours=forecast_len - ) - return date + return datetime.strptime(base_date, "%Y%m%d%H0000") + timedelta(hours=forecast_len) def retrieve_data(filename, timestamp): """ @@ -49,14 +47,15 @@ def retrieve_data(filename, timestamp): """ data = xr.open_dataset(filename) # Add date to data - data = data.assign_coords(time=timestamp) - return data + return data.assign_coords(time=timestamp) # Date limits start_time = datetime.strptime(bounds.get_time_min(), "%Y-%m-%d") end_time = datetime.strptime(bounds.get_time_max(), "%Y-%m-%d") # Go through reverse alphabetical list until date < start_date found + if self.files is None: + raise ValueError("files parameter is required for ECMWFSigWaveHeightDataLoader") self.files = sorted(self.files, reverse=True) for file in self.files: base_date = extract_base_date(file) @@ -67,9 +66,7 @@ def retrieve_data(filename, timestamp): else: raise FileNotFoundError("No file found prior to start time") # Limit files to be read in - self.files = [ - file for file in self.files if extract_base_date(file) == closest_date - ] + self.files = [file for file in self.files if extract_base_date(file) == closest_date] data_array = [] relevant_files = [] @@ -86,9 +83,7 @@ def retrieve_data(filename, timestamp): "\tNo files found for date range " + f"[ {bounds.get_time_min()} : {bounds.get_time_max()} ]" ) - raise FileNotFoundError( - "No ECMWF Wave files found within specified time range!" - ) + raise FileNotFoundError("No ECMWF Wave files found within specified time range!") data = xr.concat(data_array, "time") # Extract just SWH diff --git a/meshiphi/dataloaders/scalar/era5_max_wave_height.py b/meshiphi/dataloaders/scalar/era5_max_wave_height.py index d551ea65..93ebe2dd 100644 --- a/meshiphi/dataloaders/scalar/era5_max_wave_height.py +++ b/meshiphi/dataloaders/scalar/era5_max_wave_height.py @@ -1,9 +1,9 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +from datetime import datetime +from os.path import basename import xarray as xr -from datetime import datetime -from os.path import basename +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader class ERA5MaxWaveHeightDataLoader(ScalarDataLoader): @@ -21,10 +21,11 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'hmax' """ time_range = [ - datetime.strptime(time_str, "%Y-%m-%d") - for time_str in bounds.get_time_range() + datetime.strptime(time_str, "%Y-%m-%d") for time_str in bounds.get_time_range() ] # Reduce files to those within date range + if self.files is None: + raise ValueError("files parameter is required for ERA5MaxWaveHeightDataLoader") self.files = [ file for file in self.files @@ -48,6 +49,4 @@ def import_data(self, bounds): # Reverse order of lat as array goes from max to min data = data.reindex(lat=data.lat[::-1]) # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/scalar/era5_mean_wave_direction.py b/meshiphi/dataloaders/scalar/era5_mean_wave_direction.py index e26423ed..342a829c 100644 --- a/meshiphi/dataloaders/scalar/era5_mean_wave_direction.py +++ b/meshiphi/dataloaders/scalar/era5_mean_wave_direction.py @@ -1,8 +1,10 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader -import xarray as xr from datetime import datetime from os.path import basename +import xarray as xr + +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader + class ERA5MeanWaveDirDataLoader(ScalarDataLoader): def import_data(self, bounds): @@ -19,10 +21,11 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'mwd' """ time_range = [ - datetime.strptime(time_str, "%Y-%m-%d") - for time_str in bounds.get_time_range() + datetime.strptime(time_str, "%Y-%m-%d") for time_str in bounds.get_time_range() ] # Reduce files to those within date range + if self.files is None: + raise ValueError("files parameter is required for ERA5MeanWaveDirectionDataLoader") self.files = [ file for file in self.files @@ -46,6 +49,4 @@ def import_data(self, bounds): # Reverse order of lat as array goes from max to min data = data.reindex(lat=data.lat[::-1]) # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/scalar/era5_sig_wave_height.py b/meshiphi/dataloaders/scalar/era5_sig_wave_height.py index 0b2b8053..ada754ea 100644 --- a/meshiphi/dataloaders/scalar/era5_sig_wave_height.py +++ b/meshiphi/dataloaders/scalar/era5_sig_wave_height.py @@ -1,9 +1,9 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +from datetime import datetime +from os.path import basename import xarray as xr -from datetime import datetime -from os.path import basename +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader class ERA5SigWaveHeightDataLoader(ScalarDataLoader): @@ -21,10 +21,11 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'swh' """ time_range = [ - datetime.strptime(time_str, "%Y-%m-%d") - for time_str in bounds.get_time_range() + datetime.strptime(time_str, "%Y-%m-%d") for time_str in bounds.get_time_range() ] # Reduce files to those within date range + if self.files is None: + raise ValueError("files parameter is required for ERA5SigWaveHeightDataLoader") self.files = [ file for file in self.files @@ -49,6 +50,4 @@ def import_data(self, bounds): # Reverse order of lat as array goes from max to min data = data.reindex(lat=data.lat[::-1]) # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/scalar/era5_wave_period.py b/meshiphi/dataloaders/scalar/era5_wave_period.py index cdd1b633..a9d3d0ed 100644 --- a/meshiphi/dataloaders/scalar/era5_wave_period.py +++ b/meshiphi/dataloaders/scalar/era5_wave_period.py @@ -1,9 +1,9 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +from datetime import datetime +from os.path import basename import xarray as xr -from datetime import datetime -from os.path import basename +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader class ERA5WavePeriodDataLoader(ScalarDataLoader): @@ -21,10 +21,11 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'mwp' """ time_range = [ - datetime.strptime(time_str, "%Y-%m-%d") - for time_str in bounds.get_time_range() + datetime.strptime(time_str, "%Y-%m-%d") for time_str in bounds.get_time_range() ] # Reduce files to those within date range + if self.files is None: + raise ValueError("files parameter is required for ERA5WavePeriodDataLoader") self.files = [ file for file in self.files @@ -48,6 +49,4 @@ def import_data(self, bounds): # Reverse order of lat as array goes from max to min data = data.reindex(lat=data.lat[::-1]) # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/scalar/era5_wind_dir.py b/meshiphi/dataloaders/scalar/era5_wind_dir.py index deff95d7..ff1902a8 100644 --- a/meshiphi/dataloaders/scalar/era5_wind_dir.py +++ b/meshiphi/dataloaders/scalar/era5_wind_dir.py @@ -1,10 +1,10 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +from datetime import datetime +from os.path import basename -import xarray as xr import numpy as np +import xarray as xr -from datetime import datetime -from os.path import basename +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader class ERA5WindDirDataLoader(ScalarDataLoader): @@ -22,10 +22,11 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'wind_dir' """ time_range = [ - datetime.strptime(time_str, "%Y-%m-%d") - for time_str in bounds.get_time_range() + datetime.strptime(time_str, "%Y-%m-%d") for time_str in bounds.get_time_range() ] # Reduce files to those within date range + if self.files is None: + raise ValueError("files parameter is required for ERA5WindDirDataLoader") self.files = [ file for file in self.files @@ -46,15 +47,11 @@ def import_data(self, bounds): data = data.sortby("long") # Calculate direction in degrees between 0 and 360 from vector components - data["wind_dir"] = np.mod( - 180 + (180 / np.pi) * np.arctan2(data["v10"], data["u10"]), 360 - ) + data["wind_dir"] = np.mod(180 + (180 / np.pi) * np.arctan2(data["v10"], data["u10"]), 360) # Limit to just rnn variables data = data["wind_dir"].to_dataset() # Reverse order of lat as array goes from max to min data = data.reindex(lat=data.lat[::-1]) # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/scalar/era5_wind_mag.py b/meshiphi/dataloaders/scalar/era5_wind_mag.py index 01375bf2..85f0219f 100644 --- a/meshiphi/dataloaders/scalar/era5_wind_mag.py +++ b/meshiphi/dataloaders/scalar/era5_wind_mag.py @@ -1,10 +1,10 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +from datetime import datetime +from os.path import basename -import xarray as xr import numpy as np +import xarray as xr -from datetime import datetime -from os.path import basename +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader class ERA5WindMagDataLoader(ScalarDataLoader): @@ -22,10 +22,11 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'wind_mag' """ time_range = [ - datetime.strptime(time_str, "%Y-%m-%d") - for time_str in bounds.get_time_range() + datetime.strptime(time_str, "%Y-%m-%d") for time_str in bounds.get_time_range() ] # Reduce files to those within date range + if self.files is None: + raise ValueError("files parameter is required for ERA5WindMagDataLoader") self.files = [ file for file in self.files @@ -53,6 +54,4 @@ def import_data(self, bounds): # Reverse order of lat as array goes from max to min data = data.reindex(lat=data.lat[::-1]) # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/scalar/gebco.py b/meshiphi/dataloaders/scalar/gebco.py index 0e772fb8..b4c2933d 100644 --- a/meshiphi/dataloaders/scalar/gebco.py +++ b/meshiphi/dataloaders/scalar/gebco.py @@ -1,7 +1,7 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader - import xarray as xr +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader + class GEBCODataLoader(ScalarDataLoader): def import_data(self, bounds): @@ -18,6 +18,8 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'elevation' """ # Import data from files defined in config + if self.files is None: + raise ValueError("files parameter is required for GEBCODataLoader") if len(self.files) == 1: data = xr.open_dataset(self.files[0]) else: @@ -29,6 +31,4 @@ def import_data(self, bounds): data = data["elevation"].to_dataset() # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/scalar/icenet.py b/meshiphi/dataloaders/scalar/icenet.py index 36c60faf..7fad04eb 100644 --- a/meshiphi/dataloaders/scalar/icenet.py +++ b/meshiphi/dataloaders/scalar/icenet.py @@ -1,12 +1,11 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader - import logging - from datetime import datetime, timedelta import xarray as xr -from pandas import to_timedelta from numpy import datetime64 +from pandas import to_timedelta + +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader class IceNetDataLoader(ScalarDataLoader): @@ -38,6 +37,8 @@ def filename_to_datetime(filename): min_time = datetime.strptime(bounds.get_time_min(), "%Y-%m-%d") time_range = max_time - min_time # Retrieve list of dates from filenames + if self.files is None: + raise ValueError("files parameter is required for IceNetDataLoader") file_dates = { filename_to_datetime(file): file for file in self.files diff --git a/meshiphi/dataloaders/scalar/modis.py b/meshiphi/dataloaders/scalar/modis.py index 4ba03908..296cb633 100644 --- a/meshiphi/dataloaders/scalar/modis.py +++ b/meshiphi/dataloaders/scalar/modis.py @@ -1,8 +1,9 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader - import logging + import xarray as xr +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader + logger = logging.getLogger(__name__) @@ -20,8 +21,12 @@ def import_data(self, bounds): MODIS dataset within limits of bounds. Dataset has coordinates 'lat', 'long', and variable 'SIC' """ - logger.info(f"- Opening file {self.file}") # Open Dataset + if self.files is None: + raise ValueError("files parameter is required for MODISDataLoader") + logger.info( + f"- Opening file {self.files[0] if len(self.files) == 1 else f'{len(self.files)} files'}" + ) if len(self.files) == 1: data = xr.open_dataset(self.files[0]) else: @@ -32,6 +37,4 @@ def import_data(self, bounds): # Set areas obscured by cloud to NaN values data = data.where(data.cloud != 1, drop=True) # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/scalar/scalar_csv.py b/meshiphi/dataloaders/scalar/scalar_csv.py index 1c1f1481..2ecdadc1 100644 --- a/meshiphi/dataloaders/scalar/scalar_csv.py +++ b/meshiphi/dataloaders/scalar/scalar_csv.py @@ -1,7 +1,7 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader - import pandas as pd +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader + class ScalarCSVDataLoader(ScalarDataLoader): def import_data(self, bounds): @@ -18,12 +18,12 @@ def import_data(self, bounds): and variable defined by column heading in csv file """ # Read in data + if self.files is None: + raise ValueError("files parameter is required for ScalarCSVDataLoader") df_list = [] # NOTE: All csv files must have same columns for this to work for file in self.files: df_list += [pd.read_csv(file)] data = pd.concat(df_list) # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/scalar/scalar_grf.py b/meshiphi/dataloaders/scalar/scalar_grf.py index e4e6e7d5..d72c4b0b 100644 --- a/meshiphi/dataloaders/scalar/scalar_grf.py +++ b/meshiphi/dataloaders/scalar/scalar_grf.py @@ -1,9 +1,9 @@ +import numpy as np +import pandas as pd + from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader from meshiphi.utils import gaussian_random_field -import pandas as pd -import numpy as np - class ScalarGRFDataLoader(ScalarDataLoader): def add_default_params(self, params): @@ -84,8 +84,7 @@ def grf_to_binary(grf, threshold): grf[grf >= threshold] = 1.0 grf[grf < threshold] = 0.0 # Cast 0/1 to False/True - grf = grf is True - return grf + return grf is True def grf_to_scalar(grf, threshold, min_val, max_val): # Bound GRF to threshold limits @@ -95,30 +94,40 @@ def grf_to_scalar(grf, threshold, min_val, max_val): grf = grf - np.min(grf) grf = grf / (np.max(grf) - np.min(grf)) # Scale to max/min - grf = grf * (max_val - min_val) + min_val - - return grf + return grf * (max_val - min_val) + min_val # Set seed for generation. If not specified, will be 'random' np.random.seed(self.seed) + # Validate required attributes + if self.size is None: + raise ValueError("size parameter is required for ScalarGRFDataLoader") + if self.alpha is None: + raise ValueError("alpha parameter is required for ScalarGRFDataLoader") + # Create a GRF grf = gaussian_random_field(self.size, self.alpha) # Set it to a binary mask if chosen in config if self.binary is True: + if self.threshold is None: + raise ValueError("threshold parameter is required for binary GRF") grf = grf_to_binary(grf, self.threshold) # Set it to scalar GRF field if chosen in config else: + if self.threshold is None or self.min is None or self.max is None: + raise ValueError("threshold, min, and max parameters are required for scalar GRF") grf = grf_to_scalar(grf, self.threshold, self.min, self.max) # Scale with multiplier and offset + if self.multiplier is None: + self.multiplier = 1.0 + if self.offset is None: + self.offset = 0.0 grf = self.multiplier * grf + self.offset # Set up domain of field lat_array = np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.size) - long_array = np.linspace( - bounds.get_long_min(), bounds.get_long_max(), self.size - ) + long_array = np.linspace(bounds.get_long_min(), bounds.get_long_max(), self.size) latv, longv = np.meshgrid(lat_array, long_array, indexing="ij") # Create an entry for each row in final dataframe @@ -130,6 +139,4 @@ def grf_to_scalar(grf, threshold, min_val, max_val): # Cast to dataframe data = pd.DataFrame(rows).set_index(["lat", "long"]) # Set to xarray dataset - data = data.to_xarray() - - return data + return data.to_xarray() diff --git a/meshiphi/dataloaders/scalar/shape.py b/meshiphi/dataloaders/scalar/shape.py index 45735de0..b2351319 100644 --- a/meshiphi/dataloaders/scalar/shape.py +++ b/meshiphi/dataloaders/scalar/shape.py @@ -1,8 +1,9 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader - import logging -import pandas as pd + import numpy as np +import pandas as pd + +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader logger = logging.getLogger(__name__) @@ -90,10 +91,8 @@ def import_data(self, bounds): else: raise ValueError(f"Unknown abstract shape type: {self.dataloader_name}") - data_xr = data.set_index(["lat", "long"]).to_xarray() # No need to trim data, as was defined by bounds - - return data_xr + return data.set_index(["lat", "long"]).to_xarray() def gen_circle(self, bounds): """ @@ -110,11 +109,15 @@ def gen_circle(self, bounds): self.long = np.linspace(bounds.get_long_min(), bounds.get_long_max(), self.nx) # Set centre as centre of data_grid if none specified - c_y = self.lat[int(self.ny / 2)] if not self.centre[0] else self.centre[0] - c_x = self.long[int(self.nx / 2)] if not self.centre[1] else self.centre[1] + if self.radius is None: + raise ValueError("radius parameter is required for circle") + if self.centre is None: + self.centre = (None, None) + c_y = self.lat[int(self.ny / 2)] if self.centre[0] is None else self.centre[0] + c_x = self.long[int(self.nx / 2)] if self.centre[1] is None else self.centre[1] # Create vectors for row and col idx's - y = np.vstack(np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.ny)) + y = np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.ny).reshape(-1, 1) x = np.linspace(bounds.get_long_min(), bounds.get_long_max(), self.nx) logger.info("\tCreating mask of circle") @@ -138,14 +141,13 @@ def gen_circle(self, bounds): index=[0], ) # Avoid concat with empty df - if dummy_df.empty: - dummy_df = row - else: - dummy_df = pd.concat([dummy_df, row], ignore_index=True) + dummy_df = row if dummy_df.empty else pd.concat([dummy_df, row], ignore_index=True) # Change boolean values to int dummy_df["dummy_data"] = dummy_df["dummy_data"].astype(int) # Multiply by scaling factor if present + if self.multiplier is None: + self.multiplier = 1.0 dummy_df["dummy_data"] = dummy_df["dummy_data"] * self.multiplier return dummy_df @@ -166,10 +168,9 @@ def gen_gradient(self, bounds): logger.info("\tCreating gradient of values") # Create 1D gradient - if self.vertical: - gradient = np.linspace(0, 1, self.ny) - else: - gradient = np.linspace(0, 1, self.nx) + if self.vertical is None: + self.vertical = True # Default to vertical gradient + gradient = np.linspace(0, 1, self.ny) if self.vertical else np.linspace(0, 1, self.nx) dummy_df = pd.DataFrame(columns=["lat", "long", "dummy_data"]) logger.info("- Generating dataset") @@ -188,10 +189,7 @@ def gen_gradient(self, bounds): index=[0], ) # Avoid concat with empty df - if dummy_df.empty: - dummy_df = row - else: - dummy_df = pd.concat([dummy_df, row], ignore_index=True) + dummy_df = row if dummy_df.empty else pd.concat([dummy_df, row], ignore_index=True) # Multiply by scaling factor if present dummy_df["dummy_data"] = dummy_df["dummy_data"] * self.multiplier @@ -208,9 +206,7 @@ def gen_checkerboard(self, bounds): """ logger.info("\tSetting up boundary of dataset") # Generate rows - self.lat = np.linspace( - bounds.get_lat_min(), bounds.get_lat_max(), self.ny, endpoint=False - ) + self.lat = np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.ny, endpoint=False) # Generate cols self.long = np.linspace( bounds.get_long_min(), bounds.get_long_max(), self.nx, endpoint=False @@ -218,10 +214,16 @@ def gen_checkerboard(self, bounds): logger.info("- Creating series of 0's and 1's for lat/long") # Create checkerboard pattern + if self.gridsize is None: + self.gridsize = (1, 1) + if isinstance(self.gridsize, int): + gridsize_tuple = (self.gridsize, self.gridsize) + else: + gridsize_tuple = self.gridsize # Create horizontal stripes of 0's and 1's, stripe size defined by gridsize - horizontal = np.floor((self.lat - bounds.get_lat_min()) / self.gridsize[1]) % 2 + horizontal = np.floor((self.lat - bounds.get_lat_min()) / gridsize_tuple[1]) % 2 # Create vertical stripes of 0's and 1's, stripe size defined by gridsize - vertical = np.floor((self.long - bounds.get_long_min()) / self.gridsize[0]) % 2 + vertical = np.floor((self.long - bounds.get_long_min()) / gridsize_tuple[0]) % 2 dummy_df = pd.DataFrame(columns=["lat", "long", "dummy_data"]) logger.info("- Generating dataset") # For each combination of lat/long @@ -239,10 +241,7 @@ def gen_checkerboard(self, bounds): index=[0], ) # Avoid concat with empty df - if dummy_df.empty: - dummy_df = row - else: - dummy_df = pd.concat([dummy_df, row], ignore_index=True) + dummy_df = row if dummy_df.empty else pd.concat([dummy_df, row], ignore_index=True) return dummy_df @@ -261,11 +260,15 @@ def gen_rectangle(self, bounds): self.long = np.linspace(bounds.get_long_min(), bounds.get_long_max(), self.nx) # Set centre as centre of data_grid if none specified - c_y = self.lat[int(self.ny / 2)] if not self.centre[0] else self.centre[0] - c_x = self.long[int(self.nx / 2)] if not self.centre[1] else self.centre[1] + if self.width is None or self.height is None: + raise ValueError("width and height parameters are required for rectangle") + if self.centre is None: + self.centre = (None, None) + c_y = self.lat[int(self.ny / 2)] if self.centre[0] is None else self.centre[0] + c_x = self.long[int(self.nx / 2)] if self.centre[1] is None else self.centre[1] # Create vectors for row and col idx's - y = np.vstack(np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.ny)) + y = np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.ny).reshape(-1, 1) x = np.linspace(bounds.get_long_min(), bounds.get_long_max(), self.nx) logger.info("\tCreating mask of a rectangle") @@ -273,9 +276,7 @@ def gen_rectangle(self, bounds): x_dist_from_centre = np.abs(x - c_x) y_dist_from_centre = np.abs(y - c_y) # Turn this into a mask of values within the rectangle - mask = np.logical_and( - x_dist_from_centre <= self.width, y_dist_from_centre <= self.height - ) + mask = np.logical_and(x_dist_from_centre <= self.width, y_dist_from_centre <= self.height) # Set up empty dataframe to populate with dummy data dummy_df = pd.DataFrame(columns=["lat", "long", "dummy_data"]) logger.info("\tGenerating dataset") @@ -292,14 +293,13 @@ def gen_rectangle(self, bounds): index=[0], ) # Avoid concat with empty df - if dummy_df.empty: - dummy_df = row - else: - dummy_df = pd.concat([dummy_df, row], ignore_index=True) + dummy_df = row if dummy_df.empty else pd.concat([dummy_df, row], ignore_index=True) # Change boolean values to int dummy_df["dummy_data"] = dummy_df["dummy_data"].astype(int) # Multiply by scaling factor if present + if self.multiplier is None: + self.multiplier = 1.0 dummy_df["dummy_data"] = dummy_df["dummy_data"] * self.multiplier return dummy_df diff --git a/meshiphi/dataloaders/scalar/visual_iced.py b/meshiphi/dataloaders/scalar/visual_iced.py index 6a509784..de20e57c 100644 --- a/meshiphi/dataloaders/scalar/visual_iced.py +++ b/meshiphi/dataloaders/scalar/visual_iced.py @@ -1,13 +1,14 @@ -from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader - import logging + import xarray as xr +from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader + logger = logging.getLogger(__name__) class VisualIcedDataLoader(ScalarDataLoader): - def import_data(self, bounds): + def import_data(self, _bounds): """ Reads in data from Visual_Ice NetCDF files. Renames coordinates to 'lat' and 'long'. @@ -21,6 +22,8 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'SIC' """ # Import data from files defined in config + if self.files is None: + raise ValueError("files parameter is required for VisualIcedDataLoader") if len(self.files) == 1: visual_ice = xr.open_dataset(self.files[0]) # If the file is a tiff, use the import_from_tiff method @@ -32,12 +35,8 @@ def import_data(self, bounds): logger.error("File type not supported") return None else: - logger.error( - "Multiple tiff files not supported. Only single tiff file supported" - ) - raise ValueError( - "Multiple tiff files not supported. Only single tiff file supported" - ) + logger.error("Multiple tiff files not supported. Only single tiff file supported") + raise ValueError("Multiple tiff files not supported. Only single tiff file supported") return visual_ice @@ -58,9 +57,7 @@ def import_from_nc(self, xarray_dataset): xarray_dataset = xarray_dataset.rename({"Band1": "SIC"}) # convert SIC to percentage - xarray_dataset = xarray_dataset.assign(SIC=lambda x: x.SIC * 100) - - return xarray_dataset + return xarray_dataset.assign(SIC=lambda x: x.SIC * 100) def import_from_tiff(self, xarray_dataset): """ @@ -84,6 +81,4 @@ def import_from_tiff(self, xarray_dataset): # convert back to xarray vi_dataframe = vi_dataframe.set_index(["x", "y"]) - vi_dataset = vi_dataframe.to_xarray() - - return vi_dataset + return vi_dataframe.to_xarray() diff --git a/meshiphi/dataloaders/vector/abstract_vector.py b/meshiphi/dataloaders/vector/abstract_vector.py index a3547ec3..de8c48b0 100644 --- a/meshiphi/dataloaders/vector/abstract_vector.py +++ b/meshiphi/dataloaders/vector/abstract_vector.py @@ -1,14 +1,16 @@ -from meshiphi.dataloaders.dataloader_interface import DataLoaderInterface -from abc import abstractmethod - -from pyproj import Transformer, CRS +from __future__ import annotations import logging +from abc import abstractmethod +from typing import Any + import numpy as np -import xarray as xr import pandas as pd +import xarray as xr +from pyproj import CRS, Transformer from rasterio.enums import Resampling +from meshiphi.dataloaders.dataloader_interface import DataLoaderInterface from meshiphi.mesh_generation.boundary import Boundary logger = logging.getLogger(__name__) @@ -19,7 +21,40 @@ class VectorDataLoader(DataLoaderInterface): Abstract class for all vector Datasets. """ - def __init__(self, bounds, params): + # Type hints for dynamically set attributes (set via setattr in __init__) + files: list[str] | None + data_name: str | None + dataloader_name: str + in_proj: str + out_proj: str + x_col: str + y_col: str + downsample_factors: list[int] + aggregate_type: str + min_dp: int + # Vector shape-specific attributes + # nx and ny always have defaults (101), so they're not optional + ny: int = 101 + nx: int = 101 + vertical: bool | None = None + multiplier_u: float | None = None + multiplier_v: float | None = None + radius: float | None = None + centre: tuple[float, float] | tuple[None, None] | None = None + width: float | None = None + height: float | None = None + # GRF-specific attributes + seed: int | None = None + size: int | None = None + alpha: float | None = None + min: float | None = None + max: float | None = None + vec_x: str | None = None + vec_y: str | None = None + # Reprojection attribute + fast_reprojection: bool = False + + def __init__(self, bounds: Boundary, params: dict[str, Any]) -> None: """ This is where large-scale operations are performed, such as importing data, downsampling, reprojecting, and renaming @@ -50,7 +85,7 @@ def __init__(self, bounds, params): self.data = self.import_data(bounds) # Read in and manipulate data to standard form - if "files" in params: + if "files" in params and self.files is not None: logger.info("\tFiles read:") for file in self.files: logger.info(f"\t\t{file}") @@ -94,10 +129,7 @@ def __init__(self, bounds, params): else: # Cut dataset down to initial boundary logger.info( - "\tTrimming data to initial boundary: {min} to {max}".format( - min=(bounds.get_lat_min(), bounds.get_long_min()), - max=(bounds.get_lat_max(), bounds.get_long_max()), - ) + f"\tTrimming data to initial boundary: {(bounds.get_lat_min(), bounds.get_long_min())} to {(bounds.get_lat_max(), bounds.get_long_max())}" ) self.data = self.trim_datapoints(bounds) @@ -121,7 +153,6 @@ def import_data(self, bounds): Downsampling and reprojecting happen in __init__() method """ - pass def add_default_params(self, params): """ @@ -234,8 +265,9 @@ def add_mag_dir_to_xr(data, names): # Perform operation on appropriate datatype if isinstance(data, pd.core.frame.DataFrame): return add_mag_dir_to_df(data, names) - elif isinstance(data, xr.core.dataset.Dataset): + if isinstance(data, xr.core.dataset.Dataset): return add_mag_dir_to_xr(data, names) + raise TypeError(f"Unsupported data type: {type(data)}") def calculate_coverage(self, bounds, data=None): """ @@ -257,26 +289,22 @@ def calculate_coverage(self, bounds, data=None): def calculate_coverage_from_df(bounds, data): data = data.dropna().reset_index() # If empty dataframe, 0% coverage - if data.empty: - return 0 - # If no valid coordinates within data range, 0% coverage - elif data.lat.size == 0 or data.long.size == 0: + if data.empty or data.lat.size == 0 or data.long.size == 0: return 0 # Otherwise, calculate coverage, assuming rectangular region # in mercator projection - else: - # Create a polygon to calculate overlap region from - data_boundary = Boundary( - [data.lat.min(), data.lat.max()], [data.long.min(), data.long.max()] - ) - data_polygon = data_boundary.to_polygon() - bounds_polygon = bounds.to_polygon() + # Create a polygon to calculate overlap region from + data_boundary = Boundary( + [data.lat.min(), data.lat.max()], [data.long.min(), data.long.max()] + ) + data_polygon = data_boundary.to_polygon() + bounds_polygon = bounds.to_polygon() - # Get fraction of bounds covered by data - overlap_area = data_polygon.intersection(bounds_polygon).area - total_area = bounds_polygon.area + # Get fraction of bounds covered by data + overlap_area = data_polygon.intersection(bounds_polygon).area + total_area = bounds_polygon.area - return overlap_area / total_area + return overlap_area / total_area def calculate_coverage_from_xr(bounds, data): # Remove all NaN columns/rows @@ -288,20 +316,19 @@ def calculate_coverage_from_xr(bounds, data): return 0 # Otherwise, calculate coverage, assuming rectangular region # in mercator projection - else: - # Create a polygon to calculate overlap region from - data_boundary = Boundary( - [data.lat.min().item(), data.lat.max().item()], - [data.long.min().item(), data.long.max().item()], - ) - data_polygon = data_boundary.to_polygon() - bounds_polygon = bounds.to_polygon() + # Create a polygon to calculate overlap region from + data_boundary = Boundary( + [data.lat.min().item(), data.lat.max().item()], + [data.long.min().item(), data.long.max().item()], + ) + data_polygon = data_boundary.to_polygon() + bounds_polygon = bounds.to_polygon() - # Get fraction of bounds covered by data - overlap_area = data_polygon.intersection(bounds_polygon).area - total_area = bounds_polygon.area + # Get fraction of bounds covered by data + overlap_area = data_polygon.intersection(bounds_polygon).area + total_area = bounds_polygon.area - return overlap_area / total_area + return overlap_area / total_area # Use self.data if not no explicit dataset specified if data is None: @@ -309,8 +336,9 @@ def calculate_coverage_from_xr(bounds, data): # Calculate data coverage fraction if isinstance(self.data, pd.core.frame.DataFrame): return calculate_coverage_from_df(bounds, data) - elif isinstance(self.data, xr.core.dataset.Dataset): + if isinstance(self.data, xr.core.dataset.Dataset): return calculate_coverage_from_xr(bounds, data) + raise TypeError(f"Unsupported data type: {type(self.data)}") def trim_datapoints(self, bounds, data=None): """ @@ -364,20 +392,12 @@ def trim_datapoints_from_xr(data, bounds): lat_vals = data.lat.values long_vals = data.long.values - lat_min_idx = np.searchsorted( - lat_vals, bounds.get_lat_min(), side="right" - ) - lat_max_idx = np.searchsorted( - lat_vals, bounds.get_lat_max(), side="right" - ) + lat_min_idx = np.searchsorted(lat_vals, bounds.get_lat_min(), side="right") + lat_max_idx = np.searchsorted(lat_vals, bounds.get_lat_max(), side="right") if bounds.get_long_min() < bounds.get_long_max(): - long_min_idx = np.searchsorted( - long_vals, bounds.get_long_min(), side="right" - ) - long_max_idx = np.searchsorted( - long_vals, bounds.get_long_max(), side="right" - ) + long_min_idx = np.searchsorted(long_vals, bounds.get_long_min(), side="right") + long_max_idx = np.searchsorted(long_vals, bounds.get_long_max(), side="right") # Use integer-based indexing (faster than coordinate-based) data = data.isel( @@ -386,9 +406,7 @@ def trim_datapoints_from_xr(data, bounds): ) else: # Fallback to coordinate-based selection - data = data.sel( - lat=slice(bounds.get_lat_min(), bounds.get_lat_max()) - ) + data = data.sel(lat=slice(bounds.get_lat_min(), bounds.get_lat_max())) data_lhs = data.sel(long=slice(-180, bounds.get_long_max())) data_rhs = data.sel(long=slice(bounds.get_long_min(), 180)) data = xr.concat([data_lhs, data_rhs], "long") @@ -397,19 +415,15 @@ def trim_datapoints_from_xr(data, bounds): # Fallback to standard coordinate-based selection if optimisation fails data = data.sel(lat=slice(bounds.get_lat_min(), bounds.get_lat_max())) if bounds.get_long_min() < bounds.get_long_max(): - data = data.sel( - long=slice(bounds.get_long_min(), bounds.get_long_max()) - ) + data = data.sel(long=slice(bounds.get_long_min(), bounds.get_long_max())) else: data_lhs = data.sel(long=slice(-180, bounds.get_long_max())) data_rhs = data.sel(long=slice(bounds.get_long_min(), 180)) data = xr.concat([data_lhs, data_rhs], "long") # Select data region within temporal bounds if time exists as a coordinate - if "time" in data.coords.keys(): - data = data.sel( - time=slice(bounds.get_time_min(), bounds.get_time_max()) - ) + if "time" in data.coords: + data = data.sel(time=slice(bounds.get_time_min(), bounds.get_time_max())) # Trim off any data on the min boundary to be consistent with df if len(data.lat) > 0 and bounds.get_lat_min() in data.lat.values: @@ -426,8 +440,9 @@ def trim_datapoints_from_xr(data, bounds): if isinstance(data, pd.core.frame.DataFrame): return trim_datapoints_from_df(data, bounds) - elif isinstance(data, xr.core.dataset.Dataset): + if isinstance(data, xr.core.dataset.Dataset): return trim_datapoints_from_xr(data, bounds) + raise TypeError(f"Unsupported data type: {type(data)}") def get_value(self, bounds, agg_type=None, skipna=True, data=None): """ @@ -495,9 +510,7 @@ def get_value_from_df(dps, variable_names, bounds, agg_type, skipna): elif agg_type == "STD": values = [dps[name].std(skipna=skipna) for name in variable_names] elif agg_type == "MEDIAN": - raise ValueError( - 'Aggregation type "MEDIAN" is non-sensical for vector dataset!' - ) + raise ValueError('Aggregation type "MEDIAN" is non-sensical for vector dataset!') else: raise ValueError(f"Unknown aggregation type {agg_type}") @@ -528,9 +541,7 @@ def get_value_from_xr(dps, variable_names, bounds, agg_type, skipna): f"\t{data_count} datapoints found for attribute '{self.data_name}' within bounds '{bounds}'" ) # If no data, return np.nan for each variable - if data_count == 0: - values = [np.nan for _ in variable_names] - elif np.isnan(dps._magnitude).all(): + if data_count == 0 or np.isnan(dps._magnitude).all(): values = [np.nan for _ in variable_names] # If want count elif agg_type == "COUNT": @@ -542,31 +553,21 @@ def get_value_from_xr(dps, variable_names, bounds, agg_type, skipna): values = [dps[name].count().item() for name in variable_names] elif agg_type == "MIN": # Get 2D index of minimum magnitude point - index = np.unravel_index( - dps._magnitude.argmin(skipna=skipna), dps._magnitude.shape - ) + index = np.unravel_index(dps._magnitude.argmin(skipna=skipna), dps._magnitude.shape) # Get cartesian vector from index values = [dps[name][index].item() for name in variable_names] elif agg_type == "MAX": # Get 2D index of minimum magnitude point - index = np.unravel_index( - dps._magnitude.argmax(skipna=skipna), dps._magnitude.shape - ) + index = np.unravel_index(dps._magnitude.argmax(skipna=skipna), dps._magnitude.shape) # Get cartesian vector from index values = [dps[name][index].item() for name in variable_names] # And the rest self explanatory elif agg_type == "MEAN": - values = [ - dps[name].mean(skipna=skipna).item() for name in variable_names - ] + values = [dps[name].mean(skipna=skipna).item() for name in variable_names] elif agg_type == "STD": - values = [ - dps[name].std(skipna=skipna).item() for name in variable_names - ] + values = [dps[name].std(skipna=skipna).item() for name in variable_names] elif agg_type == "MEDIAN": - raise ValueError( - 'Aggregation type "MEDIAN" is non-sensical for vector dataset!' - ) + raise ValueError('Aggregation type "MEDIAN" is non-sensical for vector dataset!') else: raise ValueError(f"Unknown aggregation type {agg_type}") @@ -577,26 +578,17 @@ def get_value_from_xr(dps, variable_names, bounds, agg_type, skipna): agg_type = self.aggregate_type # Limit data to boundary - if data is None: - dps = self.trim_datapoints(bounds, data=data) - else: - dps = data + dps = self.trim_datapoints(bounds, data=data) if data is None else data # Get list of values if isinstance(self.data, pd.core.frame.DataFrame): - values = get_value_from_df( - dps, self.data_name_list, bounds, agg_type, skipna - ) + values = get_value_from_df(dps, self.data_name_list, bounds, agg_type, skipna) elif isinstance(self.data, xr.core.dataset.Dataset): - values = get_value_from_xr( - dps, self.data_name_list, bounds, agg_type, skipna - ) + values = get_value_from_xr(dps, self.data_name_list, bounds, agg_type, skipna) # Put in dict to map variable to values - return { - self.data_name_list[i]: values[i] for i in range(len(self.data_name_list)) - } + return {self.data_name_list[i]: values[i] for i in range(len(self.data_name_list))} - def get_hom_condition(self, bounds, splitting_conds, agg_type="MEAN", data=None): + def get_hom_condition(self, bounds, splitting_conds, _agg_type="MEAN", data=None): """ Retrieves homogeneity condition of data within boundary. @@ -653,18 +645,12 @@ def get_hom_condition(self, bounds, splitting_conds, agg_type="MEAN", data=None) num_over_threshold = (flow > sc["threshold"]).sum() num_non_nan = np.count_nonzero(~np.isnan(flow)) - if num_non_nan > 0: - frac_over_threshold = num_over_threshold / num_non_nan - else: - frac_over_threshold = 0 + frac_over_threshold = num_over_threshold / num_non_nan if num_non_nan > 0 else 0 if frac_over_threshold <= sc["lower_bound"]: hom_type = "CLR" elif frac_over_threshold >= sc["upper_bound"]: - if sc["split_lock"] is True: - hom_type = "HOM" - else: - hom_type = "CLR" + hom_type = "HOM" if sc["split_lock"] is True else "CLR" else: hom_type = "HET" @@ -674,9 +660,7 @@ def get_hom_condition(self, bounds, splitting_conds, agg_type="MEAN", data=None) return hom_type - def reproject( - self, in_proj="EPSG:4326", out_proj="EPSG:4326", x_col="lat", y_col="long" - ): + def reproject(self, in_proj="EPSG:4326", out_proj="EPSG:4326", x_col="lat", y_col="long"): """ Reprojects data using pyProj.Transformer self.data can be pd.DataFrame or xr.Dataset @@ -727,9 +711,9 @@ def reproject_df(data, in_proj, out_proj, x_col, y_col): ('time' if in original dataset), and data_name """ # Do the reprojection - x, y = Transformer.from_crs( - CRS(in_proj), CRS(out_proj), always_xy=True - ).transform(data[x_col].to_numpy(), data[y_col].to_numpy()) + x, y = Transformer.from_crs(CRS(in_proj), CRS(out_proj), always_xy=True).transform( + data[x_col].to_numpy(), data[y_col].to_numpy() + ) # Replace columns with reprojected columns called 'lat'/'long' if x_col != "long": data = data.drop(x_col, axis=1) @@ -781,27 +765,24 @@ def reproject_xr(data, in_proj, out_proj, x_col, y_col, fast=False): # Rename coordinates data = data.rename({x_col: "long", y_col: "lat"}) # Reorder coords in case they are wrong - data = data.sortby("lat", ascending=True) - data = data.sortby("long", ascending=True) - return data + return data.sortby("lat", ascending=True).sortby("long", ascending=True) # If want accurate results - else: - df = data.to_dataframe().reset_index().dropna() - return reproject_df(df, in_proj, out_proj, x_col, y_col) + df = data.to_dataframe().reset_index().dropna() + return reproject_df(df, in_proj, out_proj, x_col, y_col) # If no reprojection to do if in_proj == out_proj: logger.debug("\tself.reproject() called but don't need to") return self.data - else: - logger.info(f"\tReprojecting data from {in_proj} to {out_proj}") + logger.info(f"\tReprojecting data from {in_proj} to {out_proj}") # Choose appropriate method of reprojection based on data type if isinstance(self.data, pd.core.frame.DataFrame): return reproject_df(self.data, in_proj, out_proj, x_col, y_col) - elif isinstance(self.data, xr.core.dataset.Dataset): + if isinstance(self.data, xr.core.dataset.Dataset): return reproject_xr( self.data, in_proj, out_proj, x_col, y_col, fast=self.fast_reprojection ) + raise TypeError(f"Unsupported data type: {type(self.data)}") def downsample(self, agg_type=None): """ @@ -866,7 +847,7 @@ def downsample_xr(data, ds, agg_type): data = data.thin(long=ds[0]) return data - def downsample_df(data, ds, agg_type): + def downsample_df(data, _ds, _agg_type): """ Downsample pandas dataframe Not implemented as it just adds to processing time, @@ -883,16 +864,16 @@ def downsample_df(data, ds, agg_type): agg_type = self.aggregate_type # If no downsampling - if self.downsample_factors == (1, 1) or self.downsample_factors == [1, 1]: + if list(self.downsample_factors) == [1, 1]: logger.debug("\tself.downsample() called but don't have to") return self.data - else: - logger.info(f"\tDownsampling data by {self.downsample_factors}") + logger.info(f"\tDownsampling data by {self.downsample_factors}") # Otherwise, downsample appropriately if isinstance(self.data, pd.core.frame.DataFrame): return downsample_df(self.data, self.downsample_factors, agg_type) - elif isinstance(self.data, xr.core.dataset.Dataset): + if isinstance(self.data, xr.core.dataset.Dataset): return downsample_xr(self.data, self.downsample_factors, agg_type) + raise TypeError(f"Unsupported data type: {type(self.data)}") def get_data_col_name(self): """ @@ -912,9 +893,7 @@ def get_data_names_from_df(data): # Store name of data column for future reference columns = data.columns # Filter out lat, long, time columns leaves us with data column name - filtered_cols = filter( - lambda col: col not in ["lat", "long", "time"], columns - ) + filtered_cols = filter(lambda col: col not in ["lat", "long", "time"], columns) data_names = list(filtered_cols) # Turn into comma seperated string and return return ",".join(data_names) @@ -932,8 +911,9 @@ def get_data_names_from_xr(data): # Choose method of extraction based on data type if isinstance(self.data, pd.core.frame.DataFrame): return get_data_names_from_df(self.data) - elif isinstance(self.data, xr.core.dataset.Dataset): + if isinstance(self.data, xr.core.dataset.Dataset): return get_data_names_from_xr(self.data) + raise TypeError(f"Unsupported data type: {type(self.data)}") def get_data_col_name_list(self): """ @@ -987,8 +967,9 @@ def set_names_xr(data, name_dict): # Change data name depending on data type if isinstance(self.data, pd.core.frame.DataFrame): return set_names_df(self.data, name_dict) - elif isinstance(self.data, xr.core.dataset.Dataset): + if isinstance(self.data, xr.core.dataset.Dataset): return set_names_xr(self.data, name_dict) + raise TypeError(f"Unsupported data type: {type(self.data)}") def set_data_col_name_list(self, new_names): """ @@ -1005,9 +986,7 @@ def set_data_col_name_list(self, new_names): """ # Check validity of input if not isinstance(new_names, list): - raise TypeError( - f"'new_names' must be a list! Instead it is a {type(new_names)}" - ) + raise TypeError(f"'new_names' must be a list! Instead it is a {type(new_names)}") if len(new_names) != 2: raise ValueError( f"'new_names' must have a length of 2! Instead it has length {len(new_names)}" @@ -1048,19 +1027,14 @@ def calc_curl(self, bounds, data=None, collapse=True, agg_type="MAX"): Raises: ValueError: If agg_type is not 'MAX' or 'MEAN' """ - if data is None: - dps = self.trim_datapoints(bounds, data=data) - else: - dps = data + dps = self.trim_datapoints(bounds, data=data) if data is None else data # Create a meshgrid of vectors from the data vector_field = self._create_vector_meshgrid(dps, self.data_name_list) # Get component values for each vector fx, fy = vector_field[:, :, 0], vector_field[:, :, 1] # If not enough datapoints to compute gradient if 1 in fx.shape or 1 in fy.shape: - logger.debug( - "\tUnable to compute gradient across cell for curl calculation" - ) + logger.debug("\tUnable to compute gradient across cell for curl calculation") curl = np.nan else: # Compute partial derivatives @@ -1074,18 +1048,14 @@ def calc_curl(self, bounds, data=None, collapse=True, agg_type="MAX"): logger.debug("\tAll NaN cellbox encountered") return np.nan # If want to collapse to max mag value, return scalar - elif collapse: + if collapse: if agg_type == "MAX": return max(np.nanmax(curl), np.nanmin(curl), key=abs) - elif agg_type == "MEAN": + if agg_type == "MEAN": return np.nanmean(curl) - else: - raise ValueError( - f"agg_type '{agg_type}' not understood! Requires 'MAX' or 'MEAN'" - ) + raise ValueError(f"agg_type '{agg_type}' not understood! Requires 'MAX' or 'MEAN'") # Else return field - else: - return curl + return curl def calc_dmag(self, bounds, data=None, collapse=True, agg_type="MEAN"): """ @@ -1114,10 +1084,7 @@ def calc_dmag(self, bounds, data=None, collapse=True, agg_type="MEAN"): Raises: ValueError: If agg_type is not 'MAX' or 'MEAN' """ - if data is None: - dps = self.trim_datapoints(bounds, data=data) - else: - dps = data + dps = self.trim_datapoints(bounds, data=data) if data is None else data data_names = self.data_name_list each_vector = dps[data_names].to_numpy() @@ -1130,22 +1097,18 @@ def calc_dmag(self, bounds, data=None, collapse=True, agg_type="MEAN"): logger.debug("\tEmpty cellbox encountered") return np.nan # If d_mag is nan - elif np.isnan(d_mag).all(): + if np.isnan(d_mag).all(): logger.debug("\tAll NaN cellbox encountered") return np.nan # If want to collapse to max mag value, return scalar - elif collapse: + if collapse: if agg_type == "MAX": return np.nanmax(d_mag) - elif agg_type == "MEAN": + if agg_type == "MEAN": return np.nanmean(d_mag) - else: - raise ValueError( - f"agg_type '{agg_type}' not understood! Requires 'MAX' or 'MEAN'" - ) + raise ValueError(f"agg_type '{agg_type}' not understood! Requires 'MAX' or 'MEAN'") # Else return field - else: - return d_mag + return d_mag @staticmethod def _create_vector_meshgrid(data, data_name_list): @@ -1172,9 +1135,7 @@ def meshgrid_from_df(data, data_name_list): vector_x_field = data.pivot(index="lat", columns="long", values=x) vector_y_field = data.pivot(index="lat", columns="long", values=y) # Combine into field of vectors - vector_field = np.stack((vector_x_field, vector_y_field), axis=-1) - vector_field = np.swapaxes(vector_field, 0, 1) - return vector_field + return np.swapaxes(np.stack((vector_x_field, vector_y_field), axis=-1), 0, 1) def meshgrid_from_xr(data, data_name_list): # Extract out each variable and combine as tuple @@ -1184,5 +1145,6 @@ def meshgrid_from_xr(data, data_name_list): if isinstance(data, pd.core.frame.DataFrame): return meshgrid_from_df(data, data_name_list) - elif isinstance(data, xr.core.dataset.Dataset): + if isinstance(data, xr.core.dataset.Dataset): return meshgrid_from_xr(data, data_name_list) + raise TypeError(f"Unsupported data type: {type(data)}") diff --git a/meshiphi/dataloaders/vector/baltic_current.py b/meshiphi/dataloaders/vector/baltic_current.py index df577cc8..f7606502 100644 --- a/meshiphi/dataloaders/vector/baltic_current.py +++ b/meshiphi/dataloaders/vector/baltic_current.py @@ -1,8 +1,7 @@ -from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader - - import xarray as xr +from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader + class BalticCurrentDataLoader(VectorDataLoader): def import_data(self, bounds): @@ -20,6 +19,8 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'uC', 'vC' """ # Open Dataset + if self.files is None: + raise ValueError("files parameter is required for BalticCurrentDataLoader") if len(self.files) == 1: data = xr.open_dataset(self.files[0]) else: @@ -29,14 +30,10 @@ def import_data(self, bounds): data = data.isel(depth=0) data = data.reset_coords(names="depth", drop=True) # Change column names - data = data.rename( - {"latitude": "lat", "longitude": "long", "uo": "uC", "vo": "vC"} - ) + data = data.rename({"latitude": "lat", "longitude": "long", "uo": "uC", "vo": "vC"}) # Trim to initial datapoints data = self.trim_datapoints(bounds, data=data) # Reduce along time dimension - data = data.mean(dim="time") - - return data + return data.mean(dim="time") diff --git a/meshiphi/dataloaders/vector/duacs_current.py b/meshiphi/dataloaders/vector/duacs_current.py index 8c0d3d6f..b5f32bb1 100644 --- a/meshiphi/dataloaders/vector/duacs_current.py +++ b/meshiphi/dataloaders/vector/duacs_current.py @@ -1,10 +1,9 @@ -from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader - +from datetime import datetime +from os.path import basename import xarray as xr -from datetime import datetime -from os.path import basename +from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader class DuacsCurrentDataLoader(VectorDataLoader): @@ -23,10 +22,11 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'uC', 'vC' """ time_range = [ - datetime.strptime(time_str, "%Y-%m-%d") - for time_str in bounds.get_time_range() + datetime.strptime(time_str, "%Y-%m-%d") for time_str in bounds.get_time_range() ] # Reduce files to those within date range + if self.files is None: + raise ValueError("files parameter is required for DuacsCurrentDataLoader") self.files = [ file for file in self.files @@ -41,15 +41,11 @@ def import_data(self, bounds): else: data = xr.open_mfdataset(self.files).compute() # Change column names - data = data.rename( - {"latitude": "lat", "longitude": "long", "ugos": "uC", "vgos": "vC"} - ) + data = data.rename({"latitude": "lat", "longitude": "long", "ugos": "uC", "vgos": "vC"}) # Drop unnecessary variable, if present - if "crs" in data.keys(): + if "crs" in data: data = data.drop_vars("crs") # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/vector/era5_wave_direction_vector.py b/meshiphi/dataloaders/vector/era5_wave_direction_vector.py index de0230b9..b897e2c8 100644 --- a/meshiphi/dataloaders/vector/era5_wave_direction_vector.py +++ b/meshiphi/dataloaders/vector/era5_wave_direction_vector.py @@ -1,12 +1,10 @@ -import numpy as np - -from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader - +from datetime import datetime +from os.path import basename +import numpy as np import xarray as xr -from datetime import datetime -from os.path import basename +from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader class ERA5WaveDirectionLoader(VectorDataLoader): @@ -25,10 +23,11 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variables 'uW', 'vW' """ time_range = [ - datetime.strptime(time_str, "%Y-%m-%d") - for time_str in bounds.get_time_range() + datetime.strptime(time_str, "%Y-%m-%d") for time_str in bounds.get_time_range() ] # Reduce files to those within date range + if self.files is None: + raise ValueError("files parameter is required for ERA5WaveDirectionLoader") self.files = [ file for file in self.files @@ -58,15 +57,8 @@ def import_data(self, bounds): # Limit to just uW and vW data = data[["uW", "vW"]] - # Set min time to start of month to ensure we include data as we only have a - # monthly cadence. Assuming time is in str format - time_min = datetime.strptime(bounds.get_time_min(), "%Y-%m-%d") - time_min = datetime.strftime(time_min, "%Y-%m-01") - # Reverse order of lat as array goes from max to min data = data.reindex(lat=data.lat[::-1]) # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/vector/era5_wind.py b/meshiphi/dataloaders/vector/era5_wind.py index 0c319c6c..2000dfe1 100644 --- a/meshiphi/dataloaders/vector/era5_wind.py +++ b/meshiphi/dataloaders/vector/era5_wind.py @@ -1,10 +1,9 @@ -from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader - +from datetime import datetime +from os.path import basename import xarray as xr -from datetime import datetime -from os.path import basename +from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader class ERA5WindDataLoader(VectorDataLoader): @@ -22,10 +21,11 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variables 'u10', 'v10' """ time_range = [ - datetime.strptime(time_str, "%Y-%m-%d") - for time_str in bounds.get_time_range() + datetime.strptime(time_str, "%Y-%m-%d") for time_str in bounds.get_time_range() ] # Reduce files to those within date range + if self.files is None: + raise ValueError("files parameter is required for ERA5WindDataLoader") self.files = [ file for file in self.files @@ -41,15 +41,8 @@ def import_data(self, bounds): # Change column names data = data.rename({"latitude": "lat", "longitude": "long"}) - # Set min time to start of month to ensure we include data as we only have a - # monthly cadence. Assuming time is in str format - time_min = datetime.strptime(bounds.get_time_min(), "%Y-%m-%d") - time_min = datetime.strftime(time_min, "%Y-%m-01") - # Reverse order of lat as array goes from max to min data = data.reindex(lat=data.lat[::-1]) # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/vector/north_sea_current.py b/meshiphi/dataloaders/vector/north_sea_current.py index 635e5ca1..851f55f3 100644 --- a/meshiphi/dataloaders/vector/north_sea_current.py +++ b/meshiphi/dataloaders/vector/north_sea_current.py @@ -1,8 +1,7 @@ -from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader - - import xarray as xr +from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader + class NorthSeaCurrentDataLoader(VectorDataLoader): def import_data(self, bounds): @@ -20,6 +19,8 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'uC', 'vC' """ # Open Dataset + if self.files is None: + raise ValueError("files parameter is required for NorthSeaCurrentDataLoader") if len(self.files) == 1: data = xr.open_dataset(self.files[0]) else: @@ -30,6 +31,4 @@ def import_data(self, bounds): data = data[["uC", "vC"]] # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/vector/oras5_current.py b/meshiphi/dataloaders/vector/oras5_current.py index 5698febc..774a7491 100644 --- a/meshiphi/dataloaders/vector/oras5_current.py +++ b/meshiphi/dataloaders/vector/oras5_current.py @@ -1,8 +1,7 @@ -from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader - - import xarray as xr +from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader + # TODO Read in 2 files, combine to one object class ORAS5CurrentDataLoader(VectorDataLoader): @@ -21,19 +20,17 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'uC', 'vC' """ # Open Dataset + if self.files is None: + raise ValueError("files parameter is required for ORAS5CurrentDataLoader") if len(self.files) == 1: data = xr.open_dataset(self.files[0]) else: data = xr.open_mfdataset(self.files) # Change column names - data = data.rename( - {"nav_lon": "long", "nav_lat": "lat", "uo": "uC", "vo": "vC"} - ) + data = data.rename({"nav_lon": "long", "nav_lat": "lat", "uo": "uC", "vo": "vC"}) # Limit to just these coords and variables data = data[["lat", "long", "uC", "vC"]] # Limit to initial boundary - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/vector/sose.py b/meshiphi/dataloaders/vector/sose.py index 5eaebcf1..30473425 100644 --- a/meshiphi/dataloaders/vector/sose.py +++ b/meshiphi/dataloaders/vector/sose.py @@ -1,7 +1,7 @@ -from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader - import xarray as xr +from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader + class SOSEDataLoader(VectorDataLoader): def import_data(self, bounds): @@ -20,6 +20,8 @@ def import_data(self, bounds): """ # Open dataset and cast to pandas df + if self.files is None: + raise ValueError("files parameter is required for SOSEDataLoader") if len(self.files) == 1: data = xr.open_dataset(self.files[0]) else: @@ -32,6 +34,4 @@ def import_data(self, bounds): # Extract relevant columns df = df[["lat", "long", "uC", "vC"]] # Trim to initial datapoints - df = self.trim_datapoints(bounds, data=df) - - return df + return self.trim_datapoints(bounds, data=df) diff --git a/meshiphi/dataloaders/vector/vector_csv.py b/meshiphi/dataloaders/vector/vector_csv.py index fcec46de..dd849db1 100644 --- a/meshiphi/dataloaders/vector/vector_csv.py +++ b/meshiphi/dataloaders/vector/vector_csv.py @@ -1,7 +1,7 @@ -from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader - import pandas as pd +from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader + class VectorCSVDataLoader(VectorDataLoader): def import_data(self, bounds): @@ -18,12 +18,12 @@ def import_data(self, bounds): and variable defined by column heading in csv file """ # Read in data + if self.files is None: + raise ValueError("files parameter is required for VectorCSVDataLoader") df_list = [] # NOTE: All csv files must have same columns for this to work for file in self.files: df_list += [pd.read_csv(file)] data = pd.concat(df_list) # Trim to initial datapoints - data = self.trim_datapoints(bounds, data=data) - - return data + return self.trim_datapoints(bounds, data=data) diff --git a/meshiphi/dataloaders/vector/vector_grf.py b/meshiphi/dataloaders/vector/vector_grf.py index 587532aa..0f73f307 100644 --- a/meshiphi/dataloaders/vector/vector_grf.py +++ b/meshiphi/dataloaders/vector/vector_grf.py @@ -1,9 +1,9 @@ +import numpy as np +import pandas as pd + from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader from meshiphi.utils import gaussian_random_field -import pandas as pd -import numpy as np - class VectorGRFDataLoader(VectorDataLoader): def add_default_params(self, params): @@ -34,10 +34,12 @@ def add_default_params(self, params): # Column/variable names if params["data_name"] is None: params["data_name"] = "uC,vC" - if "vec_x" not in params: - params["vec_x"] = params["data_name"].split(",")[0] - if "vec_y" not in params: - params["vec_y"] = params["data_name"].split(",")[1] + data_name = params["data_name"] + if data_name is not None: + if "vec_x" not in params: + params["vec_x"] = data_name.split(",")[0] + if "vec_y" not in params: + params["vec_y"] = data_name.split(",")[1] # Min/Max magnitude if "min" not in params: params["min"] = 0 @@ -74,6 +76,14 @@ def grf_to_vector(magnitudes, directions, min_val, max_val): # Set seed for generation. If not specified, will be 'random' np.random.seed(self.seed) + # Validate required attributes + if self.size is None: + raise ValueError("size parameter is required for VectorGRFDataLoader") + if self.alpha is None: + raise ValueError("alpha parameter is required for VectorGRFDataLoader") + if self.vec_x is None or self.vec_y is None: + raise ValueError("vec_x and vec_y parameters are required for VectorGRFDataLoader") + # Create a GRF of magnitudes and angles magnitudes = gaussian_random_field(self.size, self.alpha) directions = gaussian_random_field(self.size, self.alpha) @@ -83,9 +93,7 @@ def grf_to_vector(magnitudes, directions, min_val, max_val): # Set up domain of field lat_array = np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.size) - long_array = np.linspace( - bounds.get_long_min(), bounds.get_long_max(), self.size - ) + long_array = np.linspace(bounds.get_long_min(), bounds.get_long_max(), self.size) latv, longv = np.meshgrid(lat_array, long_array, indexing="ij") # Create an entry for each row in final dataframe @@ -102,6 +110,4 @@ def grf_to_vector(magnitudes, directions, min_val, max_val): # Cast to dataframe data = pd.DataFrame(rows).set_index(["lat", "long"]) # Set to xarray dataset - data = data.to_xarray() - - return data + return data.to_xarray() diff --git a/meshiphi/dataloaders/vector/vector_shape.py b/meshiphi/dataloaders/vector/vector_shape.py index adbb549b..d83ec386 100644 --- a/meshiphi/dataloaders/vector/vector_shape.py +++ b/meshiphi/dataloaders/vector/vector_shape.py @@ -1,8 +1,9 @@ -from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader - import logging -import pandas as pd + import numpy as np +import pandas as pd + +from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader logger = logging.getLogger(__name__) @@ -84,10 +85,8 @@ def import_data(self, bounds): else: raise ValueError(f"Unknown vector shape type: {self.dataloader_name}") - data_xr = data.set_index(["lat", "long"]).to_xarray() # No need to trim data, as was defined by bounds - - return data_xr + return data.set_index(["lat", "long"]).to_xarray() def gen_gradient(self, bounds): """ @@ -105,10 +104,9 @@ def gen_gradient(self, bounds): logger.info("\tCreating gradient of values") # Create 1D gradient - if self.vertical: - gradient = np.linspace(0, 1, self.ny) - else: - gradient = np.linspace(0, 1, self.nx) + if self.vertical is None: + self.vertical = True # Default to vertical gradient + gradient = np.linspace(0, 1, self.ny) if self.vertical else np.linspace(0, 1, self.nx) dummy_df = pd.DataFrame(columns=["lat", "long", "dummy_data_u", "dummy_data_v"]) logger.info("- Generating vector dataset") @@ -139,12 +137,13 @@ def gen_gradient(self, bounds): index=[0], ) # Avoid concat with empty df - if dummy_df.empty: - dummy_df = row - else: - dummy_df = pd.concat([dummy_df, row], ignore_index=True) + dummy_df = row if dummy_df.empty else pd.concat([dummy_df, row], ignore_index=True) # Multiply by scaling factor if present + if self.multiplier_u is None: + self.multiplier_u = 1.0 + if self.multiplier_v is None: + self.multiplier_v = 1.0 dummy_df["dummy_data_u"] = dummy_df["dummy_data_u"] * self.multiplier_u dummy_df["dummy_data_v"] = dummy_df["dummy_data_v"] * self.multiplier_v @@ -165,11 +164,15 @@ def gen_circle(self, bounds): self.long = np.linspace(bounds.get_long_min(), bounds.get_long_max(), self.nx) # Set centre as centre of data_grid if none specified - c_y = self.lat[int(self.ny / 2)] if not self.centre[0] else self.centre[0] - c_x = self.long[int(self.nx / 2)] if not self.centre[1] else self.centre[1] + if self.radius is None: + raise ValueError("radius parameter is required for vector_circle") + if self.centre is None: + self.centre = (None, None) + c_y = self.lat[int(self.ny / 2)] if self.centre[0] is None else self.centre[0] + c_x = self.long[int(self.nx / 2)] if self.centre[1] is None else self.centre[1] # Create vectors for row and col indices - y = np.vstack(np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.ny)) + y = np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.ny).reshape(-1, 1) x = np.linspace(bounds.get_long_min(), bounds.get_long_max(), self.nx) logger.info("\tCreating mask of circle") @@ -194,10 +197,7 @@ def gen_circle(self, bounds): index=[0], ) # Avoid concat with empty df - if dummy_df.empty: - dummy_df = row - else: - dummy_df = pd.concat([dummy_df, row], ignore_index=True) + dummy_df = row if dummy_df.empty else pd.concat([dummy_df, row], ignore_index=True) # Change boolean values to int dummy_df["dummy_data_u"] = dummy_df["dummy_data_u"].astype(int) @@ -223,11 +223,15 @@ def gen_rectangle(self, bounds): self.long = np.linspace(bounds.get_long_min(), bounds.get_long_max(), self.nx) # Set centre as centre of data_grid if none specified - c_y = self.lat[int(self.ny / 2)] if not self.centre[0] else self.centre[0] - c_x = self.long[int(self.nx / 2)] if not self.centre[1] else self.centre[1] + if self.width is None or self.height is None: + raise ValueError("width and height parameters are required for vector_rectangle") + if self.centre is None: + self.centre = (None, None) + c_y = self.lat[int(self.ny / 2)] if self.centre[0] is None else self.centre[0] + c_x = self.long[int(self.nx / 2)] if self.centre[1] is None else self.centre[1] # Create vectors for row and col indices - y = np.vstack(np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.ny)) + y = np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.ny).reshape(-1, 1) x = np.linspace(bounds.get_long_min(), bounds.get_long_max(), self.nx) logger.info("\tCreating mask of a rectangle") @@ -235,9 +239,7 @@ def gen_rectangle(self, bounds): x_dist_from_centre = np.abs(x - c_x) y_dist_from_centre = np.abs(y - c_y) # Turn this into a mask of values within the rectangle - mask = np.logical_and( - x_dist_from_centre <= self.width, y_dist_from_centre <= self.height - ) + mask = np.logical_and(x_dist_from_centre <= self.width, y_dist_from_centre <= self.height) # Set up empty dataframe to populate with dummy data dummy_df = pd.DataFrame(columns=["lat", "long", "dummy_data_u", "dummy_data_v"]) logger.info("\tGenerating vector dataset") @@ -255,10 +257,7 @@ def gen_rectangle(self, bounds): index=[0], ) # Avoid concat with empty df - if dummy_df.empty: - dummy_df = row - else: - dummy_df = pd.concat([dummy_df, row], ignore_index=True) + dummy_df = row if dummy_df.empty else pd.concat([dummy_df, row], ignore_index=True) # Change boolean values to int dummy_df["dummy_data_u"] = dummy_df["dummy_data_u"].astype(int) diff --git a/meshiphi/mesh_generation/aggregated_cellbox.py b/meshiphi/mesh_generation/aggregated_cellbox.py index 6b94668f..5dddbf3d 100644 --- a/meshiphi/mesh_generation/aggregated_cellbox.py +++ b/meshiphi/mesh_generation/aggregated_cellbox.py @@ -1,7 +1,12 @@ +from __future__ import annotations + +from typing import Any, cast + +import numpy as np +import shapely.wkt from shapely.geometry import Point + from meshiphi.mesh_generation.boundary import Boundary -import shapely.wkt -import numpy as np class AggregatedCellBox: @@ -17,7 +22,7 @@ class AggregatedCellBox: """ @classmethod - def from_json(cls, cellbox_json): + def from_json(cls, cellbox_json: dict[str, Any]) -> AggregatedCellBox: """ Args: @@ -50,9 +55,7 @@ def load_bounds(cellbox_json): long_range = [bounds[0], bounds[2]] # Or something is wrong with the mesh else: - raise TypeError( - f"Expected Polygon or MultiPolygon, instead got {shape.geom_type}" - ) + raise TypeError(f"Expected Polygon or MultiPolygon, instead got {shape.geom_type}") return Boundary(lat_range, long_range) @@ -66,10 +69,9 @@ def load_agg_data(cellbox_json): boundary = load_bounds(cellbox_json) agg_data = load_agg_data(cellbox_json) - obj = AggregatedCellBox(boundary, agg_data, cellbox_id) - return obj + return AggregatedCellBox(boundary, agg_data, cellbox_id) - def __init__(self, boundary, agg_data, id): + def __init__(self, boundary: Boundary, agg_data: dict[str, Any], id: str) -> None: """ Args: @@ -83,43 +85,43 @@ def __init__(self, boundary, agg_data, id): self.id = id ######## setters and getters ######## - def set_bounds(self, boundary): + def set_bounds(self, boundary: Boundary) -> None: """ set the boundary of the CellBox """ self.boundary = boundary - def set_agg_data(self, agg_data): + def set_agg_data(self, agg_data: dict[str, Any]) -> None: """ sets the agg_data """ self.agg_data = agg_data - def set_id(self, id): + def set_id(self, id: str) -> None: """ sets the cellbox id """ self.id = id - def get_bounds(self): + def get_bounds(self) -> Boundary: """ get the boundary of the CellBox """ return self.boundary - def get_agg_data(self): + def get_agg_data(self) -> dict[str, Any]: """ returns the agg_data """ return self.agg_data - def get_id(self): + def get_id(self) -> str: """ returns the id """ return self.id - def to_json(self): + def to_json(self) -> dict[str, Any]: """ convert cellbox to JSON @@ -153,9 +155,7 @@ def to_json(self): for key, value in agg_data.items(): if isinstance(value, (list, tuple)): # Handle lists/arrays that might contain NaN values - clean_value = [ - None if (isinstance(v, float) and np.isnan(v)) else v for v in value - ] + clean_value = [None if (isinstance(v, float) and np.isnan(v)) else v for v in value] elif isinstance(value, float) and np.isnan(value): # Convert single NaN values to None clean_value = None @@ -168,7 +168,7 @@ def to_json(self): return cell_json - def contains_point(self, lat, long): + def contains_point(self, lat: float, long: float) -> bool: """ Returns true if a given lat/long coordinate is contained within this cellbox. @@ -182,8 +182,8 @@ def contains_point(self, lat, long): """ shapely_boundary = self.boundary.to_polygon() point = Point(long, lat) - point_within_bounds = shapely_boundary.contains(point) - point_on_bounds = shapely_boundary.boundary.contains(point) + point_within_bounds = cast("bool", shapely_boundary.contains(point)) + point_on_bounds = cast("bool", shapely_boundary.boundary.contains(point)) point_on_north_edge = shapely_boundary.bounds[2] == point.x point_on_east_edge = shapely_boundary.bounds[3] == point.y @@ -191,7 +191,7 @@ def contains_point(self, lat, long): point_on_bounds and (point_on_north_edge or point_on_east_edge) ) - def __eq__(self, other): + def __eq__(self, other: object) -> bool: if isinstance(other, AggregatedCellBox): eq_checks = [] diff --git a/meshiphi/mesh_generation/boundary.py b/meshiphi/mesh_generation/boundary.py index 9d1e9e40..93b88321 100644 --- a/meshiphi/mesh_generation/boundary.py +++ b/meshiphi/mesh_generation/boundary.py @@ -1,8 +1,10 @@ -from datetime import datetime -from datetime import timedelta +from __future__ import annotations -from math import cos, sin, asin, sqrt, radians -from shapely import wkt, MultiPolygon +from datetime import datetime, timedelta +from math import asin, cos, radians, sin, sqrt +from typing import Any + +from shapely import MultiPolygon, wkt class Boundary: @@ -23,7 +25,7 @@ class Boundary: """ @classmethod - def from_json(cls, config): + def from_json(cls, config: dict[str, Any]) -> Boundary: """ constructs a boundary object from json input Args: @@ -45,12 +47,10 @@ def from_json(cls, config): else: time_range = None - obj = Boundary(lat_range, long_range, time_range) - - return obj + return Boundary(lat_range, long_range, time_range) @classmethod - def from_poly_string(cls, poly_string): + def from_poly_string(cls, poly_string: str) -> Boundary: """ Creates a Boundary object from a string representation of a polygon. """ @@ -66,9 +66,7 @@ def from_poly_string(cls, poly_string): # pos_coords on +180 side of antimeridian # neg_coords on -180 side of antimeridian - pos_coords, neg_coords = [ - coord_string.split(",") for coord_string in coord_strings - ] + pos_coords, neg_coords = [coord_string.split(",") for coord_string in coord_strings] # Remove leading whitespace from WKT elements pos_coords = [pc.lstrip() for pc in pos_coords] neg_coords = [nc.lstrip() for nc in neg_coords] @@ -103,11 +101,14 @@ def from_poly_string(cls, poly_string): long_range = [long_min, long_max] lat_range = [lat_min, lat_max] - bounds = Boundary(lat_range, long_range) - - return bounds + return Boundary(lat_range, long_range) - def __init__(self, lat_range, long_range, time_range=None): + def __init__( + self, + lat_range: list[float], + long_range: list[float], + time_range: list[str] | None = None, + ) -> None: """ Args: @@ -129,7 +130,7 @@ def __init__(self, lat_range, long_range, time_range=None): self.time_range = time_range @staticmethod - def parse_datetime(datetime_str): + def parse_datetime(datetime_str: str) -> str: """ Attempts to parse a string containing reference to system time into datetime format. If given the string 'TODAY', will return system time. @@ -152,47 +153,52 @@ def parse_datetime(datetime_str): try: datetime_object = datetime.strptime(datetime_str, DATE_FORMAT).date() return datetime_object.strftime(DATE_FORMAT) - except ValueError: + except ValueError as err: # check if datetime_str contains reference to system-time. if datetime_str.strip() == "TODAY": today = datetime.today() return today.strftime(DATE_FORMAT) - elif "TODAY" not in datetime_str: + if "TODAY" not in datetime_str: raise ValueError( f'Incorrect date format given. Cannot convert "{datetime_str}" to date.' - ) + ) from err # check for increment to system time. if "+" in datetime_str: - increment = datetime_str.split("+")[1].strip() + increment_str = datetime_str.split("+")[1].strip() try: - increment = float(increment) + increment = float(increment_str) datetime_object = datetime.today() + timedelta(days=increment) return datetime_object.strftime(DATE_FORMAT) - except ValueError: + except ValueError as err: raise ValueError( f'Incorrect date format given. Cannot convert "{datetime_str}" to date. ' - + f'Time increment "{increment}" cannot be cast to float' - ) + + f'Time increment "{increment_str}" cannot be cast to float' + ) from err # check for decrement to system time. elif "-" in datetime_str: - decrement = datetime_str.split("-")[1].strip() + decrement_str = datetime_str.split("-")[1].strip() try: - decrement = float(decrement) + decrement = float(decrement_str) datetime_object = datetime.today() - timedelta(days=decrement) return datetime_object.strftime(DATE_FORMAT) - except ValueError: + except ValueError as err: raise ValueError( f'Incorrect date format given. Cannot convert "{datetime_str}" to date. ' - + f'Time decrement "{decrement}" cannot be cast to float' - ) + + f'Time decrement "{decrement_str}" cannot be cast to float' + ) from err else: raise ValueError( f'Incorrect date format given. Cannot convert "{datetime_str}" to date' - ) + ) from None - def validate_bounds(self, lat_range, long_range, time_range): + def validate_bounds( + self, + lat_range: list[float], + long_range: list[float], + time_range: list[str], + ) -> None: """ method to check the bounds are valid Args: @@ -214,9 +220,7 @@ def validate_bounds(self, lat_range, long_range, time_range): if datetime.strptime(time_range[0], "%Y-%m-%d") > datetime.strptime( time_range[1], "%Y-%m-%d" ): - raise ValueError( - "Boundary: Start time range should be smaller than range end" - ) + raise ValueError("Boundary: Start time range should be smaller than range end") else: raise ValueError( f"Boundary: Time range needs two dates, instead got {len(time_range)}" @@ -256,8 +260,7 @@ def get_height(self): height (float): the height of the CellBox given in degrees latitude. """ - height = self.lat_range[1] - self.lat_range[0] - return height + return self.lat_range[1] - self.lat_range[0] def get_width(self): """ @@ -274,13 +277,13 @@ def get_width(self): width = (180 - self.long_range[0]) + (self.long_range[1] + 180) return width - def get_time_range(self): + def get_time_range(self) -> list[str]: """ returns the time range """ return self.time_range - def getdcx(self): + def getdcx(self) -> float: """ returns x-distance from the edge to the centroid of the cellbox @@ -288,9 +291,9 @@ def getdcx(self): dcx (float): the x-distance from the edge of the CellBox to the centroid of the CellBox. Given in degrees longitude """ - return self.get_width() / 2 + return float(self.get_width() / 2) - def getdcy(self): + def getdcy(self) -> float: """ returns y-distance from the edge to the centroid of the cellbox @@ -298,39 +301,39 @@ def getdcy(self): dxy (float): the y-distance from the edge of the CellBox to the centroid of the CellBox. Given in degrees latitude """ - return self.get_height() / 2 + return float(self.get_height() / 2) - def get_lat_min(self): + def get_lat_min(self) -> float: """ returns the min latitude """ return self.lat_range[0] - def get_lat_max(self): + def get_lat_max(self) -> float: """ returns the max latitude """ return self.lat_range[1] - def get_long_min(self): + def get_long_min(self) -> float: """ returns the min longtitude """ return self.long_range[0] - def get_long_max(self): + def get_long_max(self) -> float: """ returns the max longtitude """ return self.long_range[1] - def get_time_min(self): + def get_time_min(self) -> str: """ returns the min of time range """ return self.time_range[0] - def get_time_max(self): + def get_time_max(self) -> str: """ returns the max of time range """ @@ -344,7 +347,7 @@ def get_bounds(self): Returns: bounds (list): The geo-spatial boundaries of this CellBox. """ - bounds = [ + return [ [self.long_range[0], self.lat_range[0]], [self.long_range[0], self.lat_range[1]], [self.long_range[1], self.lat_range[1]], @@ -354,7 +357,6 @@ def get_bounds(self): self.lat_range[0], ], ] - return bounds def calc_size(self): """ @@ -461,7 +463,7 @@ def split(self): long_mid = self.get_long_min() + (self.get_width() / 2) # 0 = south_west, 1 = north_west, 2 = south_east, 3 = north_east - bounds = [ + return [ Boundary( [self.get_lat_min(), lat_mid], [self.get_long_min(), long_mid], @@ -484,28 +486,16 @@ def split(self): ), ] - return bounds - def __str__(self): - lat_range = ( - "lat range :[" - + str(self.get_lat_min()) - + "," - + str(self.get_lat_max()) - + "]" - ) + lat_range = "lat range :[" + str(self.get_lat_min()) + "," + str(self.get_lat_max()) + "]" long_range = ( - "long range :[" - + str(self.get_long_min()) - + "," - + str(self.get_long_max()) - + "]" + "long range :[" + str(self.get_long_min()) + "," + str(self.get_long_max()) + "]" ) time_range = "time range :" + str(self.get_time_range()) return "{" + lat_range + ", " + long_range + ", " + time_range + "}" - def __eq__(self, other): + def __eq__(self, other: object) -> bool: """ Evaluates whether two Boundary objects have the same lat/long min/max, and start/end time @@ -516,6 +506,8 @@ def __eq__(self, other): Returns: bool: True if all boundary edges are equal """ + if not isinstance(other, Boundary): + return False # List to store each check for equality boundary_checks = [] # Add bool indicating if spatial boundary matches between the two @@ -524,23 +516,20 @@ def __eq__(self, other): boundary_checks += [(self.get_long_min() == other.get_long_min())] boundary_checks += [(self.get_long_max() == other.get_long_max())] - # Check if obects have the attributes to be tested - has_start_time = [hasattr(self, "start_time"), hasattr(other, "start_time")] - has_end_time = [hasattr(self, "end_time"), hasattr(other, "end_time")] + # Check if objects have time_range to be tested + has_time_range = [ + hasattr(self, "time_range") and self.time_range != [], + hasattr(other, "time_range") and other.time_range != [], + ] - # If they both have the attribute, check that they are equal - if all(has_start_time): - boundary_checks += [(self.get_start_time() == other.get_start_time())] - # If only one has the attribute, they aren't equal - elif any(has_start_time): + # If they both have time_range, check that they are equal + if all(has_time_range): + boundary_checks += [(self.get_time_min() == other.get_time_min())] + boundary_checks += [(self.get_time_max() == other.get_time_max())] + # If only one has time_range, they aren't equal + elif any(has_time_range): boundary_checks += [False] # Otherwise, we don't need to append to the check list - # Same logic, just with other temporal boundary - if all(has_end_time): - boundary_checks += [(self.get_end_time() == other.get_end_time())] - elif any(has_end_time): - boundary_checks += [False] - # See if every check has passed return all(boundary_checks) diff --git a/meshiphi/mesh_generation/cellbox.py b/meshiphi/mesh_generation/cellbox.py index ec7a637d..ce7b8d2e 100644 --- a/meshiphi/mesh_generation/cellbox.py +++ b/meshiphi/mesh_generation/cellbox.py @@ -20,7 +20,7 @@ from __future__ import annotations import logging -from typing import cast +from typing import Any, cast import numpy as np from shapely.geometry import Point @@ -239,6 +239,8 @@ def should_split_breadth_first(self): will result in the CellBox being split. """ hom_conditions = [] + if self.data_source is None: + return False for current_data_source in self.data_source: data_loader = current_data_source.get_data_loader() data_subset = current_data_source.get_data_subset() @@ -339,7 +341,7 @@ def aggregate(self): Returns: AggregatedCellbox: object contains the aggregated data within cellbox bounds. """ - agg_dict = {} + agg_dict: dict[str, Any] = {} for source in self.get_data_source(): loader = source.get_data_loader() data_subset = source.get_data_subset() @@ -422,7 +424,9 @@ def deallocate_cellbox(self): parent = self.parent # free up the memory space used by the parent cellboxes chain while isinstance(parent, CellBox): - for source in self.parent.get_data_source(): + if parent.get_data_source() is None: + break + for source in parent.get_data_source(): loader = source.get_data_loader() del loader del source diff --git a/meshiphi/mesh_generation/metadata.py b/meshiphi/mesh_generation/metadata.py index 9187c610..799124be 100644 --- a/meshiphi/mesh_generation/metadata.py +++ b/meshiphi/mesh_generation/metadata.py @@ -1,3 +1,8 @@ +from __future__ import annotations + +from typing import Any + + class Metadata: """ A Metadata is a class that defines the datasource for a certain Cellbox and the assocated splitting conditions. @@ -13,11 +18,11 @@ class Metadata: def __init__( self, - data_loader, - splitting_conditions=None, - value_fill_type="", - data_subset=None, - ): + data_loader: Any, # DataLoaderInterface or concrete implementations + splitting_conditions: list[dict[str, Any]] | None = None, + value_fill_type: str = "", + data_subset: Any = None, + ) -> None: """ Args: @@ -39,56 +44,56 @@ def __init__( else: self.data_subset = data_subset - def get_data_loader(self): + def get_data_loader(self) -> Any: """ returns the data loader """ return self.data_loader - def set_data_loader(self, data_loader): + def set_data_loader(self, data_loader: Any) -> None: """ sets the data loader """ self.data_loader = data_loader - def get_splitting_conditions(self): + def get_splitting_conditions(self) -> list[dict[str, Any]] | None: """ returns a list of the splitting conditions """ return self.splitting_conditions - def set_splitting_conditions(self, splitting_conditions): + def set_splitting_conditions(self, splitting_conditions: list[dict[str, Any]] | None) -> None: """ sets the splitting conditions """ self.splitting_conditions = splitting_conditions - def get_value_fill_type(self): + def get_value_fill_type(self) -> str: """ returns thevalue fill type """ return self.value_fill_type - def set_value_fill_type(self, value_fill_type): + def set_value_fill_type(self, value_fill_type: str) -> None: """ sets the value fill type """ self.value_fill_type = value_fill_type - def get_data_subset(self): + def get_data_subset(self) -> Any: """ gets the data subset """ return self.data_subset - def set_data_subset(self, data_subset): + def set_data_subset(self, data_subset: Any) -> None: """ sets the data subset """ self.data_subset = data_subset - def __eq__(self, other): + def __eq__(self, other: object) -> bool: if isinstance(other, Metadata): eq_check = [] diff --git a/meshiphi/mesh_generation/neighbour_graph.py b/meshiphi/mesh_generation/neighbour_graph.py index a423d724..d589feb2 100644 --- a/meshiphi/mesh_generation/neighbour_graph.py +++ b/meshiphi/mesh_generation/neighbour_graph.py @@ -1,3 +1,7 @@ +from __future__ import annotations + +from typing import Any + from meshiphi.mesh_generation.direction import Direction @@ -30,16 +34,16 @@ class NeighbourGraph: """ - def __init__(self, cellboxes=None, grid_width=0): + def __init__(self, cellboxes: list[Any] | None = None, grid_width: int = 0) -> None: # initialize graph with an empty one - self.neighbour_graph = {} + self.neighbour_graph: dict[str, Any] = {} if cellboxes is None: cellboxes = [] self.initialise_neighbour_graph(cellboxes, grid_width) self._is_global_mesh = False @classmethod - def from_json(cls, ng_json): + def from_json(cls, ng_json: dict[str, Any]) -> NeighbourGraph: """ method that initializes a graph from a json object Args: @@ -72,7 +76,7 @@ def increment_ids(self, inc): self.neighbour_graph = new_ng - def get_graph(self): + def get_graph(self) -> dict[str, Any]: """ returns the graph dict """ @@ -131,9 +135,7 @@ def remove_node(self, cellbox_index): """ self.neighbour_graph.pop(cellbox_index) - def update_neighbours( - self, cellbox_indx, new_neighbours_indx, direction, cellboxes - ): + def update_neighbours(self, cellbox_indx, new_neighbours_indx, direction, cellboxes): """ method that updates the neighbour of a certain cellbox in a specific direction. It removes cellbox_indx from the neighbour_map of its neighbours in a specific direction and add new_neighbour_indx @@ -180,9 +182,7 @@ def remove_node_from_neighbours(self, cellbox_indx, direction): try: self.neighbour_graph[indx][-1 * direction].remove(cellbox_indx) except KeyError: - self.neighbour_graph[str(indx)][str(-1 * direction)].remove( - int(cellbox_indx) - ) + self.neighbour_graph[str(indx)][str(-1 * direction)].remove(int(cellbox_indx)) def update_corner_neighbours( self, @@ -195,33 +195,25 @@ def update_corner_neighbours( """ method that updates the corner neighbours of cellbox_indx with the given indeces """ - north_east_corner_indx = self.neighbour_graph[cellbox_indx][ - Direction.north_east - ] + north_east_corner_indx = self.neighbour_graph[cellbox_indx][Direction.north_east] if len(north_east_corner_indx) > 0: self.neighbour_graph[north_east_corner_indx[0]][Direction.south_west] = [ north_east_indx ] - north_west_corner_indx = self.neighbour_graph[cellbox_indx][ - Direction.north_west - ] + north_west_corner_indx = self.neighbour_graph[cellbox_indx][Direction.north_west] if len(north_west_corner_indx) > 0: self.neighbour_graph[north_west_corner_indx[0]][Direction.south_east] = [ north_west_indx ] - south_east_corner_indx = self.neighbour_graph[cellbox_indx][ - Direction.south_east - ] + south_east_corner_indx = self.neighbour_graph[cellbox_indx][Direction.south_east] if len(south_east_corner_indx) > 0: self.neighbour_graph[south_east_corner_indx[0]][Direction.north_west] = [ south_east_indx ] - south_west_corner_indx = self.neighbour_graph[cellbox_indx][ - Direction.south_west - ] + south_west_corner_indx = self.neighbour_graph[cellbox_indx][Direction.south_west] if len(south_west_corner_indx) > 0: self.neighbour_graph[south_west_corner_indx[0]][Direction.north_east] = [ south_west_indx @@ -255,9 +247,7 @@ def get_neighbour_case_bounds(self, bounds_a, bounds_b): long_b = bounds_b.get_long_min() lat_b = bounds_b.get_lat_min() - if (long_a + bounds_a.get_width()) == long_b and ( - lat_a + bounds_a.get_height() - ) == lat_b: + if (long_a + bounds_a.get_width()) == long_b and (lat_a + bounds_a.get_height()) == lat_b: return Direction.north_east if ( @@ -267,9 +257,7 @@ def get_neighbour_case_bounds(self, bounds_a, bounds_b): ): return Direction.east - if (long_a + bounds_a.get_width()) == long_b and ( - lat_a == lat_b + bounds_b.get_height() - ): + if (long_a + bounds_a.get_width()) == long_b and (lat_a == lat_b + bounds_b.get_height()): return Direction.south_east if ( @@ -279,9 +267,7 @@ def get_neighbour_case_bounds(self, bounds_a, bounds_b): ): return Direction.south - if long_a == (long_b + bounds_b.get_width()) and lat_a == ( - lat_b + bounds_b.get_height() - ): + if long_a == (long_b + bounds_b.get_width()) and lat_a == (lat_b + bounds_b.get_height()): return Direction.south_west if ( @@ -291,9 +277,7 @@ def get_neighbour_case_bounds(self, bounds_a, bounds_b): ): return Direction.west - if long_a == (long_b + bounds_b.get_width()) and ( - lat_a + bounds_a.get_height() == lat_b - ): + if long_a == (long_b + bounds_b.get_width()) and (lat_a + bounds_a.get_height() == lat_b): return Direction.north_west if ( @@ -346,11 +330,8 @@ def on_global_bound(cellbox_a, cellbox_b): Returns: bool: a boolean representing if the two cellboxes are touching on the global bound (-180,180). """ - return ( - long_a == -180 - and cellbox_b.get_bounds().get_long_max() == 180 - or long_b == -180 - and cellbox_a.get_bounds().get_long_max() == 180 + return (long_a == -180 and cellbox_b.get_bounds().get_long_max() == 180) or ( + long_b == -180 and cellbox_a.get_bounds().get_long_max() == 180 ) if self.is_global_mesh() and on_global_bound(cellbox_a, cellbox_b): @@ -486,17 +467,17 @@ def initialise_neighbour_graph(self, cellboxes, grid_width): """ for cellbox in cellboxes: cellbox_indx = cellboxes.index(cellbox) - neighbour_map = self.initialise_map( - cellbox_indx, grid_width, len(cellboxes) - ) + neighbour_map = self.initialise_map(cellbox_indx, grid_width, len(cellboxes)) self.add_node(cellbox_indx, neighbour_map) - def initialise_map(self, cellbox_indx, grid_width, cellboxes_length): + def initialise_map( + self, cellbox_indx: int, grid_width: int, cellboxes_length: int + ) -> dict[int, list[int]]: """ initialse the neighbour map of a cellbox with the given cellbox_index """ - neighbour_map = { + neighbour_map: dict[int, list[int]] = { Direction.north_east: [], Direction.east: [], Direction.south_east: [], @@ -512,28 +493,20 @@ def initialise_map(self, cellbox_indx, grid_width, cellboxes_length): neighbour_map[Direction.east].append(cellbox_indx + 1) # south-east neighbours if cellbox_indx + grid_width < cellboxes_length: - neighbour_map[Direction.north_east].append( - int((cellbox_indx + grid_width) + 1) - ) + neighbour_map[Direction.north_east].append(int((cellbox_indx + grid_width) + 1)) # north-east neighbours if cellbox_indx - grid_width >= 0: - neighbour_map[Direction.south_east].append( - int((cellbox_indx - grid_width) + 1) - ) + neighbour_map[Direction.south_east].append(int((cellbox_indx - grid_width) + 1)) # add west neighbours to neighbour graph if cellbox_indx % grid_width != 0: neighbour_map[Direction.west].append(cellbox_indx - 1) # add south-west neighbours to neighbour graph if cellbox_indx + grid_width < cellboxes_length: - neighbour_map[Direction.north_west].append( - int((cellbox_indx + grid_width) - 1) - ) + neighbour_map[Direction.north_west].append(int((cellbox_indx + grid_width) - 1)) # add north-west neighbours to neighbour graph if cellbox_indx - grid_width >= 0: - neighbour_map[Direction.south_west].append( - int((cellbox_indx - grid_width) - 1) - ) + neighbour_map[Direction.south_west].append(int((cellbox_indx - grid_width) - 1)) # add south neighbours to neighbour graph if cellbox_indx + grid_width < cellboxes_length: diff --git a/meshiphi/mesh_plotting/mesh_plotter.py b/meshiphi/mesh_plotting/mesh_plotter.py index f08d09ae..487a5a39 100644 --- a/meshiphi/mesh_plotting/mesh_plotter.py +++ b/meshiphi/mesh_plotting/mesh_plotter.py @@ -1,8 +1,13 @@ +from typing import TYPE_CHECKING, cast + import cartopy.crs as ccrs -import shapely import matplotlib import matplotlib.pyplot as plt import pandas as pd +import shapely + +if TYPE_CHECKING: + from cartopy.mpl.geoaxes import GeoAxes class MeshPlotter: @@ -21,15 +26,11 @@ def __init__(self, mesh_json, figscale=10): self.figscale = figscale - self.aspect_ratio = (self.long_max - self.long_min) / ( - self.lat_max - self.lat_min - ) + self.aspect_ratio = (self.long_max - self.long_min) / (self.lat_max - self.lat_min) - self.fig = plt.figure( - figsize=(self.figscale * self.aspect_ratio, self.figscale) - ) + self.fig = plt.figure(figsize=(self.figscale * self.aspect_ratio, self.figscale)) - self.plot = plt.axes(projection=ccrs.PlateCarree()) + self.plot = cast("GeoAxes", plt.axes(projection=ccrs.PlateCarree())) self.plot.set_extent([self.long_min, self.long_max, self.lat_min, self.lat_max]) def plot_bool(self, value_name, colour): diff --git a/meshiphi/mesh_validation/full_mesh_validator.py b/meshiphi/mesh_validation/full_mesh_validator.py index 5e134eec..b6a812c5 100644 --- a/meshiphi/mesh_validation/full_mesh_validator.py +++ b/meshiphi/mesh_validation/full_mesh_validator.py @@ -1,6 +1,7 @@ -from meshiphi.mesh_generation.environment_mesh import EnvironmentMesh import pandas as pd +from meshiphi.mesh_generation.environment_mesh import EnvironmentMesh + class FullMeshValidator: """ @@ -49,12 +50,12 @@ def validate_vector(self, sd_name, md_name): # Get vector value from cellbox cell_vector_0 = "cell_" + md_name[0] cell_vector_1 = "cell_" + md_name[1] - raw_slice[cell_vector_0] = self.mesh.agg_cellboxes[ - cell_count - ].get_agg_data()[md_name[0]] - raw_slice[cell_vector_1] = self.mesh.agg_cellboxes[ - cell_count - ].get_agg_data()[md_name[1]] + raw_slice[cell_vector_0] = self.mesh.agg_cellboxes[cell_count].get_agg_data()[ + md_name[0] + ] + raw_slice[cell_vector_1] = self.mesh.agg_cellboxes[cell_count].get_agg_data()[ + md_name[1] + ] # Calculate difference vector @@ -73,14 +74,12 @@ def validate_vector(self, sd_name, md_name): mean_d_vector_mag = full_raw_processed["d_vector_mag"].mean() rms_d_vector_mag = (full_raw_processed["d_vector_mag^2"].mean()) ** 0.5 - validation_results = { + return { "cellbox_count": len(self.mesh.agg_cellboxes), "mean_d_vector_mag": mean_d_vector_mag, "root_mean_squared_d_vector_mag": rms_d_vector_mag, } - return validation_results - def validate_scalar(self, sd_name, md_name): """ Validate the mesh by comparing the scalar field in the mesh to the source data @@ -100,9 +99,7 @@ def validate_scalar(self, sd_name, md_name): # Get scalar value from cellbox cell_scalar = "cell_" + md_name - raw_slice[cell_scalar] = self.mesh.agg_cellboxes[cell_count].get_agg_data()[ - md_name - ] + raw_slice[cell_scalar] = self.mesh.agg_cellboxes[cell_count].get_agg_data()[md_name] # Calculate difference scalar raw_slice["d_scalar"] = abs(raw_slice[sd_name] - raw_slice[cell_scalar]) @@ -113,14 +110,12 @@ def validate_scalar(self, sd_name, md_name): mean_err = full_raw_processed["d_scalar"].mean() rmse = (full_raw_processed["d_scalar^2"].mean()) ** 0.5 - validation_results = { + return { "cellbox_count": len(self.mesh.agg_cellboxes), "mean_err": mean_err, "root_mean_squared_err": rmse, } - return validation_results - def slice_data(self, bounds): """ return a slice of the source data within the bounds @@ -134,8 +129,4 @@ def slice_data(self, bounds): raw_slice = self.source_data[ self.source_data["lat"].between(bounds.lat_range[0], bounds.lat_range[1]) ] - raw_slice = raw_slice[ - raw_slice["long"].between(bounds.long_range[0], bounds.long_range[1]) - ] - - return raw_slice + return raw_slice[raw_slice["long"].between(bounds.long_range[0], bounds.long_range[1])] diff --git a/meshiphi/mesh_validation/mesh_comparator.py b/meshiphi/mesh_validation/mesh_comparator.py index 5ccfdf10..b3142629 100644 --- a/meshiphi/mesh_validation/mesh_comparator.py +++ b/meshiphi/mesh_validation/mesh_comparator.py @@ -1,4 +1,5 @@ import pandas as pd + from meshiphi.utils import round_to_sigfig @@ -25,10 +26,11 @@ def _return_as(df, return_type): # Parse return_type to provide correct output if return_type in ["cellboxes", "cb"]: return df["cellboxes"] - elif return_type in ["id"]: + if return_type in ["id"]: return df["id"] - elif return_type in ["dataframe", "df"]: + if return_type in ["dataframe", "df"]: return df + return None def compare_cellbox_boundaries(self, mesh1, mesh2, return_type=None): """ @@ -107,11 +109,9 @@ def compare_cellbox_values(self, mesh1, mesh2, return_type=None): # Round all float values in lists to significant figure list_cols = df.select_dtypes(include=list).columns for col in list_cols: - round_col = list() + round_col = [] for val in df[col]: - if isinstance(val, list) and all( - [isinstance(x, float) for x in val] - ): + if isinstance(val, list) and all(isinstance(x, float) for x in val): round_col.append(round_to_sigfig(val, self.SIG_FIG_TOLERANCE)) else: round_col.append(val) @@ -151,11 +151,10 @@ def compare_cellbox_attributes(self, mesh1, mesh2, return_type=None): for cb1_geom, cb1_series in cellboxes1.items(): # If not matching geometries, skip - if cb1_geom not in cellboxes2.keys(): + if cb1_geom not in cellboxes2: continue # Otherwise, extract other cellbox to compare - else: - cb2_series = cellboxes2[cb1_geom] + cb2_series = cellboxes2[cb1_geom] # If attributes not the same between the two cellboxes with similar boundary if not cb1_series.index.equals(cb2_series.index): @@ -192,17 +191,13 @@ def compare_neighbour_graph_values(self, mesh1, mesh2, return_type=None): mismatched_rows = [] - for node in ng_2.keys(): - if node in ng_1.keys(): + for node in ng_2: + if node in ng_1: neighbours_1 = ng_1[node] neighbours_2 = ng_2[node] - sorted_neighbours_1 = { - k: sorted(neighbours_1[k]) for k in neighbours_1.keys() - } - sorted_neighbours_2 = { - k: sorted(neighbours_2[k]) for k in neighbours_2.keys() - } + sorted_neighbours_1 = {k: sorted(neighbours_1[k]) for k in neighbours_1} + sorted_neighbours_2 = {k: sorted(neighbours_2[k]) for k in neighbours_2} if sorted_neighbours_1 != sorted_neighbours_2: cellbox_row = cellboxes2.loc[cellboxes2["id"] == node] diff --git a/meshiphi/mesh_validation/mesh_validator.py b/meshiphi/mesh_validation/mesh_validator.py index a0bc5ba3..8d0a9d09 100644 --- a/meshiphi/mesh_validation/mesh_validator.py +++ b/meshiphi/mesh_validation/mesh_validator.py @@ -1,14 +1,17 @@ -from meshiphi.mesh_validation.sampler import Sampler - +from __future__ import annotations -from meshiphi.mesh_generation.mesh_builder import MeshBuilder -import numpy as np import json -import math import logging -from meshiphi.mesh_generation.boundary import Boundary -from sklearn.metrics import mean_squared_error +import math +from typing import Any + +import numpy as np import xarray as xr +from sklearn.metrics import mean_squared_error + +from meshiphi.mesh_generation.boundary import Boundary +from meshiphi.mesh_generation.mesh_builder import MeshBuilder +from meshiphi.mesh_validation.sampler import Sampler logger = logging.getLogger(__name__) @@ -33,7 +36,7 @@ def __init__(self, mesh_config_file): """ self.conf = None self.validation_length = 0.1 - with open(mesh_config_file, "r") as config_file: + with open(mesh_config_file) as config_file: self.conf = json.load(config_file)["config"]["mesh_info"] mesh_builder = MeshBuilder(self.conf) @@ -66,8 +69,7 @@ def validate_mesh(self, number_of_samples=10): actual_value = np.append(actual_value, self.get_value_from_data(sample)) mesh_value = np.append(mesh_value, self.get_values_from_mesh(sample)) # calculate the RMSE over the samples. - distance = math.sqrt(mean_squared_error(actual_value, mesh_value)) - return distance + return math.sqrt(mean_squared_error(actual_value, mesh_value)) def get_value_from_data(self, sample): """ @@ -77,7 +79,7 @@ def get_value_from_data(self, sample): Returns: a numpy array that contains all the data within the sampled lat and long range """ - values = [] + values: list[Any] = [] # calculate the sampling range based on the validation length lat_end, long_end = self.get_range_end(sample) lat_range = [sample[0], lat_end] @@ -86,10 +88,9 @@ def get_value_from_data(self, sample): for source in self.mesh.cellboxes[0].get_data_source(): data_loader = source.get_data_loader() data_name = data_loader.get_data_col_name() - dp = data_loader.trim_datapoints( - Boundary(lat_range, long_range, time_range) - ) - values = np.append(values, dp[data_name]) + dp = data_loader.trim_datapoints(Boundary(lat_range, long_range, time_range)) + values_array = np.append(values, dp[data_name]) + values = values_array.tolist() logger.info("values from data are: {}".format(" ".join(map(str, values)))) return values @@ -119,7 +120,7 @@ def get_values_from_mesh(self, sample): a numpy array that contains the mesh's data within the sampled lat and long range """ # TODO make sure to handle the vector data - values = [] + values: list[Any] = [] # calculate the sampling range based on the validation length lat_end, long_end = self.get_range_end(sample) lat_range = [sample[0], lat_end] @@ -128,20 +129,19 @@ def get_values_from_mesh(self, sample): for source in self.mesh.cellboxes[0].get_data_source(): data_loader = source.get_data_loader() - dp = data_loader.trim_datapoints( - Boundary(lat_range, long_range, time_range) - ) + dp = data_loader.trim_datapoints(Boundary(lat_range, long_range, time_range)) if isinstance(dp, xr.core.dataset.Dataset): dp = dp.to_dataframe().reset_index() - for index, point in dp.iterrows(): + for _index, point in dp.iterrows(): lat = point["lat"] long = point["long"] for agg_cellbox in self.env_mesh.agg_cellboxes: if agg_cellbox.contains_point(lat, long): - values = np.append( + values_array = np.append( values, agg_cellbox.agg_data[data_loader.data_name] ) # get the agg_value + values = values_array.tolist() break # break to make sure we avoid getting multiple values (for lat and long on bounds of multiple cellboxes) logger.info("values from mesh are: {}".format(" ".join(map(str, values)))) diff --git a/meshiphi/mesh_validation/sampler.py b/meshiphi/mesh_validation/sampler.py index 40d37255..f4f70cfd 100644 --- a/meshiphi/mesh_validation/sampler.py +++ b/meshiphi/mesh_validation/sampler.py @@ -1,5 +1,7 @@ -from scipy.stats import qmc +from __future__ import annotations + import numpy as np +from scipy.stats import qmc class Sampler: @@ -36,7 +38,5 @@ def generate_samples(self, ranges): # map samples to ranges for sample in samples: for i in range(len(ranges)): - mapped_samples.append( - ranges[i][0] + sample[i] * (ranges[i][1] - ranges[i][0]) - ) + mapped_samples.append(ranges[i][0] + sample[i] * (ranges[i][1] - ranges[i][0])) return np.array(mapped_samples).reshape(len(samples), len(ranges)) diff --git a/meshiphi/test_automation/test_automater.py b/meshiphi/test_automation/test_automater.py index dc9cd1b5..ad7accc9 100644 --- a/meshiphi/test_automation/test_automater.py +++ b/meshiphi/test_automation/test_automater.py @@ -1,20 +1,21 @@ -import tempfile -import os -import shutil +from __future__ import annotations + import json import logging +import os +import shutil import subprocess as sp -import pandas as pd -import geopandas as gpd -from shapely import wkt - +import tempfile +import geopandas as gpd import matplotlib.pyplot as plt -from matplotlib.patches import Patch +import pandas as pd from matplotlib.colors import to_rgba +from matplotlib.patches import Patch +from shapely import wkt -from meshiphi.mesh_validation.mesh_comparator import MeshComparator from meshiphi import REGRESSION_TESTS_BY_FILE, UNIT_TESTS_BY_FILE +from meshiphi.mesh_validation.mesh_comparator import MeshComparator logger = logging.getLogger(__name__) @@ -60,14 +61,12 @@ def __init__( self._double_separator = "=" * remaining_terminal_width self._single_separator = "-" * remaining_terminal_width # Initialise arrays with each test, organise by status - self.passes = [] - self.fails = [] - self.errors = [] + self.passes: list[TestInfo] = [] + self.fails: list[TestInfo] = [] + self.errors: list[TestInfo] = [] # Get files that are different between branches - diff_files = self.get_diff_filenames( - from_branch=from_branch, into_branch=into_branch - ) + diff_files = self.get_diff_filenames(from_branch=from_branch, into_branch=into_branch) # Run relevant tests logger.info(self._double_separator) @@ -159,9 +158,7 @@ def run_regression_tests(self, diff_files, save_to=None, plot=False): # Get base directory for regression tests reg_test_dir = os.path.join(self.repo_dir, "tests", "regression_tests") logger.info("Attempting regression tests...") - self._run_tests( - diff_files, reg_test_dir, REGRESSION_TESTS_BY_FILE, save_to=save_to - ) + self._run_tests(diff_files, reg_test_dir, REGRESSION_TESTS_BY_FILE, save_to=save_to) # Summarise mesh stats of regression tests if save_to: @@ -232,9 +229,9 @@ def parse_pytest_stdout(stdout): errors = [] # Get index of line in stdout that contains short summary info stdout_lines = stdout.split("\n") - summary_idx = [ - idx for idx, s in enumerate(stdout_lines) if "short test summary info" in s - ][0] + summary_idx = next( + (idx for idx, s in enumerate(stdout_lines) if "short test summary info" in s), 0 + ) # Iterate through pytest summary output for line in stdout_lines[summary_idx:]: # Only read the lines with all necessary info @@ -306,8 +303,8 @@ def get_diff_filenames(from_branch=None, into_branch=None): command += ["--merge-base", from_branch, into_branch] # Run git diff - git_diff = sp.run(command, stdout=sp.PIPE) - git_diff = git_diff.stdout.decode("utf-8") + git_diff_result = sp.run(command, stdout=sp.PIPE, check=False) + git_diff = git_diff_result.stdout.decode("utf-8") # Extract list of files that have been modified raw_filenames = git_diff.split("\n") @@ -315,13 +312,11 @@ def get_diff_filenames(from_branch=None, into_branch=None): diff_files = [] if from_branch: logger.info( - "Following files different between " - + f'"{from_branch}" and "{into_branch}"' + "Following files different between " + f'"{from_branch}" and "{into_branch}"' ) else: logger.info( - "Following files different between " - + f'current branch and "{into_branch}"' + "Following files different between " + f'current branch and "{into_branch}"' ) for filename in raw_filenames: @@ -369,7 +364,7 @@ def extract_test_meshes(test_output_file): dict: Ground truth 'old' json mesh dict: Updated 'new' json mesh """ - with open(test_output_file, "r") as fp: + with open(test_output_file) as fp: test_json = json.load(fp) old_json = test_json["old_mesh"] new_json = test_json["new_mesh"] @@ -393,7 +388,7 @@ def compare_meshes(self, old_json, new_json): mc = MeshComparator() - mesh_comparison = { + return { "new_mesh": pd.DataFrame(new_json["cellboxes"]).set_index("geometry"), "bounds": mc.compare_cellbox_boundaries(old_json, new_json), "values": mc.compare_cellbox_values(old_json, new_json), @@ -401,8 +396,6 @@ def compare_meshes(self, old_json, new_json): "neighbour_graph": mc.compare_neighbour_graph_values(old_json, new_json), } - return mesh_comparison - def save_tests(self, tmp_dir, output_folder, passes=False, fails=True, errors=True): """ Saves copy of newly generated test meshes to 'pytest_meshiphi' folder @@ -437,9 +430,7 @@ def save_tests(self, tmp_dir, output_folder, passes=False, fails=True, errors=Tr # Skip over non-json files (i.e. plots if generated, pytest subdirectories) if extension != ".json": continue - old_json, new_json = self.extract_test_meshes( - pytest_output_basename + ".json" - ) + old_json, new_json = self.extract_test_meshes(pytest_output_basename + ".json") comparison = self.compare_meshes(old_json, new_json) # Remove full new mesh from comparison dict del comparison["new_mesh"] @@ -460,7 +451,7 @@ def save_tests(self, tmp_dir, output_folder, passes=False, fails=True, errors=Tr # Try / Except in case plotting not done try: shutil.copyfile(pytest_output_basename + ".svg", plot_filename) - except IOError as e: + except OSError as e: logger.debug(e) def plot_test(self, test_output, save_to=None): @@ -515,12 +506,10 @@ def add_df_to_ax(df, ax, c="black", ids=False, label=None, a=0.2): legend_entry = Patch(facecolor=to_rgba(c, a), edgecolor=c, label=label) # Set a column to have roughly the centrepoint of each cellbox - gdf["coords"] = gdf["geometry"].apply( - lambda x: x.representative_point().coords[:][0] - ) + gdf["coords"] = gdf["geometry"].apply(lambda x: x.representative_point().coords[:][0]) if ids: # Print cellbox id within the cellbox - for idx, row in gdf.iterrows(): + for _idx, row in gdf.iterrows(): # Scale cell ID label to fit nicely within cellbox fontsize = 2 * row["dcx"] * (2 ** row["dcx"]) ax.annotate( @@ -615,8 +604,7 @@ def print_summary(comparison, summary_key): num_diff_cbs = len(diff_df.index) # Write to terminal logger.info( - f"\t{num_diff_cbs}/{num_new_cbs} are different in the " - "newly generated mesh" + f"\t{num_diff_cbs}/{num_new_cbs} are different in the newly generated mesh" ) logger.debug( "\tDifferent cellboxes have the following id's in the " @@ -640,9 +628,9 @@ def summarise_test_stats(self): # Out of every test run all_tests = self.passes + self.fails + self.errors # Get list of unique files (i.e. unique test sets) - diff_test_files = set([ti.file for ti in all_tests]) + diff_test_files = {ti.file for ti in all_tests} # Set up empty array to store status for calculating stats - status_by_file = {test: [] for test in diff_test_files} + status_by_file: dict[str, list[str]] = {test: [] for test in diff_test_files} # Append status to each unique test set for test in all_tests: status_by_file[test.file] += [test.status] @@ -653,7 +641,7 @@ def summarise_test_stats(self): class TestInfo: - def __init__(self, file, test, reference, status): + def __init__(self, file: str, test: str, reference: str, status: str) -> None: self.file = file self.test = test self.reference = reference diff --git a/meshiphi/utils.py b/meshiphi/utils.py index 5388bd4b..9782edf7 100644 --- a/meshiphi/utils.py +++ b/meshiphi/utils.py @@ -5,11 +5,16 @@ import logging import time import tracemalloc -import numpy as np +from calendar import monthrange from datetime import datetime, timedelta from functools import wraps -from calendar import monthrange -from scipy.fftpack import fftshift + +import numpy as np + +try: + from scipy.fft import fftshift +except ImportError: + from scipy.fftpack import fftshift logger = logging.getLogger(__name__) @@ -21,14 +26,13 @@ def longitude_domain(long): # Allow input type to be list or ndarray if isinstance(long, list): return [longitude_domain(x) for x in long] - elif isinstance(long, np.ndarray): + if isinstance(long, np.ndarray): return np.array([longitude_domain(x) for x in long]) # Return same format as input at antimeridian - elif long in [-180, 180]: + if long in [-180, 180]: return long # Otherwise convert it to be within domain - else: - return (long + 180) % 360 - 180 + return (long + 180) % 360 - 180 def longitude_distance(long_a, long_b): @@ -41,8 +45,7 @@ def longitude_distance(long_a, long_b): if long_dist > 180: return 360 - long_dist - else: - return long_dist + return long_dist def frac_of_month(year, month, start_date=None, end_date=None): @@ -101,10 +104,7 @@ def convert_decimal_days(decimal_days, mins=False): new_time = f"{minutes} minutes" else: hours = round(hours, 2) - if days: - new_time = f"{round(days)} days {hours} hours" - else: - new_time = f"{hours} hours" + new_time = f"{round(days)} days {hours} hours" if days else f"{hours} hours" return new_time @@ -141,9 +141,7 @@ def round_to_sigfig(x, sigfig=5): # np.array.astype(int) to enable np.around to work later dec_pl = ( sigfig - - np.floor( - np.log10(np.abs(x), where=loggable_idxs, out=np.zeros_like(x)) - ).astype(int) + - np.floor(np.log10(np.abs(x), where=loggable_idxs, out=np.zeros_like(x))).astype(int) - 1 ) # Round to sig figs @@ -152,11 +150,10 @@ def round_to_sigfig(x, sigfig=5): if orig_type in [int, float]: return rounded.item() # Return as python list - elif orig_type is list: + if orig_type is list: return rounded.tolist() # Otherwise, return np.array - else: - return rounded + return rounded def divergence(flow): @@ -172,8 +169,7 @@ def curl(flow): Fx, Fy = flow[:, :, 0], flow[:, :, 1] dFx_dy = np.gradient(Fx, axis=1) dFy_dx = np.gradient(Fy, axis=0) - curl = dFy_dx - dFx_dy - return curl + return dFy_dx - dFx_dy # GRF functions @@ -195,8 +191,7 @@ def fftind(size): # Create array k_ind = np.mgrid[:size, :size] - int((size + 1) / 2) # Fourier shift - k_ind = fftshift(k_ind) - return k_ind + return fftshift(k_ind) def gaussian_random_field(size, alpha): @@ -226,18 +221,14 @@ def gaussian_random_field(size, alpha): # Draws a complex gaussian random noise with normal # (circular) distribution - noise = np.random.normal(size=(size, size)) + 1j * np.random.normal( - size=(size, size) - ) + noise = np.random.normal(size=(size, size)) + 1j * np.random.normal(size=(size, size)) # To real space grf = np.fft.ifft2(noise * amplitude).real # Normalise the GRF: grf = grf - np.min(grf) - grf = grf / (np.max(grf) - np.min(grf)) - - return grf + return grf / (np.max(grf) - np.min(grf)) def memory_trace(func): @@ -249,7 +240,7 @@ def wrapper(*args, **kwargs): top_stats = snapshot.statistics("traceback") stat = top_stats[0] - logging.info("{} memory blocks: {.1f} KiB".format(stat.count, stat.size / 1024)) + logging.info(f"{stat.count} memory blocks: {stat.size / 1024:.1f} KiB") logging.info("\n".join(stat.traceback.format())) return res @@ -262,9 +253,7 @@ def wrapper(*args, **kwargs): start = time.perf_counter() res = func(*args, **kwargs) end = time.perf_counter() - logger.info( - "Timed call to {} took {:02f} seconds".format(func.__name__, end - start) - ) + logger.info(f"Timed call to {func.__name__} took {end - start:02f} seconds") return res return wrapper diff --git a/pyproject.toml b/pyproject.toml index e87e291d..f84903b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -183,7 +183,7 @@ strict_equality = true show_error_codes = true follow_imports = "normal" -# Ignore missing imports for third-party libraries without type stubs +# Ignore missing imports and untyped libraries for third-party libraries without type stubs [[tool.mypy.overrides]] module = [ "osgeo.*", @@ -196,18 +196,16 @@ module = [ "matplotlib.*", "pandas.*", "numpy.*", + "sklearn.*", + "scipy.*", + "jsonschema", + "jsonschema.*", + "tqdm", + "tqdm.*", ] ignore_missing_imports = true -# Allow dynamic attributes for classes that use setattr (set via params dict) -[[tool.mypy.overrides]] -module = "meshiphi.dataloaders.lut.abstract_lut" -ignore_errors = true - -[[tool.mypy.overrides]] -module = "meshiphi.dataloaders.scalar.abstract_scalar" -ignore_errors = true - +# Disable disallow_untyped_calls for the config validation module which uses jsonschema [[tool.mypy.overrides]] -module = "meshiphi.dataloaders.vector.abstract_vector" -ignore_errors = true +module = "meshiphi.config_validation.*" +disallow_untyped_calls = false diff --git a/tests/conftest.py b/tests/conftest.py index f9d94f90..e7c4eee6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,20 +1,23 @@ """ Shared pytest configuration and constants for MeshiPhi tests. """ + +import copy import json +import logging import os import time -import logging -import copy from pathlib import Path + import pytest + import meshiphi -from meshiphi.mesh_generation.metadata import Metadata from meshiphi.dataloaders.factory import DataLoaderFactory -from meshiphi.mesh_generation.cellbox import CellBox +from meshiphi.mesh_generation.aggregated_cellbox import AggregatedCellBox from meshiphi.mesh_generation.boundary import Boundary +from meshiphi.mesh_generation.cellbox import CellBox +from meshiphi.mesh_generation.metadata import Metadata from meshiphi.mesh_generation.neighbour_graph import NeighbourGraph -from meshiphi.mesh_generation.aggregated_cellbox import AggregatedCellBox # Logger for test execution LOGGER = logging.getLogger(__name__) @@ -25,33 +28,35 @@ # Test directory paths TESTS_DIR = Path(__file__).parent -REGRESSION_TESTS_DIR = TESTS_DIR / 'regression_tests' -UNIT_TESTS_DIR = TESTS_DIR / 'unit_tests' +REGRESSION_TESTS_DIR = TESTS_DIR / "regression_tests" +UNIT_TESTS_DIR = TESTS_DIR / "unit_tests" # File locations of all meshes to be recalculated for regression testing TEST_ENV_MESHES = [ - 'grf_normal.json', - 'grf_downsample.json', - 'grf_reprojection.json', - 'grf_sparse.json' + "grf_normal.json", + "grf_downsample.json", + "grf_reprojection.json", + "grf_sparse.json", ] TEST_ABSTRACT_MESHES = [ - 'vgrad.json', - 'hgrad.json', - 'checkerboard_1.json', - 'checkerboard_2.json', - 'checkerboard_3.json', - 'circle.json', - 'circle_quadrant_split.json', - 'circle_quadrant_nosplit.json' + "vgrad.json", + "hgrad.json", + "checkerboard_1.json", + "checkerboard_2.json", + "checkerboard_3.json", + "circle.json", + "circle_quadrant_split.json", + "circle_quadrant_nosplit.json", ] # Build full paths -ALL_TEST_MESHES = ( - [str(REGRESSION_TESTS_DIR / 'example_meshes/env_meshes' / mesh) for mesh in TEST_ENV_MESHES] + - [str(REGRESSION_TESTS_DIR / 'example_meshes/abstract_env_meshes' / mesh) for mesh in TEST_ABSTRACT_MESHES] -) +ALL_TEST_MESHES = [ + str(REGRESSION_TESTS_DIR / "example_meshes/env_meshes" / mesh) for mesh in TEST_ENV_MESHES +] + [ + str(REGRESSION_TESTS_DIR / "example_meshes/abstract_env_meshes" / mesh) + for mesh in TEST_ABSTRACT_MESHES +] # Unit test helper functions @@ -76,78 +81,73 @@ def create_cellbox(bounds, id=0, parent=None, params=None, splitting_conds=None, new_cellbox = CellBox(bounds, id) new_cellbox.data_source = [metadata] new_cellbox.parent = parent - + return new_cellbox def create_dataloader(bounds, params=None, min_dp=5): """ Create a dataloader for testing. - + Args: bounds (Boundary): Boundary for the dataloader params (dict, optional): Dataloader parameters. Defaults to None. min_dp (int, optional): Minimum datapoints. Defaults to 5. - + Returns: DataLoader: Configured dataloader instance """ if params is None: params = { - 'dataloader_name': 'rectangle', - 'data_name': 'dummy_data', - 'width': bounds.get_width()/4, - 'height': bounds.get_height()/4, - 'centre': (bounds.getcx(), bounds.getcy()), - 'nx': 15, - 'ny': 15, + "dataloader_name": "rectangle", + "data_name": "dummy_data", + "width": bounds.get_width() / 4, + "height": bounds.get_height() / 4, + "centre": (bounds.getcx(), bounds.getcy()), + "nx": 15, + "ny": 15, "aggregate_type": "MEAN", - "value_fill_type": 'parent' + "value_fill_type": "parent", } - dataloader = DataLoaderFactory().get_dataloader(params['dataloader_name'], - bounds, - params, - min_dp=min_dp) - return dataloader + return DataLoaderFactory().get_dataloader( + params["dataloader_name"], bounds, params, min_dp=min_dp + ) def create_metadata(bounds, dataloader, splitting_conds=None): """ Create metadata for testing. - + Args: bounds (Boundary): Boundary for the metadata dataloader (DataLoader): Dataloader instance splitting_conds (list, optional): Splitting conditions. Defaults to None. - + Returns: Metadata: Configured metadata instance """ if splitting_conds is None: - splitting_conds = [{ - 'threshold': 0.5, - 'upper_bound': 0.75, - 'lower_bound': 0.25 - }] - data_source = Metadata(dataloader, - splitting_conditions=splitting_conds, - value_fill_type='parent', - data_subset=dataloader.trim_datapoints(bounds)) - return data_source + splitting_conds = [{"threshold": 0.5, "upper_bound": 0.75, "lower_bound": 0.25}] + return Metadata( + dataloader, + splitting_conditions=splitting_conds, + value_fill_type="parent", + data_subset=dataloader.trim_datapoints(bounds), + ) def compare_cellbox_lists(s, t): """ Compare two lists of cellboxes for equality (order-independent). - + Args: s (list): First list of cellboxes t (list): Second list of cellboxes - + Returns: bool: True if lists contain same elements, False otherwise """ - t = list(t) # make a mutable copy + t = list(t) # make a mutable copy try: for elem in s: t.remove(elem) @@ -164,7 +164,7 @@ def json_dict_to_file(json_dict, filename): json_dict (dict): Dict to write to JSON filename (str): Path to file being written """ - with open(filename, 'w') as fp: + with open(filename, "w") as fp: json.dump(json_dict, fp, indent=4) @@ -178,19 +178,18 @@ def file_to_json_dict(filename): Returns: dict: Dictionary with JSON contents """ - with open(filename, 'r') as fp: - json_dict = json.load(fp) - return json_dict + with open(filename) as fp: + return json.load(fp) def create_ng_from_dict(ng_dict, global_mesh=False): """ Create a NeighbourGraph from a dictionary. - + Args: ng_dict (dict): Dictionary representation of neighbour graph global_mesh (bool, optional): Whether this is a global mesh. Defaults to False. - + Returns: NeighbourGraph: Configured neighbour graph instance """ @@ -217,19 +216,19 @@ def calculate_env_mesh(mesh_config): new_mesh = mesh_builder.build_environmental_mesh() end = time.perf_counter() - + cellbox_count = len(new_mesh.agg_cellboxes) elapsed_time = end - start - LOGGER.info(f'Mesh containing {cellbox_count} cellboxes built in {elapsed_time:.2f} seconds') + LOGGER.info(f"Mesh containing {cellbox_count} cellboxes built in {elapsed_time:.2f} seconds") return new_mesh.to_json() -@pytest.fixture(scope='session', autouse=False, params=ALL_TEST_MESHES) +@pytest.fixture(scope="session", autouse=False, params=ALL_TEST_MESHES) def mesh_pair(request): """ Creates a pair of JSON objects: one newly generated, one as old reference. - + Args: request (fixture): Pytest fixture object including list of meshes to regenerate @@ -237,21 +236,17 @@ def mesh_pair(request): dict: Dictionary containing test name, old mesh, and newly generated mesh """ mesh_path = request.param - LOGGER.info(f'Test File: {mesh_path}') - - with open(mesh_path, 'r') as fp: + LOGGER.info(f"Test File: {mesh_path}") + + with open(mesh_path) as fp: old_mesh = json.load(fp) - - mesh_config = old_mesh['config']['mesh_info'] + + mesh_config = old_mesh["config"]["mesh_info"] new_mesh = calculate_env_mesh(mesh_config) - + test_name = os.path.basename(mesh_path) - return { - "test": test_name, - "old_mesh": old_mesh, - "new_mesh": new_mesh - } + return {"test": test_name, "old_mesh": old_mesh, "new_mesh": new_mesh} # Unit test fixtures @@ -264,7 +259,7 @@ def arbitrary_boundary(): @pytest.fixture def temporal_boundary(): """Boundary with temporal component for testing.""" - return Boundary([10, 20], [30, 40], ['1970-01-01', '2021-12-31']) + return Boundary([10, 20], [30, 40], ["1970-01-01", "2021-12-31"]) @pytest.fixture @@ -294,33 +289,33 @@ def dummy_cellbox(arbitrary_boundary): @pytest.fixture def dummy_agg_cellbox(): """Basic aggregated cellbox for testing.""" - arbitrary_agg_data = {'dummy_data': 1} - return AggregatedCellBox(Boundary([45, 60], [45, 60]), arbitrary_agg_data, '0') + arbitrary_agg_data = {"dummy_data": 1} + return AggregatedCellBox(Boundary([45, 60], [45, 60]), arbitrary_agg_data, "0") @pytest.fixture def arbitrary_agg_cellbox(): """Arbitrary aggregated cellbox for testing.""" - arbitrary_agg_data = {'dummy_data': 1} - return AggregatedCellBox(Boundary([45, 60], [45, 60]), arbitrary_agg_data, '1') + arbitrary_agg_data = {"dummy_data": 1} + return AggregatedCellBox(Boundary([45, 60], [45, 60]), arbitrary_agg_data, "1") @pytest.fixture def equatorial_agg_cellbox(): """Aggregated cellbox crossing equator.""" - arbitrary_agg_data = {'dummy_data': 1} - return AggregatedCellBox(Boundary([-10, 10], [45, 60]), arbitrary_agg_data, '2') + arbitrary_agg_data = {"dummy_data": 1} + return AggregatedCellBox(Boundary([-10, 10], [45, 60]), arbitrary_agg_data, "2") @pytest.fixture def meridian_agg_cellbox(): """Aggregated cellbox crossing prime meridian.""" - arbitrary_agg_data = {'dummy_data': 1} - return AggregatedCellBox(Boundary([45, 60], [-10, 10]), arbitrary_agg_data, '3') + arbitrary_agg_data = {"dummy_data": 1} + return AggregatedCellBox(Boundary([45, 60], [-10, 10]), arbitrary_agg_data, "3") @pytest.fixture def antimeridian_agg_cellbox(): """Aggregated cellbox crossing antimeridian.""" - arbitrary_agg_data = {'dummy_data': 1} - return AggregatedCellBox(Boundary([45, 60], [170, -170]), arbitrary_agg_data, '4') + arbitrary_agg_data = {"dummy_data": 1} + return AggregatedCellBox(Boundary([45, 60], [170, -170]), arbitrary_agg_data, "4") diff --git a/tests/regression_tests/test_comparisons.py b/tests/regression_tests/test_comparisons.py index d5911eca..38540a08 100644 --- a/tests/regression_tests/test_comparisons.py +++ b/tests/regression_tests/test_comparisons.py @@ -8,10 +8,7 @@ import pandas as pd import pytest -from .utils import ( - round_dataframe_values, - extract_common_boundaries_for_comparison -) +from .utils import extract_common_boundaries_for_comparison, round_dataframe_values # Apply markers to all tests in this module pytestmark = pytest.mark.slow @@ -19,6 +16,7 @@ # Cellbox comparison test functions + def test_mesh_cellbox_count(mesh_pair): """ Test that cellbox count is preserved between mesh versions. @@ -31,15 +29,16 @@ def test_mesh_cellbox_count(mesh_pair): """ mesh_a = mesh_pair["old_mesh"] mesh_b = mesh_pair["new_mesh"] - - regression_mesh = mesh_a['cellboxes'] - new_mesh = mesh_b['cellboxes'] + + regression_mesh = mesh_a["cellboxes"] + new_mesh = mesh_b["cellboxes"] cellbox_count_a = len(regression_mesh) cellbox_count_b = len(new_mesh) - assert cellbox_count_a == cellbox_count_b, \ + assert cellbox_count_a == cellbox_count_b, ( f"Incorrect number of cellboxes in new mesh. Expected: {cellbox_count_a}, got: {cellbox_count_b}" + ) def test_mesh_cellbox_ids(mesh_pair): @@ -54,12 +53,12 @@ def test_mesh_cellbox_ids(mesh_pair): """ mesh_a = mesh_pair["old_mesh"] mesh_b = mesh_pair["new_mesh"] - - regression_mesh = mesh_a['cellboxes'] - new_mesh = mesh_b['cellboxes'] - indxed_a = {cellbox['id']: cellbox for cellbox in regression_mesh} - indxed_b = {cellbox['id']: cellbox for cellbox in new_mesh} + regression_mesh = mesh_a["cellboxes"] + new_mesh = mesh_b["cellboxes"] + + indxed_a = {cellbox["id"]: cellbox for cellbox in regression_mesh} + indxed_b = {cellbox["id"]: cellbox for cellbox in new_mesh} regression_mesh_ids = set(indxed_a.keys()) new_mesh_ids = set(indxed_b.keys()) @@ -67,9 +66,10 @@ def test_mesh_cellbox_ids(mesh_pair): missing_a_ids = list(new_mesh_ids - regression_mesh_ids) missing_b_ids = list(regression_mesh_ids - new_mesh_ids) - assert indxed_a.keys() == indxed_b.keys(), \ - f"Mismatch in cellbox IDs. ID's {missing_a_ids} have appeared in the new mesh. " \ + assert set(indxed_a) == set(indxed_b), ( + f"Mismatch in cellbox IDs. ID's {missing_a_ids} have appeared in the new mesh. " f"ID's {missing_b_ids} are missing from the new mesh" + ) def test_mesh_cellbox_values(mesh_pair): @@ -84,25 +84,26 @@ def test_mesh_cellbox_values(mesh_pair): """ mesh_a = mesh_pair["old_mesh"] mesh_b = mesh_pair["new_mesh"] - + # Retrieve cellboxes from meshes as dataframes - df_a = pd.DataFrame(mesh_a['cellboxes']).set_index('geometry') - df_b = pd.DataFrame(mesh_b['cellboxes']).set_index('geometry') - + df_a = pd.DataFrame(mesh_a["cellboxes"]).set_index("geometry") + df_b = pd.DataFrame(mesh_b["cellboxes"]).set_index("geometry") + # Extract only cellboxes with same boundaries, drop ID as it may differ common_bounds = extract_common_boundaries_for_comparison(mesh_a, mesh_b) - df_a = df_a.loc[common_bounds].drop(columns=['id']) - df_b = df_b.loc[common_bounds].drop(columns=['id']) + df_a = df_a.loc[common_bounds].drop(columns=["id"]) + df_b = df_b.loc[common_bounds].drop(columns=["id"]) # Round values to significant figures for comparison df_a = round_dataframe_values(df_a) df_b = round_dataframe_values(df_b) # Find differences - diff = df_a.compare(df_b).rename({'self': 'old', 'other': 'new'}) + diff = df_a.compare(df_b).rename({"self": "old", "other": "new"}) - assert diff.empty, \ - f'Mismatch between values in common cellboxes:\n{diff.to_string(max_colwidth=10)}' + assert diff.empty, ( + f"Mismatch between values in common cellboxes:\n{diff.to_string(max_colwidth=10)}" + ) def test_mesh_cellbox_attributes(mesh_pair): @@ -121,9 +122,9 @@ def test_mesh_cellbox_attributes(mesh_pair): """ mesh_a = mesh_pair["old_mesh"] mesh_b = mesh_pair["new_mesh"] - - regression_mesh = mesh_a['cellboxes'] - new_mesh = mesh_b['cellboxes'] + + regression_mesh = mesh_a["cellboxes"] + new_mesh = mesh_b["cellboxes"] regression_mesh_attributes = set(regression_mesh[0].keys()) new_mesh_attributes = set(new_mesh[0].keys()) @@ -131,13 +132,15 @@ def test_mesh_cellbox_attributes(mesh_pair): missing_a_attributes = list(new_mesh_attributes - regression_mesh_attributes) missing_b_attributes = list(regression_mesh_attributes - new_mesh_attributes) - assert regression_mesh_attributes == new_mesh_attributes, \ - f"Mismatch in cellbox attributes. Attributes {missing_a_attributes} have appeared in the new mesh. " \ + assert regression_mesh_attributes == new_mesh_attributes, ( + f"Mismatch in cellbox attributes. Attributes {missing_a_attributes} have appeared in the new mesh. " f"Attributes {missing_b_attributes} are missing in the new mesh" + ) # Neighbour graph comparison test functions + def test_mesh_neighbour_graph_count(mesh_pair): """ Test that neighbour graph node count is preserved between mesh versions. @@ -150,16 +153,17 @@ def test_mesh_neighbour_graph_count(mesh_pair): """ mesh_a = mesh_pair["old_mesh"] mesh_b = mesh_pair["new_mesh"] - - regression_graph = mesh_a['neighbour_graph'] - new_graph = mesh_b['neighbour_graph'] + + regression_graph = mesh_a["neighbour_graph"] + new_graph = mesh_b["neighbour_graph"] regression_graph_count = len(regression_graph.keys()) new_graph_count = len(new_graph.keys()) - assert regression_graph_count == new_graph_count, \ - f"Incorrect number of nodes in neighbour graph. Expected: {regression_graph_count} nodes, " \ + assert regression_graph_count == new_graph_count, ( + f"Incorrect number of nodes in neighbour graph. Expected: {regression_graph_count} nodes, " f"got: {new_graph_count} nodes." + ) def test_mesh_neighbour_graph_ids(mesh_pair): @@ -174,9 +178,9 @@ def test_mesh_neighbour_graph_ids(mesh_pair): """ mesh_a = mesh_pair["old_mesh"] mesh_b = mesh_pair["new_mesh"] - - regression_graph = mesh_a['neighbour_graph'] - new_graph = mesh_b['neighbour_graph'] + + regression_graph = mesh_a["neighbour_graph"] + new_graph = mesh_b["neighbour_graph"] regression_graph_ids = set(regression_graph.keys()) new_graph_ids = set(new_graph.keys()) @@ -184,9 +188,10 @@ def test_mesh_neighbour_graph_ids(mesh_pair): missing_a_keys = list(new_graph_ids - regression_graph_ids) missing_b_keys = list(regression_graph_ids - new_graph_ids) - assert regression_graph_ids == new_graph_ids, \ - f"Mismatch in neighbour graph nodes. {len(missing_a_keys)} nodes have appeared in the new graph. " \ + assert regression_graph_ids == new_graph_ids, ( + f"Mismatch in neighbour graph nodes. {len(missing_a_keys)} nodes have appeared in the new graph. " f"{len(missing_b_keys)} nodes are missing from the new graph." + ) def test_mesh_neighbour_graph_values(mesh_pair): @@ -201,25 +206,26 @@ def test_mesh_neighbour_graph_values(mesh_pair): """ mesh_a = mesh_pair["old_mesh"] mesh_b = mesh_pair["new_mesh"] - - regression_graph = mesh_a['neighbour_graph'] - new_graph = mesh_b['neighbour_graph'] + + regression_graph = mesh_a["neighbour_graph"] + new_graph = mesh_b["neighbour_graph"] mismatch_neighbours = {} - for node in regression_graph.keys(): + for node in regression_graph: # Prevent crashing if node not found (will be detected by test_neighbour_graph_ids) - if node in new_graph.keys(): + if node in new_graph: neighbours_a = regression_graph[node] neighbours_b = new_graph[node] # Sort the lists of neighbours as ordering is not important - sorted_neighbours_a = {k: sorted(neighbours_a[k]) for k in neighbours_a.keys()} - sorted_neighbours_b = {k: sorted(neighbours_b[k]) for k in neighbours_b.keys()} + sorted_neighbours_a = {k: sorted(neighbours_a[k]) for k in neighbours_a} + sorted_neighbours_b = {k: sorted(neighbours_b[k]) for k in neighbours_b} if sorted_neighbours_b != sorted_neighbours_a: mismatch_neighbours[node] = sorted_neighbours_b - assert not mismatch_neighbours, \ - f"Mismatch in neighbour graph neighbours. {len(mismatch_neighbours.keys())} nodes " \ + assert not mismatch_neighbours, ( + f"Mismatch in neighbour graph neighbours. {len(mismatch_neighbours)} nodes " f"have changed in the new mesh." + ) diff --git a/tests/regression_tests/test_mesh.py b/tests/regression_tests/test_mesh.py index 2fc193b0..db46fe2a 100644 --- a/tests/regression_tests/test_mesh.py +++ b/tests/regression_tests/test_mesh.py @@ -4,8 +4,9 @@ """ import json -import os import logging +import os + import pytest import meshiphi diff --git a/tests/regression_tests/utils.py b/tests/regression_tests/utils.py index bb5de1b4..aa423fbb 100644 --- a/tests/regression_tests/utils.py +++ b/tests/regression_tests/utils.py @@ -22,7 +22,7 @@ def round_dataframe_values(df): float_cols = df.select_dtypes(include=float).columns for col in float_cols: df[col] = round_to_sigfig(df[col].to_numpy(), sigfig=SIG_FIG_TOLERANCE) - + # Round list columns that contain floats list_cols = df.select_dtypes(include=list).columns for col in list_cols: @@ -33,7 +33,7 @@ def round_dataframe_values(df): else: round_col.append(val) df[col] = round_col - + return df @@ -48,9 +48,7 @@ def extract_common_boundaries_for_comparison(mesh_a, mesh_b): Returns: list: List of common cellbox geometries (as strings) """ - bounds_a = [cb['geometry'] for cb in mesh_a['cellboxes']] - bounds_b = [cb['geometry'] for cb in mesh_b['cellboxes']] - - common_bounds = [geom for geom in bounds_a if geom in bounds_b] + bounds_a = [cb["geometry"] for cb in mesh_a["cellboxes"]] + bounds_b = [cb["geometry"] for cb in mesh_b["cellboxes"]] - return common_bounds + return [geom for geom in bounds_a if geom in bounds_b] diff --git a/tests/unit_tests/test_aggregated_cellbox.py b/tests/unit_tests/test_aggregated_cellbox.py index b4f9d56d..f73ebcd2 100644 --- a/tests/unit_tests/test_aggregated_cellbox.py +++ b/tests/unit_tests/test_aggregated_cellbox.py @@ -3,6 +3,7 @@ """ import pytest + from meshiphi.mesh_generation.aggregated_cellbox import AggregatedCellBox from meshiphi.mesh_generation.boundary import Boundary @@ -29,15 +30,13 @@ def test_from_json(): agg_cb_data = {k: agg_cb_json[k] for k in data_keys if k in agg_cb_json} agg_cb_id = agg_cb_json["id"] - agg_cb_initialised_normally = AggregatedCellBox( - agg_cb_boundary, agg_cb_data, agg_cb_id - ) + agg_cb_initialised_normally = AggregatedCellBox(agg_cb_boundary, agg_cb_data, agg_cb_id) assert agg_cb_from_json == agg_cb_initialised_normally @pytest.mark.parametrize( - "attr,test_value,getter,setter", + ("attr", "test_value", "getter", "setter"), [ ("boundary", Boundary([10, 20], [30, 40]), "get_bounds", "set_bounds"), ("id", "123", "get_id", "set_id"), @@ -81,7 +80,7 @@ def test_to_json(): @pytest.mark.parametrize( - "cellbox_fixture,lat,lon,expected", + ("cellbox_fixture", "lat", "lon", "expected"), [ ("arbitrary_agg_cellbox", 50, 50, True), ("equatorial_agg_cellbox", 0, 50, True), diff --git a/tests/unit_tests/test_boundary.py b/tests/unit_tests/test_boundary.py index 41fd87ff..8220998e 100644 --- a/tests/unit_tests/test_boundary.py +++ b/tests/unit_tests/test_boundary.py @@ -2,14 +2,15 @@ Boundary class tests. """ +from datetime import datetime, timedelta + import pytest import shapely -from datetime import datetime, timedelta from meshiphi.mesh_generation.boundary import Boundary -def test_load_from_json(arbitrary_boundary): +def test_load_from_json(temporal_boundary): """Test loading boundary from JSON config""" boundary_config = { "region": { @@ -22,11 +23,11 @@ def test_load_from_json(arbitrary_boundary): } } boundary = Boundary.from_json(boundary_config) - assert boundary == arbitrary_boundary + assert boundary == temporal_boundary @pytest.mark.parametrize( - "boundary_fixture,poly_string", + ("boundary_fixture", "poly_string"), [ ("arbitrary_boundary", "POLYGON ((30 10, 30 20, 40 20, 40 10, 30 10))"), ("meridian_boundary", "POLYGON ((-10 -50, -10 -40, 10 -40, 10 -50, -10 -50))"), @@ -44,7 +45,7 @@ def test_from_poly_string(boundary_fixture, poly_string, request): @pytest.mark.parametrize( - "date_string,day_offset", [("TODAY", 0), ("TODAY - 5", -5), ("TODAY + 5", 5)] + ("date_string", "day_offset"), [("TODAY", 0), ("TODAY - 5", -5), ("TODAY + 5", 5)] ) def test_parse_datetime_relative(date_string, day_offset): """Test parsing relative datetime strings""" @@ -65,12 +66,12 @@ def test_parse_datetime_absolute(): ) def test_parse_datetime_malformed(malformed_date): """Test that malformed dates raise ValueError""" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Incorrect date format given"): Boundary.parse_datetime(malformed_date) @pytest.mark.parametrize( - "lat_range,long_range,time_range,description", + ("lat_range", "long_range", "time_range", "description"), [ ([20, 10], [10, 20], ["2000-01-01", "2000-12-31"], "invalid_lat"), ([10, 20], [-190, 190], ["2000-01-01", "2000-12-31"], "invalid_long"), @@ -81,12 +82,12 @@ def test_parse_datetime_malformed(malformed_date): ) def test_validate_bounds(lat_range, long_range, time_range, description): """Test that invalid bounds raise ValueError""" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Boundary:"): Boundary(lat_range, long_range, time_range) @pytest.mark.parametrize( - "boundary_fixture,expected_bounds", + ("boundary_fixture", "expected_bounds"), [ ( "arbitrary_boundary", @@ -125,7 +126,7 @@ def test_get_bounds(boundary_fixture, expected_bounds, request): @pytest.mark.parametrize( - "boundary_fixture,expected_cx", + ("boundary_fixture", "expected_cx"), [ ("arbitrary_boundary", 35), ("meridian_boundary", 0), @@ -140,7 +141,7 @@ def test_getcx(boundary_fixture, expected_cx, request): @pytest.mark.parametrize( - "boundary_fixture,expected_cy", + ("boundary_fixture", "expected_cy"), [ ("arbitrary_boundary", 15), ("meridian_boundary", -45), @@ -155,7 +156,7 @@ def test_getcy(boundary_fixture, expected_cy, request): @pytest.mark.parametrize( - "boundary_fixture,expected_height", + ("boundary_fixture", "expected_height"), [ ("arbitrary_boundary", 10), ("meridian_boundary", 10), @@ -170,7 +171,7 @@ def test_get_height(boundary_fixture, expected_height, request): @pytest.mark.parametrize( - "boundary_fixture,expected_width", + ("boundary_fixture", "expected_width"), [ ("arbitrary_boundary", 10), ("meridian_boundary", 20), @@ -190,7 +191,7 @@ def test_get_time_range(temporal_boundary): @pytest.mark.parametrize( - "boundary_fixture,expected_dcx", + ("boundary_fixture", "expected_dcx"), [ ("arbitrary_boundary", 5), ("meridian_boundary", 10), @@ -205,7 +206,7 @@ def test_getdcx(boundary_fixture, expected_dcx, request): @pytest.mark.parametrize( - "boundary_fixture,expected_dcy", + ("boundary_fixture", "expected_dcy"), [ ("arbitrary_boundary", 5), ("meridian_boundary", 5), @@ -220,7 +221,7 @@ def test_getdcy(boundary_fixture, expected_dcy, request): @pytest.mark.parametrize( - "boundary_fixture,expected_min", + ("boundary_fixture", "expected_min"), [ ("arbitrary_boundary", 10), ("meridian_boundary", -50), @@ -235,7 +236,7 @@ def test_get_lat_min(boundary_fixture, expected_min, request): @pytest.mark.parametrize( - "boundary_fixture,expected_max", + ("boundary_fixture", "expected_max"), [ ("arbitrary_boundary", 20), ("meridian_boundary", -40), @@ -250,7 +251,7 @@ def test_get_lat_max(boundary_fixture, expected_max, request): @pytest.mark.parametrize( - "boundary_fixture,expected_min", + ("boundary_fixture", "expected_min"), [ ("arbitrary_boundary", 30), ("meridian_boundary", -10), @@ -265,7 +266,7 @@ def test_get_long_min(boundary_fixture, expected_min, request): @pytest.mark.parametrize( - "boundary_fixture,expected_max", + ("boundary_fixture", "expected_max"), [ ("arbitrary_boundary", 40), ("meridian_boundary", 10), @@ -290,7 +291,7 @@ def test_get_time_max(temporal_boundary): @pytest.mark.parametrize( - "boundary_fixture,expected_size", + ("boundary_fixture", "expected_size"), [ ("arbitrary_boundary", 1092308.5466932291), ("meridian_boundary", 1354908.6430361348), @@ -305,7 +306,7 @@ def test_calc_size(boundary_fixture, expected_size, request): @pytest.mark.parametrize( - "boundary_fixture,poly_wkt", + ("boundary_fixture", "poly_wkt"), [ ("arbitrary_boundary", "POLYGON ((30 10, 30 20, 40 20, 40 10, 30 10))"), ("meridian_boundary", "POLYGON ((-10 -50, -10 -40, 10 -40, 10 -50, -10 -50))"), @@ -324,7 +325,7 @@ def test_to_polygon(boundary_fixture, poly_wkt, request): @pytest.mark.parametrize( - "boundary_fixture,expected_poly_string", + ("boundary_fixture", "expected_poly_string"), [ ("arbitrary_boundary", "POLYGON ((30 10, 30 20, 40 20, 40 10, 30 10))"), ("meridian_boundary", "POLYGON ((-10 -50, -10 -40, 10 -40, 10 -50, -10 -50))"), diff --git a/tests/unit_tests/test_cellbox.py b/tests/unit_tests/test_cellbox.py index 92cb71c4..741fb393 100644 --- a/tests/unit_tests/test_cellbox.py +++ b/tests/unit_tests/test_cellbox.py @@ -1,7 +1,8 @@ import pytest -from meshiphi.mesh_generation.metadata import Metadata -from meshiphi.mesh_generation.cellbox import CellBox + from meshiphi.mesh_generation.boundary import Boundary +from meshiphi.mesh_generation.cellbox import CellBox +from meshiphi.mesh_generation.metadata import Metadata from tests.conftest import create_cellbox, create_dataloader, create_metadata @@ -43,7 +44,7 @@ def min_cellbox(standard_test_bounds): def test_getter_setter_minimum_datapoints(dummy_cellbox): """Test minimum datapoints getter and setter""" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="can not be negative"): dummy_cellbox.set_minimum_datapoints(-1) dummy_cellbox.set_minimum_datapoints(5) @@ -81,9 +82,7 @@ def test_getter_setter_data_source(dummy_cellbox): assert dummy_cellbox.get_data_source() == arbitrary_data_source2 -@pytest.mark.parametrize( - "bounds", [Boundary([10, 30], [30, 50]), Boundary([0, 20], [20, 40])] -) +@pytest.mark.parametrize("bounds", [Boundary([10, 30], [30, 50]), Boundary([0, 20], [20, 40])]) def test_getter_setter_parent(dummy_cellbox, bounds): """Test parent getter and setter""" arbitrary_cellbox = create_cellbox(bounds) @@ -97,7 +96,7 @@ def test_getter_setter_parent(dummy_cellbox, bounds): def test_getter_setter_split_depth(dummy_cellbox): """Test split depth getter and setter""" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="can not be negative"): dummy_cellbox.set_split_depth(-1) dummy_cellbox.set_split_depth(5) @@ -116,9 +115,7 @@ def test_getter_setter_id(dummy_cellbox): assert dummy_cellbox.get_id() == 321 -@pytest.mark.parametrize( - "bounds", [Boundary([30, 50], [50, 70]), Boundary([20, 40], [40, 60])] -) +@pytest.mark.parametrize("bounds", [Boundary([30, 50], [50, 70]), Boundary([20, 40], [40, 60])]) def test_getter_setter_bounds(dummy_cellbox, bounds): """Test bounds getter and setter""" dummy_cellbox.set_bounds(bounds) @@ -130,7 +127,7 @@ def test_getter_setter_bounds(dummy_cellbox, bounds): @pytest.mark.parametrize( - "cellbox_fixture,expected,description", + ("cellbox_fixture", "expected", "description"), [ ("het_cellbox", True, "heterogeneous"), ("hom_cellbox", False, "homogeneous"), @@ -145,7 +142,7 @@ def test_should_split(cellbox_fixture, expected, description, request): @pytest.mark.parametrize( - "cellbox_fixture,expected,description", + ("cellbox_fixture", "expected", "description"), [ ("het_cellbox", True, "heterogeneous"), ("hom_cellbox", False, "homogeneous"), @@ -242,9 +239,7 @@ def test_check_vector_data(): "multiplier_v": 1, } - vector_parent_cb = create_cellbox( - vector_bounds, params=vector_params, id=1, parent=None - ) + vector_parent_cb = create_cellbox(vector_bounds, params=vector_params, id=1, parent=None) vector_child_cb = vector_parent_cb.split(1)[0] arbitrary_cb = create_cellbox( @@ -276,7 +271,7 @@ def test_check_vector_data(): == parent_agg_val ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Dataloader not found in parent"): arbitrary_cb.check_vector_data( arbitrary_cb.data_source[0], vector_child_cb.data_source[0].get_data_loader(), diff --git a/tests/unit_tests/test_env_mesh.py b/tests/unit_tests/test_env_mesh.py index f0bdf477..c937ed82 100644 --- a/tests/unit_tests/test_env_mesh.py +++ b/tests/unit_tests/test_env_mesh.py @@ -2,14 +2,16 @@ EnvironmentMesh class tests. """ -import pytest import json import os import tempfile + +import pytest + +from meshiphi.mesh_generation.aggregated_cellbox import AggregatedCellBox +from meshiphi.mesh_generation.boundary import Boundary from meshiphi.mesh_generation.environment_mesh import EnvironmentMesh from meshiphi.mesh_generation.mesh_builder import MeshBuilder -from meshiphi.mesh_generation.boundary import Boundary -from meshiphi.mesh_generation.aggregated_cellbox import AggregatedCellBox from meshiphi.mesh_generation.neighbour_graph import NeighbourGraph from tests.conftest import REGRESSION_TESTS_DIR @@ -17,11 +19,9 @@ @pytest.fixture def env_mesh_data(): """Load environment mesh data from test file.""" - json_file_path = ( - REGRESSION_TESTS_DIR / "example_meshes/env_meshes/grf_reprojection.json" - ) + json_file_path = REGRESSION_TESTS_DIR / "example_meshes/env_meshes/grf_reprojection.json" - with open(json_file_path, "r") as config_file: + with open(json_file_path) as config_file: json_file = json.load(config_file) config = json_file["config"]["mesh_info"] env_mesh = MeshBuilder(config).build_environmental_mesh() @@ -38,9 +38,7 @@ def test_load_from_json(env_mesh_data): assert loaded.bounds.get_bounds() == original.bounds.get_bounds() assert len(loaded.agg_cellboxes) == len(original.agg_cellboxes) - assert len(loaded.neighbour_graph.get_graph()) == len( - original.neighbour_graph.get_graph() - ) + assert len(loaded.neighbour_graph.get_graph()) == len(original.neighbour_graph.get_graph()) def test_update_agg_cellbox(env_mesh_data): @@ -80,9 +78,7 @@ def test_to_tif(): tmp_path = tmp.name # Create format parameters for tif export - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as params_file: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as params_file: format_params = { "data_name": "test_value", "sampling_resolution": [10, 10], # Small resolution for faster test diff --git a/tests/unit_tests/test_mesh_builder.py b/tests/unit_tests/test_mesh_builder.py index 27dda487..14ae7106 100644 --- a/tests/unit_tests/test_mesh_builder.py +++ b/tests/unit_tests/test_mesh_builder.py @@ -2,10 +2,12 @@ MeshBuilder class tests. """ -import pytest import json -from meshiphi.mesh_generation.mesh_builder import MeshBuilder + +import pytest + from meshiphi.mesh_generation.direction import Direction +from meshiphi.mesh_generation.mesh_builder import MeshBuilder from tests.conftest import UNIT_TESTS_DIR @@ -14,7 +16,7 @@ def mesh_builder(): """Create a mesh builder instance for testing.""" json_file_path = UNIT_TESTS_DIR / "resources/global_grf_normal.json" - with open(json_file_path, "r") as config_file: + with open(json_file_path) as config_file: json_file = json.load(config_file) config = json_file["config"]["mesh_info"] builder = MeshBuilder(config) @@ -31,7 +33,7 @@ def test_check_global_mesh(mesh_builder): @pytest.mark.parametrize( - "cb1_idx,cb2_idx,expected_dir,description", + ("cb1_idx", "cb2_idx", "expected_dir", "description"), [ (0, 71, Direction.west, "edge wrapping west"), (71, 0, Direction.east, "edge wrapping east"), @@ -49,9 +51,7 @@ def test_check_global_mesh(mesh_builder): (1, 74, Direction.north_east, "vertical north_east"), ], ) -def test_neighbour_relationships( - mesh_builder, cb1_idx, cb2_idx, expected_dir, description -): +def test_neighbour_relationships(mesh_builder, cb1_idx, cb2_idx, expected_dir, description): """Test neighbour relationships in global mesh""" builder = mesh_builder["builder"] actual_dir = builder.neighbour_graph.get_neighbour_case( diff --git a/tests/unit_tests/test_mesh_validator.py b/tests/unit_tests/test_mesh_validator.py index b8b63060..65a66e7e 100644 --- a/tests/unit_tests/test_mesh_validator.py +++ b/tests/unit_tests/test_mesh_validator.py @@ -3,8 +3,9 @@ """ import pytest -from meshiphi.mesh_validation.sampler import Sampler + from meshiphi.mesh_validation.mesh_validator import MeshValidator +from meshiphi.mesh_validation.sampler import Sampler from tests.conftest import REGRESSION_TESTS_DIR diff --git a/tests/unit_tests/test_neighbour_graph.py b/tests/unit_tests/test_neighbour_graph.py index 85853deb..44e0c705 100644 --- a/tests/unit_tests/test_neighbour_graph.py +++ b/tests/unit_tests/test_neighbour_graph.py @@ -2,14 +2,13 @@ import pytest +from meshiphi.mesh_generation.boundary import Boundary +from meshiphi.mesh_generation.cellbox import CellBox from meshiphi.mesh_generation.direction import Direction from meshiphi.mesh_generation.neighbour_graph import NeighbourGraph -from meshiphi.mesh_generation.cellbox import CellBox -from meshiphi.mesh_generation.boundary import Boundary from meshiphi.utils import longitude_domain from tests.conftest import create_ng_from_dict - # Define which direction each cardinal direction lies NORTHERN_DIRECTIONS = [Direction.north_east, Direction.north, Direction.north_west] EASTERN_DIRECTIONS = [Direction.north_east, Direction.east, Direction.south_east] @@ -160,8 +159,8 @@ def test_remove_node_and_update_neighbours(ng_dict_3x3): # Create a new neighbour graph manually_removed_ng_dict = copy.deepcopy(ng_dict_3x3) # Remove the central node by popping it out of neighbour lists - for node, dir_map in manually_removed_ng_dict.items(): - for direction, neighbours in dir_map.items(): + for _node, dir_map in manually_removed_ng_dict.items(): + for _direction, neighbours in dir_map.items(): if node_to_remove in neighbours: neighbours.pop(neighbours.index(node_to_remove)) # Then remove the central node entirely @@ -171,7 +170,7 @@ def test_remove_node_and_update_neighbours(ng_dict_3x3): def test_get_neighbours(neighbour_graph, ng_dict_3x3): - for cb_index in ng_dict_3x3.keys(): + for cb_index in ng_dict_3x3: for direction in ALL_DIRECTIONS: ng_neighbours = neighbour_graph.get_neighbours(cb_index, direction) assert ng_neighbours == ng_dict_3x3[cb_index][direction] @@ -310,7 +309,7 @@ def test_update_corner_neighbours(ng_dict_3x3): @pytest.mark.parametrize( - "direction,lat_offset,long_offset", + ("direction", "lat_offset", "long_offset"), [ (Direction.north, 20, 0), (Direction.north_east, 20, 20), @@ -365,7 +364,7 @@ def test_get_neighbour_case_bounds_non_touching(): @pytest.mark.parametrize( - "direction,lat_offset,long_offset", + ("direction", "lat_offset", "long_offset"), [ (Direction.north, 20, 0), (Direction.north_east, 20, 20), @@ -427,7 +426,7 @@ def test_get_neighbour_case_non_touching(): @pytest.mark.parametrize( - "direction,lat_offset,east_base", + ("direction", "lat_offset", "east_base"), [ (Direction.north_east, 20, True), (Direction.east, 0, True), From 0bcbfff85edf357eb22a86acde5529b098b8090a Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Wed, 14 Jan 2026 15:27:07 +0000 Subject: [PATCH 04/15] add typehints to config_validation --- .../config_validation/config_validator.py | 27 ++++++++++--------- meshiphi/config_validation/mesh_schema.py | 10 ++++--- pyproject.toml | 8 +----- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/meshiphi/config_validation/config_validator.py b/meshiphi/config_validation/config_validator.py index d449d53e..a420fb7a 100644 --- a/meshiphi/config_validation/config_validator.py +++ b/meshiphi/config_validation/config_validator.py @@ -1,13 +1,14 @@ import datetime import json import re +from typing import Any, Union import jsonschema from meshiphi.config_validation.mesh_schema import mesh_schema -def flexi_json_input(config): +def flexi_json_input(config: Union[str, dict[str, Any]]) -> dict[str, Any]: """ Allows flexible inputs. If a string is parsed, then assume it's a file path and read in as a json. If a dict is parsed, then assume it's already a @@ -25,7 +26,7 @@ def flexi_json_input(config): if isinstance(config, str): # If str, assume filename with open(config) as fp: - config_json = json.load(fp) + config_json: dict[str, Any] = json.load(fp) elif isinstance(config, dict): # If dict, assume it's the config config_json = config @@ -36,7 +37,7 @@ def flexi_json_input(config): return config_json -def validate_mesh_config(config): +def validate_mesh_config(config: Union[str, dict[str, Any]]) -> None: """ Validates a mesh config @@ -52,7 +53,7 @@ def validate_mesh_config(config): ValidationError: Malformed mesh config """ - def assert_valid_time(time_str): + def assert_valid_time(time_str: str) -> None: """ Checks if the time strings in the config are correctly formatted @@ -80,7 +81,7 @@ def assert_valid_time(time_str): if not correctly_formatted: raise ValueError(f"{time_str} is not a valid date format!") - def assert_valid_cellsize(bound_min, bound_max, cell_size): + def assert_valid_cellsize(bound_min: float, bound_max: float, cell_size: float) -> None: """ Ensures that the initial cellbox size can evenly divide the initial boundary. @@ -101,17 +102,17 @@ def assert_valid_cellsize(bound_min, bound_max, cell_size): jsonschema.validate(instance=config_json, schema=mesh_schema) # Check that the dates in the region are valid - assert_valid_time(config["region"]["start_time"]) - assert_valid_time(config["region"]["end_time"]) + assert_valid_time(config_json["region"]["start_time"]) + assert_valid_time(config_json["region"]["end_time"]) # Check that cellbox width and height evenly divide boundary assert_valid_cellsize( - config["region"]["lat_min"], - config["region"]["lat_max"], - config["region"]["cell_height"], + config_json["region"]["lat_min"], + config_json["region"]["lat_max"], + config_json["region"]["cell_height"], ) assert_valid_cellsize( - config["region"]["long_min"], - config["region"]["long_max"], - config["region"]["cell_width"], + config_json["region"]["long_min"], + config_json["region"]["long_max"], + config_json["region"]["cell_width"], ) diff --git a/meshiphi/config_validation/mesh_schema.py b/meshiphi/config_validation/mesh_schema.py index a0ca7e46..030ce3ca 100644 --- a/meshiphi/config_validation/mesh_schema.py +++ b/meshiphi/config_validation/mesh_schema.py @@ -1,4 +1,6 @@ -region_schema = { +from typing import Any + +region_schema: dict[str, Any] = { "type": "object", "required": [ "lat_min", @@ -23,14 +25,14 @@ }, } -dataloader_schema = { +dataloader_schema: dict[str, Any] = { "type": "object", "required": ["loader", "params"], "additionalProperties": False, "properties": {"loader": {"type": "string"}, "params": {"type": "object"}}, } -splitting_schema = { +splitting_schema: dict[str, Any] = { "type": "object", "required": ["split_depth", "minimum_datapoints"], "additionalProperties": False, @@ -40,7 +42,7 @@ }, } -mesh_schema = { +mesh_schema: dict[str, Any] = { "type": "object", "required": ["region", "data_sources", "splitting"], "properties": { diff --git a/pyproject.toml b/pyproject.toml index f84903b5..1fc765ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ dev = [ "ruff", "mypy", "types-requests", + "types-jsonschema", "pytest", "pytest-xdist", "pytest-cov", @@ -198,14 +199,7 @@ module = [ "numpy.*", "sklearn.*", "scipy.*", - "jsonschema", - "jsonschema.*", "tqdm", "tqdm.*", ] ignore_missing_imports = true - -# Disable disallow_untyped_calls for the config validation module which uses jsonschema -[[tool.mypy.overrides]] -module = "meshiphi.config_validation.*" -disallow_untyped_calls = false From 00bc10b66337a6d3547bd7cc1f89943a79ce87aa Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Wed, 14 Jan 2026 17:19:46 +0000 Subject: [PATCH 05/15] start with --strict flag --- meshiphi/dataloaders/dataloader_interface.py | 6 +- meshiphi/dataloaders/lut/abstract_lut.py | 36 +++-- meshiphi/dataloaders/lut/density.py | 11 +- meshiphi/dataloaders/lut/lut_csv.py | 6 +- meshiphi/dataloaders/lut/lut_geojson.py | 6 +- meshiphi/dataloaders/lut/lut_shapefile.py | 6 +- meshiphi/dataloaders/lut/scotland_ncmpa.py | 6 +- meshiphi/dataloaders/lut/thickness.py | 34 ++-- .../dataloaders/scalar/abstract_scalar.py | 140 ++++++++++------- meshiphi/dataloaders/scalar/baltic_sea_ice.py | 11 +- meshiphi/dataloaders/scalar/bsose_depth.py | 11 +- meshiphi/dataloaders/scalar/bsose_sea_ice.py | 10 +- .../scalar/era5_max_wave_height.py | 12 +- .../scalar/era5_mean_wave_direction.py | 12 +- .../scalar/era5_sig_wave_height.py | 12 +- .../dataloaders/scalar/era5_wave_period.py | 12 +- meshiphi/dataloaders/scalar/era5_wind_dir.py | 12 +- meshiphi/dataloaders/scalar/era5_wind_mag.py | 12 +- meshiphi/dataloaders/scalar/gebco.py | 11 +- meshiphi/dataloaders/scalar/icenet.py | 13 +- meshiphi/dataloaders/scalar/modis.py | 10 +- meshiphi/dataloaders/scalar/scalar_csv.py | 11 +- meshiphi/dataloaders/scalar/visual_iced.py | 16 +- .../dataloaders/vector/abstract_vector.py | 145 ++++++++++++----- .../mesh_generation/aggregated_cellbox.py | 6 +- meshiphi/mesh_generation/boundary.py | 29 ++-- meshiphi/mesh_generation/cellbox.py | 85 ++++++---- meshiphi/mesh_generation/direction.py | 18 +-- meshiphi/mesh_generation/environment_mesh.py | 148 ++++++++++-------- meshiphi/mesh_generation/mesh.py | 22 +-- meshiphi/mesh_generation/mesh_builder.py | 88 ++++++----- meshiphi/mesh_generation/neighbour_graph.py | 120 +++++++------- 32 files changed, 674 insertions(+), 403 deletions(-) diff --git a/meshiphi/dataloaders/dataloader_interface.py b/meshiphi/dataloaders/dataloader_interface.py index 38ab68b8..53464f49 100644 --- a/meshiphi/dataloaders/dataloader_interface.py +++ b/meshiphi/dataloaders/dataloader_interface.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import ABCMeta, abstractmethod -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from meshiphi.mesh_generation.boundary import Boundary @@ -15,7 +15,7 @@ class DataLoaderInterface(metaclass=ABCMeta): """ @classmethod - def __subclasshook__(cls, subclass): + def __subclasshook__(cls, subclass: type) -> bool: return ( hasattr(subclass, "get_value") and callable(subclass.get_value) @@ -30,7 +30,7 @@ def get_value(self, bounds: Boundary, agg_type: str) -> dict[str, float]: raise NotImplementedError @abstractmethod - def get_hom_condition(self, bounds: Boundary, splitting_conds: dict) -> str: + def get_hom_condition(self, bounds: Boundary, splitting_conds: dict[str, Any]) -> str: """Reads values within boundary to determine if data is homogeneous or heterogeneous. Return as str""" raise NotImplementedError diff --git a/meshiphi/dataloaders/lut/abstract_lut.py b/meshiphi/dataloaders/lut/abstract_lut.py index 67b820ac..17c1d3b8 100644 --- a/meshiphi/dataloaders/lut/abstract_lut.py +++ b/meshiphi/dataloaders/lut/abstract_lut.py @@ -2,7 +2,7 @@ import logging from abc import abstractmethod -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import numpy as np import pandas as pd @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) -class LutDataLoader(DataLoaderInterface): +class LutDataLoader(DataLoaderInterface): # type: ignore[misc] """ Abstract class for all LookUp Table Datasets. """ @@ -101,7 +101,7 @@ def __init__(self, bounds: Boundary, params: dict[str, Any]) -> None: self.data = self.trim_datapoints(bounds) @abstractmethod - def import_data(self, bounds): + def import_data(self, bounds: Boundary) -> Any: # gpd.GeoDataFrame """ User defined method for importing data from files, or even generating data from scratch @@ -115,7 +115,7 @@ def import_data(self, bounds): """ - def add_default_params(self, params): + def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for all LUT dataloaders. This function should be overloaded to include any extra params for a specific dataloader @@ -141,7 +141,7 @@ def add_default_params(self, params): return params - def verify_data(self, data=None): + def verify_data(self, data: Any = None) -> pd.DataFrame: """ Verifies that all geometries read in are Polygons or MultiPolygons If MultiPolygon, then split out into multiple Polygons @@ -179,7 +179,7 @@ def verify_data(self, data=None): ) return pd.concat(new_data, ignore_index=True) - def calculate_coverage(self, bounds, data=None): + def calculate_coverage(self, bounds: Boundary, data: Any = None) -> float: """ Calculates percentage of boundary covered by dataset @@ -210,9 +210,9 @@ def calculate_coverage(self, bounds, data=None): overlap_area = data_polygon.intersection(bounds_polygon).area total_area = bounds_polygon.area - return overlap_area / total_area + return cast("float", overlap_area / total_area) - def trim_datapoints(self, bounds, data=None): + def trim_datapoints(self, bounds: Boundary, data: Any = None) -> pd.DataFrame: """ Trims datapoints from self.data within boundary defined by 'bounds'. self.data can be pd.DataFrame or xr.Dataset @@ -237,7 +237,9 @@ def trim_datapoints(self, bounds, data=None): # Return only rows intersecting with cellbox boundary return data.iloc[intersections] - def get_value(self, bounds, agg_type=None, skipna=False, data=None): + def get_value( + self, bounds: Boundary, agg_type: str | None = None, skipna: bool = False, data: Any = None + ) -> dict[str, float]: """ Retrieve aggregated value from within bounds @@ -316,9 +318,13 @@ def get_value(self, bounds, agg_type=None, skipna=False, data=None): if isinstance(ret_val, np.bool_): ret_val = bool(ret_val) + if self.data_name is None: + raise ValueError("data_name is None") return {self.data_name: ret_val} - def get_hom_condition(self, bounds, splitting_conds, data=None): + def get_hom_condition( + self, bounds: Boundary, splitting_conds: dict[str, Any], data: Any = None + ) -> str: """ Retrieves homogeneity condition of data within boundary. @@ -358,19 +364,19 @@ def get_hom_condition(self, bounds, splitting_conds, data=None): # Otherwise no boundaries intersected bounds return hom_type - def reproject(self): + def reproject(self) -> None: """ Reprojection not supported by LookUpTable Dataloader """ logger.warning("Reprojection not supported by LookUpTable Dataloader") - def downsample(self): + def downsample(self) -> None: """ Downsampling not supported by LookUpTable Dataloader """ logger.warning("Downsampling not supported by LookUpTable Dataloader") - def get_data_col_name(self): + def get_data_col_name(self) -> str: """ Retrieve name of data column. Used for when data_name not defined in params. @@ -391,9 +397,9 @@ def get_data_col_name(self): assert len(unique_cols) == 1, f"Expected only one data column! Instead, found {unique_cols}" - return unique_cols[0] + return str(unique_cols[0]) - def set_data_col_name(self, new_name): + def set_data_col_name(self, new_name: str) -> pd.DataFrame: """ Sets name of data column/data variable diff --git a/meshiphi/dataloaders/lut/density.py b/meshiphi/dataloaders/lut/density.py index 735e1394..90fdc075 100644 --- a/meshiphi/dataloaders/lut/density.py +++ b/meshiphi/dataloaders/lut/density.py @@ -1,11 +1,18 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import pandas as pd from shapely import wkt from meshiphi.dataloaders.lut.abstract_lut import LutDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class DensityDataLoader(LutDataLoader): - def import_data(self, bounds): +class DensityDataLoader(LutDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Creates a simulated dataset of sea ice density based on scientific literature. diff --git a/meshiphi/dataloaders/lut/lut_csv.py b/meshiphi/dataloaders/lut/lut_csv.py index 0f04632f..eb0664bd 100644 --- a/meshiphi/dataloaders/lut/lut_csv.py +++ b/meshiphi/dataloaders/lut/lut_csv.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import numpy as np import pandas as pd from shapely import wkt @@ -7,8 +9,8 @@ from meshiphi.mesh_generation.boundary import Boundary -class LutCSV(LutDataLoader): - def import_data(self, bounds): +class LutCSV(LutDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Import a list of .csv files, assign regions a value specified in config params, regions outside this are numpy nan values. diff --git a/meshiphi/dataloaders/lut/lut_geojson.py b/meshiphi/dataloaders/lut/lut_geojson.py index abd77f2f..9113148f 100644 --- a/meshiphi/dataloaders/lut/lut_geojson.py +++ b/meshiphi/dataloaders/lut/lut_geojson.py @@ -11,8 +11,8 @@ from meshiphi.mesh_generation.boundary import Boundary -class LutGeoJSON(LutDataLoader): - def add_default_params(self, params): +class LutGeoJSON(LutDataLoader): # type: ignore[misc] + def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for LUT dataloader. Only unique addition over the regular abstracted add_default_params is the data_name @@ -36,7 +36,7 @@ def add_default_params(self, params): return params - def import_data(self, bounds): + def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Import a list of GeoJSON files, assign regions a value specified in config params, regions outside this are numpy nan values. diff --git a/meshiphi/dataloaders/lut/lut_shapefile.py b/meshiphi/dataloaders/lut/lut_shapefile.py index 111839d3..cd437779 100644 --- a/meshiphi/dataloaders/lut/lut_shapefile.py +++ b/meshiphi/dataloaders/lut/lut_shapefile.py @@ -11,8 +11,8 @@ from meshiphi.mesh_generation.boundary import Boundary -class LutShapefile(LutDataLoader): - def add_default_params(self, params): +class LutShapefile(LutDataLoader): # type: ignore[misc] + def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for LUT dataloader. Only unique addition over the regular abstracted add_default_params is the data_name @@ -36,7 +36,7 @@ def add_default_params(self, params): return params - def import_data(self, bounds): + def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Import a list of .shp files, assign regions a value specified in config params, regions outside this are numpy nan values. diff --git a/meshiphi/dataloaders/lut/scotland_ncmpa.py b/meshiphi/dataloaders/lut/scotland_ncmpa.py index d48fd405..291e1196 100644 --- a/meshiphi/dataloaders/lut/scotland_ncmpa.py +++ b/meshiphi/dataloaders/lut/scotland_ncmpa.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import geopandas as gpd import pandas as pd from shapely.ops import unary_union @@ -6,8 +8,8 @@ from meshiphi.mesh_generation.boundary import Boundary -class ScotlandNCMPA(LutDataLoader): - def import_data(self, bounds): +class ScotlandNCMPA(LutDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Creates a simulated dataset of sea ice thickness based on scientific literature. diff --git a/meshiphi/dataloaders/lut/thickness.py b/meshiphi/dataloaders/lut/thickness.py index e8343252..4a207008 100644 --- a/meshiphi/dataloaders/lut/thickness.py +++ b/meshiphi/dataloaders/lut/thickness.py @@ -1,3 +1,7 @@ +from __future__ import annotations + +from typing import Any + import pandas as pd from shapely import Polygon @@ -5,7 +9,7 @@ from meshiphi.mesh_generation.boundary import Boundary # Mapping of month number to season per hemisphere -northern_seasons = { +northern_seasons: dict[int, str] = { 1: "wi", 2: "wi", 12: "wi", @@ -19,7 +23,7 @@ 10: "au", 11: "au", } -southern_seasons = { +southern_seasons: dict[int, str] = { 1: "su", 2: "su", 12: "su", @@ -35,23 +39,29 @@ } -class ThicknessDataLoader(LutDataLoader): +class ThicknessDataLoader(LutDataLoader): # type: ignore[misc] class Region: """ Data storage object with region boundary as polygon, seasonal sea ice densities, and a dict mapping from integer month to season """ - def __init__(self, name, geometry, value_dict, seasons=southern_seasons): - self.name = name - self.geometry = geometry - self.value_dict = value_dict - self.month_to_season = seasons - - def get_value(self, month): + def __init__( + self, + name: str, + geometry: Any, + value_dict: dict[str, float], + seasons: dict[int, str] = southern_seasons, + ) -> None: + self.name: str = name + self.geometry: Any = geometry + self.value_dict: dict[str, float] = value_dict + self.month_to_season: dict[int, str] = seasons + + def get_value(self, month: int) -> float: """ Returns the sea ice density for a given month, taking into account - seasons per hemisphere + seasons per hemisphere. Args: month (int): Month as an integer (1 = Jan ... 12 = Dec) @@ -61,7 +71,7 @@ def get_value(self, month): """ return self.value_dict[self.month_to_season[month]] - def import_data(self, bounds): + def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Creates a simulated dataset of sea ice thickness based on scientific literature. diff --git a/meshiphi/dataloaders/scalar/abstract_scalar.py b/meshiphi/dataloaders/scalar/abstract_scalar.py index 9250080b..142c2fdc 100644 --- a/meshiphi/dataloaders/scalar/abstract_scalar.py +++ b/meshiphi/dataloaders/scalar/abstract_scalar.py @@ -2,7 +2,7 @@ import logging from abc import abstractmethod -from typing import Any, overload +from typing import Any, cast import numpy as np import pandas as pd @@ -132,7 +132,7 @@ def __init__(self, bounds: Boundary, params: dict[str, Any]) -> None: self.data = self.trim_datapoints(bounds) @abstractmethod - def import_data(self, bounds): + def import_data(self, bounds: Boundary) -> xr.Dataset | pd.DataFrame: """ User defined method for importing data from files, or even generating data from scratch @@ -151,7 +151,7 @@ def import_data(self, bounds): Downsampling and reprojecting happen in __init__() method """ - def add_default_params(self, params): + def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for all scalar dataloaders. This function should be overloaded to include any extra params for a specific dataloader @@ -198,7 +198,9 @@ def add_default_params(self, params): return params - def calculate_coverage(self, bounds, data=None): + def calculate_coverage( + self, bounds: Boundary, data: xr.Dataset | pd.DataFrame | None = None + ) -> float: """ Calculates percentage of boundary covered by dataset @@ -215,7 +217,7 @@ def calculate_coverage(self, bounds, data=None): Decimal fraction of boundary covered by the dataset """ - def calculate_coverage_from_df(bounds, data): + def calculate_coverage_from_df(bounds: Boundary, data: pd.DataFrame) -> float: data = data.dropna().reset_index() # If empty dataframe, 0% coverage if data.empty or data.lat.size == 0 or data.long.size == 0: @@ -233,9 +235,9 @@ def calculate_coverage_from_df(bounds, data): overlap_area = data_polygon.intersection(bounds_polygon).area total_area = bounds_polygon.area - return overlap_area / total_area + return cast("float", overlap_area / total_area) - def calculate_coverage_from_xr(bounds, data): + def calculate_coverage_from_xr(bounds: Boundary, data: xr.Dataset) -> float: # Remove all NaN columns/rows data = data.dropna(dim="lat", how="all") data = data.dropna(dim="long", how="all") @@ -256,7 +258,7 @@ def calculate_coverage_from_xr(bounds, data): overlap_area = data_polygon.intersection(bounds_polygon).area total_area = bounds_polygon.area - return overlap_area / total_area + return cast("float", overlap_area / total_area) # Use self.data if not no explicit dataset specified if data is None: @@ -268,7 +270,9 @@ def calculate_coverage_from_xr(bounds, data): return calculate_coverage_from_xr(bounds, data) raise TypeError(f"Unsupported data type: {type(self.data)}") - def trim_datapoints(self, bounds, data=None): + def trim_datapoints( + self, bounds: Boundary, data: xr.Dataset | pd.DataFrame | None = None + ) -> xr.Dataset | pd.DataFrame: """ Trims datapoints from self.data within boundary defined by 'bounds'. self.data can be pd.DataFrame or xr.Dataset @@ -281,7 +285,7 @@ def trim_datapoints(self, bounds, data=None): Trimmed dataset in same format as self.data """ - def trim_datapoints_from_df(data, bounds): + def trim_datapoints_from_df(data: pd.DataFrame, bounds: Boundary) -> pd.DataFrame: """ Extracts datapoints from a pd.DataFrame @@ -383,7 +387,7 @@ def trim_datapoints_from_df(data, bounds): # Return column of data from within bounds return data.loc[mask] - def trim_datapoints_from_xr(data, bounds): + def trim_datapoints_from_xr(data: xr.Dataset, bounds: Boundary) -> xr.Dataset: """ Extracts datapoints from a xr.Dataset @@ -458,15 +462,13 @@ def trim_datapoints_from_xr(data, bounds): return trim_datapoints_from_xr(data, bounds) raise TypeError(f"Unsupported data type: {type(data)}") - @overload - def get_value(self, bounds: Boundary, agg_type: str) -> dict[str, float]: ... - - @overload def get_value( - self, bounds: Boundary, data: Any = None, agg_type: str | None = None, skipna: bool = True - ) -> dict[str, float]: ... - - def get_value(self, bounds, data=None, agg_type=None, skipna=True): + self, + bounds: Boundary, + data: xr.Dataset | pd.DataFrame | None = None, + agg_type: str | None = None, + skipna: bool = True, + ) -> dict[str, float]: """ Retrieve aggregated value from within bounds @@ -487,7 +489,9 @@ def get_value(self, bounds, data=None, agg_type=None, skipna=True): ValueError: aggregation type not in list of available methods """ - def get_value_from_df(dps, bounds, agg_type, skipna): + def get_value_from_df( + dps: pd.Series, bounds: Boundary, agg_type: str, skipna: bool + ) -> float: """ Aggregates a value from a pd.Series. @@ -520,19 +524,21 @@ def get_value_from_df(dps, bounds, agg_type, skipna): return np.nan # Return float of aggregated value if agg_type == "MIN": - return dps.min(skipna=skipna) + return float(dps.min(skipna=skipna)) if agg_type == "MAX": - return dps.max(skipna=skipna) + return float(dps.max(skipna=skipna)) if agg_type == "MEAN": - return dps.mean(skipna=skipna) + return float(dps.mean(skipna=skipna)) if agg_type == "MEDIAN": - return dps.median(skipna=skipna) + return float(dps.median(skipna=skipna)) if agg_type == "STD": - return dps.std(skipna=skipna) + return float(dps.std(skipna=skipna)) # If aggregation_type not available raise ValueError(f"Unknown aggregation type {agg_type}") - def get_value_from_xr(dps, bounds, agg_type, skipna): + def get_value_from_xr( + dps: xr.DataArray, bounds: Boundary, agg_type: str, skipna: bool + ) -> float: """ Aggregates a value from a xr.DataArray. @@ -557,35 +563,35 @@ def get_value_from_xr(dps, bounds, agg_type, skipna): ) # If want the number of datapoints if agg_type == "COUNT": - return dps.size + return float(dps.size) # If no data if dps.size == 0 or np.isnan(dps).all(): return np.nan # Return float of aggregated value if agg_type == "MIN": if skipna: - return np.nanmin(dps) - return np.min(dps) + return float(np.nanmin(dps)) + return float(np.min(dps)) if agg_type == "MAX": if skipna: - return np.nanmax(dps) - return np.max(dps) + return float(np.nanmax(dps)) + return float(np.max(dps)) if agg_type == "MEAN": if skipna: - return np.nanmean(dps) - return np.mean(dps) + return float(np.nanmean(dps)) + return float(np.mean(dps)) if agg_type == "MEDIAN": if skipna: - return np.nanmedian(dps) - return np.median(dps) + return float(np.nanmedian(dps)) + return float(np.median(dps)) if agg_type == "STD": if skipna: - return np.nanstd(dps) - return np.std(dps) + return float(np.nanstd(dps)) + return float(np.std(dps)) if agg_type == "RMSE": if skipna: - return np.sqrt(np.nanmean((dps - np.nanmean(dps)) ** 2)) - return np.sqrt(np.mean((dps - np.mean(dps)) ** 2)) + return float(np.sqrt(np.nanmean((dps - np.nanmean(dps)) ** 2))) + return float(np.sqrt(np.mean((dps - np.mean(dps)) ** 2))) # If aggregation_type not available raise ValueError(f"Unknown aggregation type {agg_type}") @@ -602,9 +608,16 @@ def get_value_from_xr(dps, bounds, agg_type, skipna): value = get_value_from_xr(dps, bounds, agg_type, skipna) # Cast to regular float before returning so can be saved in JSON later + if self.data_name is None: + raise ValueError("data_name is None") return {self.data_name: float(value)} - def get_hom_condition(self, bounds, splitting_conds, data=None): + def get_hom_condition( + self, + bounds: Boundary, + splitting_conds: dict[str, Any], + data: xr.Dataset | pd.DataFrame | None = None, + ) -> str: """ Retrieves homogeneity condition of data within boundary. @@ -642,7 +655,7 @@ def get_hom_condition(self, bounds, splitting_conds, data=None): """ - def get_hom_condition_from_df(dps, splitting_conds): + def get_hom_condition_from_df(dps: pd.Series, splitting_conds: dict[str, Any]) -> str: """ Determined homogeneity condition from pd.Series. @@ -684,7 +697,7 @@ def get_hom_condition_from_df(dps, splitting_conds): ) return hom_type - def get_hom_condition_from_xr(dps, splitting_conds): + def get_hom_condition_from_xr(dps: xr.DataArray, splitting_conds: dict[str, Any]) -> str: """ Determined homogeneity condition from xr.DataArray. @@ -742,7 +755,13 @@ def get_hom_condition_from_xr(dps, splitting_conds): return get_hom_condition_from_xr(dps, splitting_conds) raise TypeError(f"Unknown type {type(dps)}") - def reproject(self, in_proj="EPSG:4326", out_proj="EPSG:4326", x_col="lat", y_col="long"): + def reproject( + self, + in_proj: str = "EPSG:4326", + out_proj: str = "EPSG:4326", + x_col: str = "lat", + y_col: str = "long", + ) -> xr.Dataset | pd.DataFrame: """ Reprojects data using pyProj.Transformer self.data can be pd.DataFrame or xr.Dataset @@ -766,7 +785,9 @@ def reproject(self, in_proj="EPSG:4326", out_proj="EPSG:4326", x_col="lat", y_co replacing 'x_col' and 'y_col' """ - def reproject_df(data, in_proj, out_proj, x_col, y_col): + def reproject_df( + data: pd.DataFrame, in_proj: str, out_proj: str, x_col: str, y_col: str + ) -> pd.DataFrame: """ Reprojects a pandas dataframe @@ -806,7 +827,14 @@ def reproject_df(data, in_proj, out_proj, x_col, y_col): return data - def reproject_xr(data, in_proj, out_proj, x_col, y_col, fast=False): + def reproject_xr( + data: xr.Dataset, + in_proj: str, + out_proj: str, + x_col: str, + y_col: str, + fast: bool = False, + ) -> xr.Dataset: """ Reprojects a xr.Dataset @@ -866,7 +894,7 @@ def reproject_xr(data, in_proj, out_proj, x_col, y_col, fast=False): ) raise TypeError(f"Unsupported data type: {type(self.data)}") - def downsample(self, agg_type=None): + def downsample(self, agg_type: str | None = None) -> xr.Dataset | pd.DataFrame: """ Downsamples imported data to be more easily manipulated. Data size should be reduced by a factor of m*n, where (m,n) are the @@ -883,7 +911,7 @@ def downsample(self, agg_type=None): Downsampled data """ - def downsample_xr(data, ds, agg_type): + def downsample_xr(data: xr.Dataset, ds: list[int], agg_type: str) -> xr.Dataset: """ Downsample xarray dataset according to aggregation type @@ -929,7 +957,7 @@ def downsample_xr(data, ds, agg_type): data = data.thin(long=ds[0]) return data - def downsample_df(data, _ds, _agg_type): + def downsample_df(data: pd.DataFrame, _ds: list[int], _agg_type: str) -> pd.DataFrame: """ Downsample pandas dataframe Not implemented as it just adds to processing time, @@ -957,7 +985,7 @@ def downsample_df(data, _ds, _agg_type): return downsample_xr(self.data, self.downsample_factors, agg_type) raise TypeError(f"Unsupported data type: {type(self.data)}") - def get_data_col_name(self): + def get_data_col_name(self) -> str: """ Retrieve name of data column (for pd.DataFrame), or variable (for xr.Dataset). Used for when data_name not defined in params. @@ -972,7 +1000,7 @@ def get_data_col_name(self): name """ - def get_data_name_from_df(data): + def get_data_name_from_df(data: pd.DataFrame) -> str: """ Filters out standard columns to extract only data column's name from pd.DataFrame @@ -994,9 +1022,9 @@ def get_data_name_from_df(data): f"More than 1 data column detected, cannot retrieve data \ name! Found columns: {','.join(name)}" ) - return name[0] + return str(name[0]) - def get_data_name_from_xr(data): + def get_data_name_from_xr(data: xr.Dataset) -> str: """ Extracts variable name directly from xr.Dataset metadata @@ -1015,7 +1043,7 @@ def get_data_name_from_xr(data): f"More than 1 data column detected, cannot retrieve data \ name! Found columns: {','.join(name)}" ) - return name[0] + return str(name[0]) logger.debug(f"\tRetrieving data name from {type(self.data)}") # Choose method of extraction based on data type @@ -1025,7 +1053,7 @@ def get_data_name_from_xr(data): return get_data_name_from_xr(self.data) raise TypeError(f"Unsupported data type: {type(self.data)}") - def set_data_col_name(self, new_name): + def set_data_col_name(self, new_name: str) -> xr.Dataset | pd.DataFrame: """ Sets name of data column/data variable @@ -1037,7 +1065,7 @@ def set_data_col_name(self, new_name): Data with variable name changed """ - def set_name_df(data, old_name, new_name): + def set_name_df(data: pd.DataFrame, old_name: str, new_name: str) -> pd.DataFrame: """ Renames data column in pandas dataframe @@ -1056,7 +1084,7 @@ def set_name_df(data, old_name, new_name): # Rename data column to new name return data.rename(columns={old_name: new_name}) - def set_name_xr(data, old_name, new_name): + def set_name_xr(data: xr.Dataset, old_name: str, new_name: str) -> xr.Dataset: """ Renames data variable in xarray dataset diff --git a/meshiphi/dataloaders/scalar/baltic_sea_ice.py b/meshiphi/dataloaders/scalar/baltic_sea_ice.py index 9ccc0f14..992549d0 100644 --- a/meshiphi/dataloaders/scalar/baltic_sea_ice.py +++ b/meshiphi/dataloaders/scalar/baltic_sea_ice.py @@ -1,10 +1,17 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class BalticSeaIceDataLoader(ScalarDataLoader): - def import_data(self, bounds): +class BalticSeaIceDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a Baltic Sea Ice NetCDF file. Renames coordinates to 'lat' and 'long', and renames variable to 'SIC' diff --git a/meshiphi/dataloaders/scalar/bsose_depth.py b/meshiphi/dataloaders/scalar/bsose_depth.py index b0c46b6e..8c5c650c 100644 --- a/meshiphi/dataloaders/scalar/bsose_depth.py +++ b/meshiphi/dataloaders/scalar/bsose_depth.py @@ -1,10 +1,17 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class BSOSEDepthDataLoader(ScalarDataLoader): - def import_data(self, bounds): +class BSOSEDepthDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a BSOSE Depth NetCDF file. Renames coordinates to 'lat' and 'long', and renames variable to diff --git a/meshiphi/dataloaders/scalar/bsose_sea_ice.py b/meshiphi/dataloaders/scalar/bsose_sea_ice.py index da907efe..88216be1 100644 --- a/meshiphi/dataloaders/scalar/bsose_sea_ice.py +++ b/meshiphi/dataloaders/scalar/bsose_sea_ice.py @@ -1,12 +1,18 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class BSOSESeaIceDataLoader(ScalarDataLoader): - def import_data(self, bounds): +class BSOSESeaIceDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a BSOSE Sea Ice NetCDF file. Renames coordinates to 'lat' and 'long', and renames variable to diff --git a/meshiphi/dataloaders/scalar/era5_max_wave_height.py b/meshiphi/dataloaders/scalar/era5_max_wave_height.py index 93ebe2dd..1b7e0d76 100644 --- a/meshiphi/dataloaders/scalar/era5_max_wave_height.py +++ b/meshiphi/dataloaders/scalar/era5_max_wave_height.py @@ -1,13 +1,19 @@ +from __future__ import annotations + from datetime import datetime from os.path import basename +from typing import TYPE_CHECKING import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class ERA5MaxWaveHeightDataLoader(ScalarDataLoader): - def import_data(self, bounds): +class ERA5MaxWaveHeightDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from an ERA5 NetCDF file. Renames coordinates to 'lat' and 'long' @@ -26,7 +32,7 @@ def import_data(self, bounds): # Reduce files to those within date range if self.files is None: raise ValueError("files parameter is required for ERA5MaxWaveHeightDataLoader") - self.files = [ + self.files: list[str] = [ file for file in self.files if time_range[0] diff --git a/meshiphi/dataloaders/scalar/era5_mean_wave_direction.py b/meshiphi/dataloaders/scalar/era5_mean_wave_direction.py index 342a829c..99fdb8f6 100644 --- a/meshiphi/dataloaders/scalar/era5_mean_wave_direction.py +++ b/meshiphi/dataloaders/scalar/era5_mean_wave_direction.py @@ -1,13 +1,19 @@ +from __future__ import annotations + from datetime import datetime from os.path import basename +from typing import TYPE_CHECKING import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class ERA5MeanWaveDirDataLoader(ScalarDataLoader): - def import_data(self, bounds): +class ERA5MeanWaveDirDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from an ERA5 NetCDF file. Renames coordinates to 'lat' and 'long' @@ -26,7 +32,7 @@ def import_data(self, bounds): # Reduce files to those within date range if self.files is None: raise ValueError("files parameter is required for ERA5MeanWaveDirectionDataLoader") - self.files = [ + self.files: list[str] = [ file for file in self.files if time_range[0] diff --git a/meshiphi/dataloaders/scalar/era5_sig_wave_height.py b/meshiphi/dataloaders/scalar/era5_sig_wave_height.py index ada754ea..b4338d7e 100644 --- a/meshiphi/dataloaders/scalar/era5_sig_wave_height.py +++ b/meshiphi/dataloaders/scalar/era5_sig_wave_height.py @@ -1,13 +1,19 @@ +from __future__ import annotations + from datetime import datetime from os.path import basename +from typing import TYPE_CHECKING import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class ERA5SigWaveHeightDataLoader(ScalarDataLoader): - def import_data(self, bounds): +class ERA5SigWaveHeightDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from an ERA5 NetCDF file. Renames coordinates to 'lat' and 'long' @@ -26,7 +32,7 @@ def import_data(self, bounds): # Reduce files to those within date range if self.files is None: raise ValueError("files parameter is required for ERA5SigWaveHeightDataLoader") - self.files = [ + self.files: list[str] = [ file for file in self.files if time_range[0] diff --git a/meshiphi/dataloaders/scalar/era5_wave_period.py b/meshiphi/dataloaders/scalar/era5_wave_period.py index a9d3d0ed..05502892 100644 --- a/meshiphi/dataloaders/scalar/era5_wave_period.py +++ b/meshiphi/dataloaders/scalar/era5_wave_period.py @@ -1,13 +1,19 @@ +from __future__ import annotations + from datetime import datetime from os.path import basename +from typing import TYPE_CHECKING import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class ERA5WavePeriodDataLoader(ScalarDataLoader): - def import_data(self, bounds): +class ERA5WavePeriodDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from an ERA5 NetCDF file. Renames coordinates to 'lat' and 'long' @@ -26,7 +32,7 @@ def import_data(self, bounds): # Reduce files to those within date range if self.files is None: raise ValueError("files parameter is required for ERA5WavePeriodDataLoader") - self.files = [ + self.files: list[str] = [ file for file in self.files if time_range[0] diff --git a/meshiphi/dataloaders/scalar/era5_wind_dir.py b/meshiphi/dataloaders/scalar/era5_wind_dir.py index ff1902a8..1c1adb7e 100644 --- a/meshiphi/dataloaders/scalar/era5_wind_dir.py +++ b/meshiphi/dataloaders/scalar/era5_wind_dir.py @@ -1,14 +1,20 @@ +from __future__ import annotations + from datetime import datetime from os.path import basename +from typing import TYPE_CHECKING import numpy as np import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class ERA5WindDirDataLoader(ScalarDataLoader): - def import_data(self, bounds): +class ERA5WindDirDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from an ERA5 NetCDF file. Renames coordinates to 'lat' and 'long' @@ -27,7 +33,7 @@ def import_data(self, bounds): # Reduce files to those within date range if self.files is None: raise ValueError("files parameter is required for ERA5WindDirDataLoader") - self.files = [ + self.files: list[str] = [ file for file in self.files if time_range[0] diff --git a/meshiphi/dataloaders/scalar/era5_wind_mag.py b/meshiphi/dataloaders/scalar/era5_wind_mag.py index 85f0219f..1514aee4 100644 --- a/meshiphi/dataloaders/scalar/era5_wind_mag.py +++ b/meshiphi/dataloaders/scalar/era5_wind_mag.py @@ -1,14 +1,20 @@ +from __future__ import annotations + from datetime import datetime from os.path import basename +from typing import TYPE_CHECKING import numpy as np import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class ERA5WindMagDataLoader(ScalarDataLoader): - def import_data(self, bounds): +class ERA5WindMagDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from an ERA5 NetCDF file. Renames coordinates to 'lat' and 'long' @@ -27,7 +33,7 @@ def import_data(self, bounds): # Reduce files to those within date range if self.files is None: raise ValueError("files parameter is required for ERA5WindMagDataLoader") - self.files = [ + self.files: list[str] = [ file for file in self.files if time_range[0] diff --git a/meshiphi/dataloaders/scalar/gebco.py b/meshiphi/dataloaders/scalar/gebco.py index b4c2933d..92f23682 100644 --- a/meshiphi/dataloaders/scalar/gebco.py +++ b/meshiphi/dataloaders/scalar/gebco.py @@ -1,10 +1,17 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class GEBCODataLoader(ScalarDataLoader): - def import_data(self, bounds): +class GEBCODataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from GEBCO NetCDF files. Renames coordinates to 'lat' and 'long'. diff --git a/meshiphi/dataloaders/scalar/icenet.py b/meshiphi/dataloaders/scalar/icenet.py index 7fad04eb..aaa3c8e9 100644 --- a/meshiphi/dataloaders/scalar/icenet.py +++ b/meshiphi/dataloaders/scalar/icenet.py @@ -1,15 +1,22 @@ +from __future__ import annotations + import logging from datetime import datetime, timedelta +from typing import TYPE_CHECKING +import pandas as pd import xarray as xr from numpy import datetime64 from pandas import to_timedelta from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class IceNetDataLoader(ScalarDataLoader): - def import_data(self, bounds): +class IceNetDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Reads in data from a IceNet 2 NetCDF file. Renames coordinates to 'lat' and 'long', and renames variable to @@ -24,7 +31,7 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'SIC' """ - def filename_to_datetime(filename): + def filename_to_datetime(filename: str) -> datetime: """ Converts icenet filenames to datetime objects Assumes file names are in the format diff --git a/meshiphi/dataloaders/scalar/modis.py b/meshiphi/dataloaders/scalar/modis.py index 296cb633..2d5e5ddd 100644 --- a/meshiphi/dataloaders/scalar/modis.py +++ b/meshiphi/dataloaders/scalar/modis.py @@ -1,14 +1,20 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + logger = logging.getLogger(__name__) -class MODISDataLoader(ScalarDataLoader): - def import_data(self, bounds): +class MODISDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a MODIS NetCDF file. Renames variable to 'SIC' diff --git a/meshiphi/dataloaders/scalar/scalar_csv.py b/meshiphi/dataloaders/scalar/scalar_csv.py index 2ecdadc1..dfa61b09 100644 --- a/meshiphi/dataloaders/scalar/scalar_csv.py +++ b/meshiphi/dataloaders/scalar/scalar_csv.py @@ -1,10 +1,17 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import pandas as pd from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class ScalarCSVDataLoader(ScalarDataLoader): - def import_data(self, bounds): +class ScalarCSVDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Reads in data from a CSV file. diff --git a/meshiphi/dataloaders/scalar/visual_iced.py b/meshiphi/dataloaders/scalar/visual_iced.py index de20e57c..02ba184b 100644 --- a/meshiphi/dataloaders/scalar/visual_iced.py +++ b/meshiphi/dataloaders/scalar/visual_iced.py @@ -1,14 +1,20 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + logger = logging.getLogger(__name__) -class VisualIcedDataLoader(ScalarDataLoader): - def import_data(self, _bounds): +class VisualIcedDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, _bounds: Boundary) -> xr.Dataset: """ Reads in data from Visual_Ice NetCDF files. Renames coordinates to 'lat' and 'long'. @@ -33,14 +39,14 @@ def import_data(self, _bounds): visual_ice = self.import_from_nc(visual_ice) else: logger.error("File type not supported") - return None + raise ValueError("File type not supported") else: logger.error("Multiple tiff files not supported. Only single tiff file supported") raise ValueError("Multiple tiff files not supported. Only single tiff file supported") return visual_ice - def import_from_nc(self, xarray_dataset): + def import_from_nc(self, xarray_dataset: xr.Dataset) -> xr.Dataset: """ applys transformations need to import a netcdf file @@ -59,7 +65,7 @@ def import_from_nc(self, xarray_dataset): # convert SIC to percentage return xarray_dataset.assign(SIC=lambda x: x.SIC * 100) - def import_from_tiff(self, xarray_dataset): + def import_from_tiff(self, xarray_dataset: xr.Dataset) -> xr.Dataset: """ applys transformations need to import a tiff file diff --git a/meshiphi/dataloaders/vector/abstract_vector.py b/meshiphi/dataloaders/vector/abstract_vector.py index de8c48b0..4e17afb3 100644 --- a/meshiphi/dataloaders/vector/abstract_vector.py +++ b/meshiphi/dataloaders/vector/abstract_vector.py @@ -2,7 +2,7 @@ import logging from abc import abstractmethod -from typing import Any +from typing import Any, cast import numpy as np import pandas as pd @@ -135,7 +135,7 @@ def __init__(self, bounds: Boundary, params: dict[str, Any]) -> None: self.data = self.trim_datapoints(bounds) @abstractmethod - def import_data(self, bounds): + def import_data(self, bounds: Boundary) -> xr.Dataset | pd.DataFrame: """ User defined method for importing data from files, or even generating data from scratch @@ -154,7 +154,7 @@ def import_data(self, bounds): Downsampling and reprojecting happen in __init__() method """ - def add_default_params(self, params): + def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for all scalar dataloaders. This function should be overloaded to include any extra params for a specific dataloader @@ -202,7 +202,9 @@ def add_default_params(self, params): return params - def add_mag_dir(self, data=None, data_names=None): + def add_mag_dir( + self, data: xr.Dataset | pd.DataFrame | None = None, data_names: list[str] | None = None + ) -> xr.Dataset | pd.DataFrame: """ Adds magnitude and direction variables/columns to data for easier retrieval of value @@ -223,7 +225,7 @@ def add_mag_dir(self, data=None, data_names=None): values for each. """ - def add_mag_dir_to_df(data, names): + def add_mag_dir_to_df(data: pd.DataFrame, names: list[str]) -> pd.DataFrame: """ Adds magnitude and direction columns to pd.DataFrame @@ -238,7 +240,7 @@ def add_mag_dir_to_df(data, names): data["_direction"] = np.arctan2(data[y], data[x]) return data - def add_mag_dir_to_xr(data, names): + def add_mag_dir_to_xr(data: xr.Dataset, names: list[str]) -> xr.Dataset: """ Adds magnitude and direction columns to xr.Dataset @@ -269,7 +271,9 @@ def add_mag_dir_to_xr(data, names): return add_mag_dir_to_xr(data, names) raise TypeError(f"Unsupported data type: {type(data)}") - def calculate_coverage(self, bounds, data=None): + def calculate_coverage( + self, bounds: Boundary, data: xr.Dataset | pd.DataFrame | None = None + ) -> float: """ Calculates percentage of boundary covered by dataset @@ -286,7 +290,7 @@ def calculate_coverage(self, bounds, data=None): Decimal fraction of boundary covered by the dataset """ - def calculate_coverage_from_df(bounds, data): + def calculate_coverage_from_df(bounds: Boundary, data: pd.DataFrame) -> float: data = data.dropna().reset_index() # If empty dataframe, 0% coverage if data.empty or data.lat.size == 0 or data.long.size == 0: @@ -304,9 +308,9 @@ def calculate_coverage_from_df(bounds, data): overlap_area = data_polygon.intersection(bounds_polygon).area total_area = bounds_polygon.area - return overlap_area / total_area + return cast("float", overlap_area / total_area) - def calculate_coverage_from_xr(bounds, data): + def calculate_coverage_from_xr(bounds: Boundary, data: xr.Dataset) -> float: # Remove all NaN columns/rows data = data.dropna(dim="lat", how="all") data = data.dropna(dim="long", how="all") @@ -328,7 +332,7 @@ def calculate_coverage_from_xr(bounds, data): overlap_area = data_polygon.intersection(bounds_polygon).area total_area = bounds_polygon.area - return overlap_area / total_area + return cast("float", overlap_area / total_area) # Use self.data if not no explicit dataset specified if data is None: @@ -340,7 +344,9 @@ def calculate_coverage_from_xr(bounds, data): return calculate_coverage_from_xr(bounds, data) raise TypeError(f"Unsupported data type: {type(self.data)}") - def trim_datapoints(self, bounds, data=None): + def trim_datapoints( + self, bounds: Boundary, data: xr.Dataset | pd.DataFrame | None = None + ) -> xr.Dataset | pd.DataFrame: """ Trims datapoints from self.data within boundary defined by 'bounds'. self.data can be pd.DataFrame or xr.Dataset @@ -353,7 +359,7 @@ def trim_datapoints(self, bounds, data=None): Trimmed dataset in same format as self.data """ - def trim_datapoints_from_df(data, bounds): + def trim_datapoints_from_df(data: pd.DataFrame, bounds: Boundary) -> pd.DataFrame: """ Extracts data from a pd.DataFrame """ @@ -382,7 +388,7 @@ def trim_datapoints_from_df(data, bounds): # Return column of data from within bounds return data.loc[mask] - def trim_datapoints_from_xr(data, bounds): + def trim_datapoints_from_xr(data: xr.Dataset, bounds: Boundary) -> xr.Dataset: """ Extracts data from a xr.Dataset using optimized spatial indexing """ @@ -444,7 +450,13 @@ def trim_datapoints_from_xr(data, bounds): return trim_datapoints_from_xr(data, bounds) raise TypeError(f"Unsupported data type: {type(data)}") - def get_value(self, bounds, agg_type=None, skipna=True, data=None): + def get_value( + self, + bounds: Boundary, + agg_type: str | None = None, + skipna: bool = True, + data: xr.Dataset | pd.DataFrame | None = None, + ) -> dict[str, float]: """ Retrieve aggregated value from within bounds @@ -465,7 +477,13 @@ def get_value(self, bounds, agg_type=None, skipna=True, data=None): ValueError: aggregation type not in list of available methods """ - def get_value_from_df(dps, variable_names, bounds, agg_type, skipna): + def get_value_from_df( + dps: pd.DataFrame, + variable_names: list[str], + bounds: Boundary, + agg_type: str, + skipna: bool, + ) -> list[float]: """ Aggregates a value from a pd.Series. @@ -516,7 +534,13 @@ def get_value_from_df(dps, variable_names, bounds, agg_type, skipna): return values - def get_value_from_xr(dps, variable_names, bounds, agg_type, skipna): + def get_value_from_xr( + dps: xr.Dataset, + variable_names: list[str], + bounds: Boundary, + agg_type: str, + skipna: bool, + ) -> list[float]: """ Aggregates a value from a xr.DataArray. @@ -588,7 +612,13 @@ def get_value_from_xr(dps, variable_names, bounds, agg_type, skipna): # Put in dict to map variable to values return {self.data_name_list[i]: values[i] for i in range(len(self.data_name_list))} - def get_hom_condition(self, bounds, splitting_conds, _agg_type="MEAN", data=None): + def get_hom_condition( + self, + bounds: Boundary, + splitting_conds: dict[str, Any], + _agg_type: str = "MEAN", + data: xr.Dataset | pd.DataFrame | None = None, + ) -> str: """ Retrieves homogeneity condition of data within boundary. @@ -660,7 +690,13 @@ def get_hom_condition(self, bounds, splitting_conds, _agg_type="MEAN", data=None return hom_type - def reproject(self, in_proj="EPSG:4326", out_proj="EPSG:4326", x_col="lat", y_col="long"): + def reproject( + self, + in_proj: str = "EPSG:4326", + out_proj: str = "EPSG:4326", + x_col: str = "lat", + y_col: str = "long", + ) -> xr.Dataset | pd.DataFrame: """ Reprojects data using pyProj.Transformer self.data can be pd.DataFrame or xr.Dataset @@ -684,7 +720,9 @@ def reproject(self, in_proj="EPSG:4326", out_proj="EPSG:4326", x_col="lat", y_co replacing 'x_col' and 'y_col' """ - def reproject_df(data, in_proj, out_proj, x_col, y_col): + def reproject_df( + data: pd.DataFrame, in_proj: str, out_proj: str, x_col: str, y_col: str + ) -> pd.DataFrame: """ Reprojects a pandas dataframe @@ -724,7 +762,14 @@ def reproject_df(data, in_proj, out_proj, x_col, y_col): return data - def reproject_xr(data, in_proj, out_proj, x_col, y_col, fast=False): + def reproject_xr( + data: xr.Dataset, + in_proj: str, + out_proj: str, + x_col: str, + y_col: str, + fast: bool = False, + ) -> xr.Dataset: """ Reprojects a xr.Dataset @@ -784,7 +829,7 @@ def reproject_xr(data, in_proj, out_proj, x_col, y_col, fast=False): ) raise TypeError(f"Unsupported data type: {type(self.data)}") - def downsample(self, agg_type=None): + def downsample(self, agg_type: str | None = None) -> xr.Dataset | pd.DataFrame: """ Downsamples imported data to be more easily manipulated. Data size should be reduced by a factor of m*n, where (m,n) are the @@ -801,7 +846,7 @@ def downsample(self, agg_type=None): Downsampled data """ - def downsample_xr(data, ds, agg_type): + def downsample_xr(data: xr.Dataset, ds: list[int], agg_type: str) -> xr.Dataset: """ Downsample xarray dataset according to aggregation type @@ -847,7 +892,7 @@ def downsample_xr(data, ds, agg_type): data = data.thin(long=ds[0]) return data - def downsample_df(data, _ds, _agg_type): + def downsample_df(data: pd.DataFrame, _ds: list[int], _agg_type: str) -> pd.DataFrame: """ Downsample pandas dataframe Not implemented as it just adds to processing time, @@ -875,7 +920,7 @@ def downsample_df(data, _ds, _agg_type): return downsample_xr(self.data, self.downsample_factors, agg_type) raise TypeError(f"Unsupported data type: {type(self.data)}") - def get_data_col_name(self): + def get_data_col_name(self) -> str: """ Retrieve name of data column (for pd.DataFrame), or variable (for xr.Dataset). Used for when data_name not defined in params. @@ -886,7 +931,7 @@ def get_data_col_name(self): Name of data columns, comma seperated """ - def get_data_names_from_df(data): + def get_data_names_from_df(data: pd.DataFrame) -> str: """ Filters out standard columns to extract only data column's name """ @@ -898,7 +943,7 @@ def get_data_names_from_df(data): # Turn into comma seperated string and return return ",".join(data_names) - def get_data_names_from_xr(data): + def get_data_names_from_xr(data: xr.Dataset) -> str: """ Extracts variable name directly from xr.Dataset metadata """ @@ -915,7 +960,7 @@ def get_data_names_from_xr(data): return get_data_names_from_xr(self.data) raise TypeError(f"Unsupported data type: {type(self.data)}") - def get_data_col_name_list(self): + def get_data_col_name_list(self) -> list[str]: """ Retrieve names of data columns (for pd.DataFrame), or variable (for xr.Dataset). Used for when data_name not defined in params. @@ -926,7 +971,7 @@ def get_data_col_name_list(self): """ return self.get_data_col_name().split(",") - def set_data_col_name(self, new_names): + def set_data_col_name(self, new_names: list[str]) -> xr.Dataset | pd.DataFrame: """ Sets name of data column/data variables from a comma-seperated string @@ -940,14 +985,14 @@ def set_data_col_name(self, new_names): Data with variable name changed """ - def set_names_df(data, name_dict): + def set_names_df(data: pd.DataFrame, name_dict: dict[str, str]) -> pd.DataFrame: """ Renames data columns in pandas dataframe """ # Rename data column to new name return data.rename(columns=name_dict) - def set_names_xr(data, name_dict): + def set_names_xr(data: xr.Dataset, name_dict: dict[str, str]) -> xr.Dataset: """ Renames data variables in xarray dataset """ @@ -971,7 +1016,7 @@ def set_names_xr(data, name_dict): return set_names_xr(self.data, name_dict) raise TypeError(f"Unsupported data type: {type(self.data)}") - def set_data_col_name_list(self, new_names): + def set_data_col_name_list(self, new_names: list[str]) -> xr.Dataset | pd.DataFrame: """ Sets name of data column/data variables from a list of strings. Also updates self.data_name_list with new names from list @@ -996,14 +1041,19 @@ def set_data_col_name_list(self, new_names): raise TypeError( f"'new_names' must be list of 'str'. Currently {sum(str_items)} / 2 are strings!" ) - new_data_name = ",".join(new_names) # Set names logger.info(f"\tSetting data names to {new_names}") self.data_name_list = new_names - return self.set_data_col_name(new_data_name) - - def calc_curl(self, bounds, data=None, collapse=True, agg_type="MAX"): + return self.set_data_col_name(new_names) + + def calc_curl( + self, + bounds: Boundary, + data: xr.Dataset | pd.DataFrame | None = None, + collapse: bool = True, + agg_type: str = "MAX", + ) -> float | xr.Dataset | pd.DataFrame: """ Calculates the curl of vectors in a cellbox @@ -1057,7 +1107,13 @@ def calc_curl(self, bounds, data=None, collapse=True, agg_type="MAX"): # Else return field return curl - def calc_dmag(self, bounds, data=None, collapse=True, agg_type="MEAN"): + def calc_dmag( + self, + bounds: Boundary, + data: xr.Dataset | pd.DataFrame | None = None, + collapse: bool = True, + agg_type: str = "MEAN", + ) -> float | xr.Dataset | pd.DataFrame: """ Calculates the dmag of vectors in a cellbox. dmag is defined as being the difference in magnitudes between @@ -1111,7 +1167,9 @@ def calc_dmag(self, bounds, data=None, collapse=True, agg_type="MEAN"): return d_mag @staticmethod - def _create_vector_meshgrid(data, data_name_list): + def _create_vector_meshgrid( + data: xr.Dataset | pd.DataFrame, data_name_list: list[str] + ) -> np.ndarray[Any, Any]: """ Creates a np.meshgrid containing 2D vectors from a pd.DataFrame @@ -1128,20 +1186,23 @@ def _create_vector_meshgrid(data, data_name_list): """ - def meshgrid_from_df(data, data_name_list): + def meshgrid_from_df(data: pd.DataFrame, data_name_list: list[str]) -> np.ndarray[Any, Any]: # Manipulate into meshgrid of 2D vectors x, y = data_name_list # Fields of each vector component vector_x_field = data.pivot(index="lat", columns="long", values=x) vector_y_field = data.pivot(index="lat", columns="long", values=y) # Combine into field of vectors - return np.swapaxes(np.stack((vector_x_field, vector_y_field), axis=-1), 0, 1) + return cast( + "np.ndarray[Any, Any]", + np.swapaxes(np.stack((vector_x_field, vector_y_field), axis=-1), 0, 1), + ) - def meshgrid_from_xr(data, data_name_list): + def meshgrid_from_xr(data: xr.Dataset, data_name_list: list[str]) -> np.ndarray[Any, Any]: # Extract out each variable and combine as tuple data_arrays = [data[name].values for name in data_name_list] # Zip them together to make 2D array of n-dimensional vectors - return np.dstack(data_arrays) + return cast("np.ndarray[Any, Any]", np.dstack(data_arrays)) if isinstance(data, pd.core.frame.DataFrame): return meshgrid_from_df(data, data_name_list) diff --git a/meshiphi/mesh_generation/aggregated_cellbox.py b/meshiphi/mesh_generation/aggregated_cellbox.py index 5dddbf3d..3874e2dc 100644 --- a/meshiphi/mesh_generation/aggregated_cellbox.py +++ b/meshiphi/mesh_generation/aggregated_cellbox.py @@ -30,7 +30,7 @@ def from_json(cls, cellbox_json: dict[str, Any]) -> AggregatedCellBox: """ cellbox_id = cellbox_json["id"] - def load_bounds(cellbox_json): + def load_bounds(cellbox_json: dict[str, Any]) -> Boundary: shape = shapely.wkt.loads(cellbox_json["geometry"]) # Take case where crossing antimeridian if shape.geom_type == "MultiPolygon": @@ -59,8 +59,8 @@ def load_bounds(cellbox_json): return Boundary(lat_range, long_range) - def load_agg_data(cellbox_json): - dict_obj = {} + def load_agg_data(cellbox_json: dict[str, Any]) -> dict[str, Any]: + dict_obj: dict[str, Any] = {} for key in cellbox_json: if key not in ["geometry", "cx", "cy", "dcx", "dcy", "id"]: dict_obj[key] = cellbox_json[key] diff --git a/meshiphi/mesh_generation/boundary.py b/meshiphi/mesh_generation/boundary.py index 93b88321..085e4df9 100644 --- a/meshiphi/mesh_generation/boundary.py +++ b/meshiphi/mesh_generation/boundary.py @@ -2,10 +2,13 @@ from datetime import datetime, timedelta from math import asin, cos, radians, sin, sqrt -from typing import Any +from typing import TYPE_CHECKING, Any from shapely import MultiPolygon, wkt +if TYPE_CHECKING: + from shapely.geometry import Polygon + class Boundary: """ @@ -66,7 +69,7 @@ def from_poly_string(cls, poly_string: str) -> Boundary: # pos_coords on +180 side of antimeridian # neg_coords on -180 side of antimeridian - pos_coords, neg_coords = [coord_string.split(",") for coord_string in coord_strings] + pos_coords, neg_coords = (coord_string.split(",") for coord_string in coord_strings) # Remove leading whitespace from WKT elements pos_coords = [pc.lstrip() for pc in pos_coords] neg_coords = [nc.lstrip() for nc in neg_coords] @@ -227,7 +230,7 @@ def validate_bounds( ) # Functions used for getting data from a cellbox - def getcx(self): + def getcx(self) -> float: """ returns x-position of the centroid of the cellbox @@ -242,7 +245,7 @@ def getcx(self): return cx - def getcy(self): + def getcy(self) -> float: """ returns y-position of the centroid of the cellbox @@ -252,7 +255,7 @@ def getcy(self): """ return self.lat_range[0] + self.get_height() / 2 - def get_height(self): + def get_height(self) -> float: """ returns height of the cellbox @@ -262,7 +265,7 @@ def get_height(self): """ return self.lat_range[1] - self.lat_range[0] - def get_width(self): + def get_width(self) -> float: """ returns width of the cellbox @@ -340,7 +343,7 @@ def get_time_max(self) -> str: return self.time_range[1] - def get_bounds(self): + def get_bounds(self) -> list[list[float]]: """ returns the bounds of this cellbox @@ -358,7 +361,7 @@ def get_bounds(self): ], ] - def calc_size(self): + def calc_size(self) -> float: """ Calculate the great circle distance (in meters) between two points on the earth (specified in decimal degrees) @@ -383,7 +386,7 @@ def calc_size(self): # Divide by sqrt(2) to get 'square' side length return m / sqrt(2) - def to_polygon(self): + def to_polygon(self) -> Polygon | MultiPolygon: """ Creates a shapely polygon from the extent of the boundary. Will be a rectangle in mercator projection. @@ -438,7 +441,7 @@ def to_polygon(self): polygon = MultiPolygon([polygon_1, polygon_2]) return polygon - def to_poly_string(self): + def to_poly_string(self) -> str: """ Creates a string representation of the polygon from the extent of the boundary. Will be a rectangle in mercator projection. @@ -448,9 +451,9 @@ def to_poly_string(self): String representation of the shapely polygon with corners at the min/max lat/long values of this boundary """ - return self.to_polygon().wkt + return str(self.to_polygon().wkt) - def split(self): + def split(self) -> list[Boundary]: """ Splits the boundary into four equal parts. @@ -486,7 +489,7 @@ def split(self): ), ] - def __str__(self): + def __str__(self) -> str: lat_range = "lat range :[" + str(self.get_lat_min()) + "," + str(self.get_lat_max()) + "]" long_range = ( "long range :[" + str(self.get_long_min()) + "," + str(self.get_long_max()) + "]" diff --git a/meshiphi/mesh_generation/cellbox.py b/meshiphi/mesh_generation/cellbox.py index ce7b8d2e..1683390f 100644 --- a/meshiphi/mesh_generation/cellbox.py +++ b/meshiphi/mesh_generation/cellbox.py @@ -46,7 +46,7 @@ class CellBox: """ - def __init__(self, bounds, id): + def __init__(self, bounds: Boundary, id: Any) -> None: """ Args: @@ -55,14 +55,14 @@ def __init__(self, bounds, id): """ # Box information relative to bottom left self.bounds = bounds - self.parent = None + self.parent: CellBox | None = None self.minimum_datapoints = 10 self.split_depth = 0 - self.data_source = None + self.data_source: list[Metadata] | None = None self.id = id ######## setters and getters ######## - def set_minimum_datapoints(self, minimum_datapoints): + def set_minimum_datapoints(self, minimum_datapoints: int) -> None: """ set the minimum number of data contained within CellBox boundaries """ @@ -70,7 +70,7 @@ def set_minimum_datapoints(self, minimum_datapoints): raise ValueError("CellBox: minimum number of data contained can not be negative") self.minimum_datapoints = minimum_datapoints - def set_data_source(self, data_source): + def set_data_source(self, data_source: list[Metadata]) -> None: """ a method that sets the data source of the cellbox ( which includes the data loaders, splitting conditions and aggregation type) @@ -81,7 +81,7 @@ def set_data_source(self, data_source): """ self.data_source = data_source - def set_parent(self, parent): + def set_parent(self, parent: CellBox) -> None: """ set the parent CellBox, which is the bigger CellBox that conains this CellBox Args: @@ -89,7 +89,7 @@ def set_parent(self, parent): """ self.parent = parent - def set_split_depth(self, split_depth): + def set_split_depth(self, split_depth: int) -> None: """ set the split depth of a CellBox, which represents is the number of times the CellBox has been split to reach it's current size. @@ -98,31 +98,31 @@ def set_split_depth(self, split_depth): raise ValueError("CellBox: split depth can not be negative") self.split_depth = split_depth - def set_id(self, id): + def set_id(self, id: Any) -> None: """ method ssts cellbox id """ self.id = id - def set_bounds(self, bounds): + def set_bounds(self, bounds: Boundary) -> None: """ Set the boundary of this cellbox """ self.bounds = bounds - def get_id(self): + def get_id(self) -> Any: """ method returns cellbox cell id """ return self.id - def get_minimum_datapoints(self): + def get_minimum_datapoints(self) -> int: """ get the minimum number of data contained within CellBox boundaries """ return self.minimum_datapoints - def get_data_source(self): + def get_data_source(self) -> list[Metadata] | None: """ a method that gets the data source of the cellbox (the data loaders, splitting conditions and aggregation type) @@ -132,19 +132,19 @@ def get_data_source(self): """ return self.data_source - def get_parent(self): + def get_parent(self) -> CellBox | None: """ get the parent CellBox, which is the bigger CellBox that conains this CellBox """ return self.parent - def get_bounds(self): + def get_bounds(self) -> Boundary: """ get the spatial and temporal bounds (lat range, long range and time range) of this cellbox """ return self.bounds - def get_split_depth(self): + def get_split_depth(self) -> int: """ get the split depth of a CellBox, which represents is the number of times the CellBox has been split to reach it's current size. @@ -177,7 +177,7 @@ def contains_point(self, lat: float, long: float) -> bool: ###################################################### # methods used for splitting a cellbox - def should_split(self, stop_index): + def should_split(self, stop_index: int) -> bool: """ determines if a cellbox should be split based on the homogeneity condition of each data type contained within. The homogeneity condition @@ -200,11 +200,14 @@ def should_split(self, stop_index): bool: True if the splitting_conditions of this CellBox will result in the CellBox being split. """ - hom_conditions = [] + hom_conditions: list[str] = [] current_data_source = None + data_source = self.get_data_source() + if data_source is None: + return False for index in range(stop_index): - current_data_source = self.get_data_source()[index] + current_data_source = data_source[index] data_loader = current_data_source.get_data_loader() data_subset = current_data_source.get_data_subset() for splitting_cond in current_data_source.get_splitting_conditions(): @@ -218,7 +221,7 @@ def should_split(self, stop_index): return False return hom_conditions.count("CLR") != len(hom_conditions) - def should_split_breadth_first(self): + def should_split_breadth_first(self) -> bool: """ determines if a cellbox should be split based on the homogeneity condition of each data type contained within. The homogeneity condition @@ -238,7 +241,7 @@ def should_split_breadth_first(self): should_split (bool): True if the splitting_conditions of this CellBox will result in the CellBox being split. """ - hom_conditions = [] + hom_conditions: list[str] = [] if self.data_source is None: return False for current_data_source in self.data_source: @@ -256,7 +259,7 @@ def should_split_breadth_first(self): return False return hom_conditions.count("CLR") != len(hom_conditions) - def split(self, start_id): + def split(self, start_id: int) -> list[CellBox]: """ splits the current cellbox into 4 corners, returns as a list of cellbox objects. @@ -270,11 +273,14 @@ def split(self, start_id): split_boxes = self.create_splitted_cell_boxes(start_id) # set CellBox split_depth, data_source and parent + data_source = self.get_data_source() + if data_source is None: + return split_boxes for split_box in split_boxes: split_box.set_split_depth(self.get_split_depth() + 1) # Create metadata with data subset split_box_data_sources = [] - for source in self.get_data_source(): + for source in data_source: # Extract data for each new cellbox data_subset = source.data_loader.trim_datapoints( split_box.bounds, data=source.data_subset @@ -291,7 +297,7 @@ def split(self, start_id): split_box.set_parent(self) return split_boxes - def create_splitted_cell_boxes(self, index): + def create_splitted_cell_boxes(self, index: int) -> list[CellBox]: """ method that creates 4 splitted cellbox """ @@ -333,7 +339,7 @@ def create_splitted_cell_boxes(self, index): return [north_west, north_east, south_west, south_east] - def aggregate(self): + def aggregate(self) -> AggregatedCellBox: """ aggregates CellBox data using the associated data_sources' aggregate type (ex. MEAN, MAX) and returns AggregatedCellBox object @@ -342,7 +348,10 @@ def aggregate(self): AggregatedCellbox: object contains the aggregated data within cellbox bounds. """ agg_dict: dict[str, Any] = {} - for source in self.get_data_source(): + data_source = self.get_data_source() + if data_source is None: + raise ValueError("Cannot aggregate cellbox with no data source") + for source in data_source: loader = source.get_data_loader() data_subset = source.get_data_subset() @@ -359,7 +368,10 @@ def aggregate(self): # if the agg_value empty and get_value_fill_type is parent, then use the parent bounds while parent is not None and np.isnan(agg_value[data_name]): # Search through parent metadata to find match - for parent_source in parent.get_data_source(): + parent_data_source_list = parent.get_data_source() + if parent_data_source_list is None: + break + for parent_source in parent_data_source_list: if parent_source.get_data_loader() == source.get_data_loader(): break # If no match found @@ -386,7 +398,9 @@ def aggregate(self): self.deallocate_cellbox() return agg_cellbox - def check_vector_data(self, source, loader, agg_value, data_name): + def check_vector_data( + self, source: Metadata, loader: Any, agg_value: dict[str, Any], data_name: str + ) -> dict[str, Any]: """ method that checks if the vector data is None and calls the parent get value """ @@ -398,7 +412,10 @@ def check_vector_data(self, source, loader, agg_value, data_name): # if the agg_value empty and get_value_fill_type is parent, then use the parent bounds while parent is not None and np.isnan(agg_value[name]): # Search through parent metadata to find match - for parent_source in parent.get_data_source(): + parent_data_source = parent.get_data_source() + if parent_data_source is None: + break + for parent_source in parent_data_source: if parent_source.get_data_loader() == source.get_data_loader(): break # If no match found @@ -413,20 +430,24 @@ def check_vector_data(self, source, loader, agg_value, data_name): agg_value[data_name] = source.get_value_fill_type() return agg_value - def deallocate_cellbox(self): + def deallocate_cellbox(self) -> None: """ Method to free up the memory space allocated by the cellbox """ - for source in self.get_data_source(): + data_source = self.get_data_source() + if data_source is None: + return + for source in data_source: loader = source.get_data_loader() del loader del source parent = self.parent # free up the memory space used by the parent cellboxes chain while isinstance(parent, CellBox): - if parent.get_data_source() is None: + parent_data_source = parent.get_data_source() + if parent_data_source is None: break - for source in parent.get_data_source(): + for source in parent_data_source: loader = source.get_data_loader() del loader del source diff --git a/meshiphi/mesh_generation/direction.py b/meshiphi/mesh_generation/direction.py index 5e01255e..c91bca41 100644 --- a/meshiphi/mesh_generation/direction.py +++ b/meshiphi/mesh_generation/direction.py @@ -1,14 +1,14 @@ class Direction: - north_east = 1 - east = 2 - south_east = 3 - south = 4 - south_west = -1 - west = -2 - north_west = -3 - north = -4 + north_east: int = 1 + east: int = 2 + south_east: int = 3 + south: int = 4 + south_west: int = -1 + west: int = -2 + north_west: int = -3 + north: int = -4 - def __init__(self): + def __init__(self) -> None: self.north_east = 1 self.east = 2 self.south_east = 3 diff --git a/meshiphi/mesh_generation/environment_mesh.py b/meshiphi/mesh_generation/environment_mesh.py index 5df6559b..7d160941 100644 --- a/meshiphi/mesh_generation/environment_mesh.py +++ b/meshiphi/mesh_generation/environment_mesh.py @@ -34,7 +34,7 @@ class EnvironmentMesh: """ @classmethod - def load_from_json(cls, mesh_json: dict) -> EnvironmentMesh: + def load_from_json(cls, mesh_json: dict[str, Any]) -> EnvironmentMesh: """ Constructs an Env.Mesh from a given env-mesh json file to be used by other modules (ex.Vessel Performance Modeller). @@ -100,9 +100,9 @@ def load_from_json(cls, mesh_json: dict) -> EnvironmentMesh: def __init__( self, bounds: Boundary, - agg_cellboxes: list, + agg_cellboxes: list[AggregatedCellBox], neighbour_graph: NeighbourGraph, - config: dict, + config: dict[str, Any], ) -> None: """ Args: @@ -152,7 +152,7 @@ def query_index(self, point: tuple[float, float]) -> str: return str(self.agg_cellboxes[indx].id) raise Exception("Point not within the mesh") - def _split_loc(self, point): + def _split_loc(self, point: tuple[float, float]) -> bool: """ Given a point determine if agg_cellboxes should be split or if at maximum split depth @@ -185,7 +185,7 @@ def _split_loc(self, point): self.split_and_replace(agg_cellbox_wp.id) return True - def split_points(self, points): + def split_points(self, points: list[tuple[float, float]]) -> None: """ Splitting the mesh to maximum split depth around a series of point locations @@ -199,7 +199,7 @@ def split_points(self, points): while splitting_waypoint: splitting_waypoint = self._split_loc(point) - def get_cellbox(self, cellbox_id): + def get_cellbox(self, cellbox_id: str | int) -> AggregatedCellBox: """ returns the cellbox with the given id @@ -214,7 +214,7 @@ def get_cellbox(self, cellbox_id): raise ValueError(f"Cellbox with id {cellbox_id} not found") # Merging meshes - def merge_mesh(self, mesh2): + def merge_mesh(self, mesh2: EnvironmentMesh) -> None: """ merges the given mesh with this mesh. The given mesh is not modified. @@ -266,7 +266,7 @@ def merge_mesh(self, mesh2): west_int_cellboxes = mesh2.get_left_edge_cellboxes() self.tie_western_cellbox_ng(west_ext_cellboxes, west_int_cellboxes) - def validate_merge_compatibility(self, mesh2): + def validate_merge_compatibility(self, mesh2: EnvironmentMesh) -> bool: """ checks if the given mesh is compatible with merging with this mesh @@ -298,9 +298,13 @@ def validate_merge_compatibility(self, mesh2): return False long_diff = l_bounds.get_long_min() - s_bounds.get_long_min() - return long_diff % l_cell_width == 0 + return cast("bool", long_diff % l_cell_width == 0) - def tie_northern_cellbox_ng(self, north_ext_cellboxes, north_int_cellboxes): + def tie_northern_cellbox_ng( + self, + north_ext_cellboxes: list[AggregatedCellBox], + north_int_cellboxes: list[AggregatedCellBox], + ) -> None: """ Joins the neighbour graphs of sets of cellboxes on the northen edge of a boundary between two meshes. @@ -367,7 +371,11 @@ def tie_northern_cellbox_ng(self, north_ext_cellboxes, north_int_cellboxes): for neighbour in north_east_neighbours: self.neighbour_graph.get_graph()[cellbox_s.get_id()]["1"].append(neighbour) - def tie_southern_cellbox_ng(self, south_ext_cellboxes, south_int_cellboxes): + def tie_southern_cellbox_ng( + self, + south_ext_cellboxes: list[AggregatedCellBox], + south_int_cellboxes: list[AggregatedCellBox], + ) -> None: """ Joins the neighbour graphs of sets of cellboxes on the southern edge of a boundary between two meshes. @@ -434,7 +442,11 @@ def tie_southern_cellbox_ng(self, south_ext_cellboxes, south_int_cellboxes): for neighbour in south_east_neighbours: self.neighbour_graph.get_graph()[cellbox_s.get_id()]["3"].append(neighbour) - def tie_eastern_cellbox_ng(self, east_ext_cellboxes, east_int_cellboxes): + def tie_eastern_cellbox_ng( + self, + east_ext_cellboxes: list[AggregatedCellBox], + east_int_cellboxes: list[AggregatedCellBox], + ) -> None: """ Joins the neighbour graphs of sets of cellboxes on the eastern edge of a boundary between two meshes. @@ -501,7 +513,11 @@ def tie_eastern_cellbox_ng(self, east_ext_cellboxes, east_int_cellboxes): for neighbour in north_east_neighbours: self.neighbour_graph.get_graph()[cellbox_s.get_id()]["1"].append(neighbour) - def tie_western_cellbox_ng(self, west_ext_cellboxes, west_int_cellboxes): + def tie_western_cellbox_ng( + self, + west_ext_cellboxes: list[AggregatedCellBox], + west_int_cellboxes: list[AggregatedCellBox], + ) -> None: """ Joins the neighbour graphs of sets of cellboxes on the western edge of a boundary between two meshes. @@ -568,7 +584,7 @@ def tie_western_cellbox_ng(self, west_ext_cellboxes, west_int_cellboxes): for neighbour in north_west_neighbours: self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-3"].append(neighbour) - def get_cellboxes_within_bounds(self, bounds): + def get_cellboxes_within_bounds(self, bounds: Boundary) -> list[AggregatedCellBox]: """ returns the cellboxes within the given bounds. Only cellboxes that are completely within the given bounds are returned. @@ -593,7 +609,7 @@ def get_cellboxes_within_bounds(self, bounds): return cells_within_bounds - def get_cellboxes_north_of_bounds(self, bounds): + def get_cellboxes_north_of_bounds(self, bounds: Boundary) -> list[AggregatedCellBox]: """ returns all cellboxes that are directly north of the given bounds. Only cellboxes which are touching the north edge of the boundary, @@ -617,7 +633,7 @@ def get_cellboxes_north_of_bounds(self, bounds): return north_cellboxes - def get_cellboxes_south_of_bounds(self, bounds): + def get_cellboxes_south_of_bounds(self, bounds: Boundary) -> list[AggregatedCellBox]: south_cellboxes = [] for cellbox in self.agg_cellboxes: @@ -630,7 +646,7 @@ def get_cellboxes_south_of_bounds(self, bounds): return south_cellboxes - def get_cellboxes_east_of_bounds(self, bounds): + def get_cellboxes_east_of_bounds(self, bounds: Boundary) -> list[AggregatedCellBox]: east_cellboxes = [] for cellbox in self.agg_cellboxes: @@ -643,7 +659,7 @@ def get_cellboxes_east_of_bounds(self, bounds): return east_cellboxes - def get_cellboxes_west_of_bounds(self, bounds): + def get_cellboxes_west_of_bounds(self, bounds: Boundary) -> list[AggregatedCellBox]: west_cellboxes = [] for cellbox in self.agg_cellboxes: @@ -656,7 +672,7 @@ def get_cellboxes_west_of_bounds(self, bounds): return west_cellboxes - def get_top_edge_cellboxes(self): + def get_top_edge_cellboxes(self) -> list[AggregatedCellBox]: top_cellboxes = [] mesh_bounds = self.bounds @@ -668,7 +684,7 @@ def get_top_edge_cellboxes(self): return top_cellboxes - def get_bottom_edge_cellboxes(self): + def get_bottom_edge_cellboxes(self) -> list[AggregatedCellBox]: bottom_cellboxes = [] mesh_bounds = self.bounds @@ -680,7 +696,7 @@ def get_bottom_edge_cellboxes(self): return bottom_cellboxes - def get_right_edge_cellboxes(self): + def get_right_edge_cellboxes(self) -> list[AggregatedCellBox]: right_cellboxes = [] mesh_bounds = self.bounds @@ -692,7 +708,7 @@ def get_right_edge_cellboxes(self): return right_cellboxes - def get_left_edge_cellboxes(self): + def get_left_edge_cellboxes(self) -> list[AggregatedCellBox]: left_cellboxes = [] mesh_bounds = self.bounds @@ -704,7 +720,7 @@ def get_left_edge_cellboxes(self): return left_cellboxes - def remove_cellboxes_within_bounds(self, bounds): + def remove_cellboxes_within_bounds(self, bounds: Boundary) -> None: """ removes the cellboxes within the given bounds. Only cellboxes that are completely within the given bounds are removed. @@ -716,7 +732,7 @@ def remove_cellboxes_within_bounds(self, bounds): for cellbox in cells_within_bounds: self.remove_cellbox(cellbox) - def get_max_cellbox_id(self): + def get_max_cellbox_id(self) -> int: """ returns the maximum cellbox id in the mesh @@ -729,7 +745,7 @@ def get_max_cellbox_id(self): max_id = int(cellbox.get_id()) return max_id - def remove_cellbox(self, cellbox): + def remove_cellbox(self, cellbox: AggregatedCellBox) -> None: """ removes the given cellbox from the mesh @@ -740,13 +756,13 @@ def remove_cellbox(self, cellbox): self.neighbour_graph.remove_node_and_update_neighbours(cellbox.get_id()) - def add_cellbox(self, cellbox): + def add_cellbox(self, cellbox: AggregatedCellBox) -> None: """ adds the given cellbox to the mesh """ self.agg_cellboxes.append(cellbox) - def increment_ids(self, increment): + def increment_ids(self, increment: int) -> None: for cellbox in self.agg_cellboxes: cellbox.set_id(str(int(cellbox.get_id()) + increment)) cellbox.agg_data["id"] = str(int(cellbox.get_id()) + increment) @@ -754,7 +770,7 @@ def increment_ids(self, increment): self.neighbour_graph.increment_ids(increment) # Splitting - def sim_split_cellbox(self, cellbox_id): + def sim_split_cellbox(self, cellbox_id: str) -> list[AggregatedCellBox]: """ splits the cellbox with the given id @@ -788,7 +804,7 @@ def sim_split_cellbox(self, cellbox_id): return [cellbox1, cellbox2, cellbox3, cellbox4] - def split_and_replace(self, cellbox_id): + def split_and_replace(self, cellbox_id: str) -> None: """ splits the cellbox with the given id and replaces it with the new cellboxes @@ -996,11 +1012,11 @@ def split_and_replace(self, cellbox_id): def fill_se_neighbour_map( self, - se_neighbour_map, - se_neighbour_id, - south_neighbour_index, - east_neighbour_indx, - ): + se_neighbour_map: dict[str, list[Any]], + se_neighbour_id: int, + south_neighbour_index: list[Any], + east_neighbour_indx: list[Any], + ) -> dict[str, list[Any]]: """ fills the south east neighbour map with the given values Args: @@ -1031,11 +1047,11 @@ def fill_se_neighbour_map( def fill_ne_neighbour_map( self, - ne_neighbour_map, - ne_neighbour_id, - north_neighbour_indx, - east_neighbour_indx, - ): + ne_neighbour_map: dict[str, list[Any]], + ne_neighbour_id: int, + north_neighbour_indx: list[Any], + east_neighbour_indx: list[Any], + ) -> dict[str, list[Any]]: """ fills the north east neighbour map with the given values Args: @@ -1066,11 +1082,11 @@ def fill_ne_neighbour_map( def fill_sw_neighbour_map( self, - sw_neighbour_map, - sw_neighbour_id, - south_neighbour_index, - west_neighbour_indx, - ): + sw_neighbour_map: dict[str, list[Any]], + sw_neighbour_id: int, + south_neighbour_index: list[Any], + west_neighbour_indx: list[Any], + ) -> dict[str, list[Any]]: """ fills the south west neighbour map with the given values Args: @@ -1101,11 +1117,11 @@ def fill_sw_neighbour_map( def fill_nw_neighbour_map( self, - nw_neighbour_map, - nw_neighbour_id, - north_neighbour_indx, - west_neighbour_indx, - ): + nw_neighbour_map: dict[str, list[Any]], + nw_neighbour_id: int, + north_neighbour_indx: list[Any], + west_neighbour_indx: list[Any], + ) -> dict[str, list[Any]]: """ fills the north west neighbour map with the given values Args: @@ -1134,7 +1150,7 @@ def fill_nw_neighbour_map( return nw_neighbour_map - def to_json(self) -> dict: + def to_json(self) -> dict[str, Any]: """ Returns this Mesh converted to a JSON object.\n @@ -1156,12 +1172,12 @@ def to_json(self) -> dict: return cast("dict[str, Any]", json.loads(json.dumps(output, indent=4))) - def to_shapely(self): + def to_shapely(self) -> Any: # GeoDataFrame msh_shapely = gpd.GeoDataFrame(pd.DataFrame(self.cellboxes_to_json())).set_index("id") msh_shapely["geometry"] = msh_shapely["geometry"].apply(wkt.loads) return gpd.GeoDataFrame(msh_shapely, crs="EPSG:4326", geometry="geometry") - def to_png(self, params_file, path): + def to_png(self, params_file: str | None, path: str) -> None: """ exports a mesh and saves to a png file Args: @@ -1185,7 +1201,7 @@ def to_png(self, params_file, path): plotter.save(path) - def to_geojson(self, params_file=None): + def to_geojson(self, params_file: str | None = None) -> dict[str, Any]: """ Returns the cellboxes of this mesh converted to a geoJSON format.\n @@ -1241,7 +1257,7 @@ def to_geojson(self, params_file=None): return geojson - def to_tif(self, params_file, path): + def to_tif(self, params_file: str | None, path: str) -> None: """ generates a representation of the mesh in geotif image format.\n @@ -1275,7 +1291,7 @@ def to_tif(self, params_file, path): "GDAL is required for tif export. Install with: conda install -c conda-forge gdal" ) from err - def generate_samples(): + def generate_samples() -> Any: """ generates uniform lat, long samples covering the image resolution space.\n @@ -1302,7 +1318,7 @@ def generate_samples(): # shape the samples in 2d array (each entry in the array holds sample lat and long return np.reshape(samples, (nlines * ncols, 2)) - def get_sample_value(sample): + def get_sample_value(sample: Any) -> float: """ finds the aggregated cellbox that contains the sample lat and long and returns the value within.\n @@ -1334,7 +1350,7 @@ def get_sample_value(sample): break return value - def get_geo_transform(extent, nlines, ncols): + def get_geo_transform(extent: list[float], nlines: int, ncols: int) -> list[float]: """ transforms from the image coordinate space (row, column) to the georeferenced coordinate space. \n Returns: @@ -1351,7 +1367,7 @@ def get_geo_transform(extent, nlines, ncols): resy = (extent[3] - extent[1]) / nlines return [extent[0], resx, 0, extent[3], 0, -resy] - def load_params(params_file): + def load_params(params_file: str | None) -> dict[str, Any]: """ loads the parameters of the tif export and override the default values.\n @@ -1382,7 +1398,7 @@ def load_params(params_file): params["colour_conf"] = input_params["colour_conf"] return params - def transform_proj(path, params, default_proj): + def transform_proj(path: str, params: dict[str, Any], default_proj: int) -> None: """ method that transforms the generated tif into another projection @@ -1398,7 +1414,7 @@ def transform_proj(path, params, default_proj): # transform to target proj and save gdal.Warp(str(path), str(path), dstSRS=dest.ExportToWkt()) - def set_colour(data, input_file, params): + def set_colour(data: Any, input_file: str, params: dict[str, Any]) -> None: """ method that changes the color of the generated tif instead of using the default greyscale. It defines a scale of RGB colors based on the range of data values(an example file is in unit_tests/resources/colour_conf.txt).\n @@ -1411,8 +1427,8 @@ def set_colour(data, input_file, params): """ fp, color_file = tempfile.mkstemp(suffix=".txt") data = data[~np.isnan(data)] # get rid of the nans before calculating range - _max = np.nanmax(data) - _min = np.nanmin(data) + _max: float = np.nanmax(data) + _min: float = np.nanmin(data) _range = _max - _min inf_color = "255 0 0" # red color for inf value colors = [ @@ -1432,8 +1448,8 @@ def set_colour(data, input_file, params): cmd = "gdaldem color-relief " + input_file + " " + color_file + " " + input_file subprocess.check_call(cmd, shell=True) # remove additional files created while generating the tif - file_path = os.path.abspath(input_file) - dir_path = os.path.split(file_path)[0] + file_path_str = os.path.abspath(input_file) + dir_path = os.path.split(file_path_str)[0] additional_files = ["grid_data", "grid_data.aux.xml"] for file in additional_files: file_path = Path(dir_path + "/" + file) @@ -1482,7 +1498,7 @@ def set_colour(data, input_file, params): set_colour(data, path, params) logger.info(f"Generated GeoTIFF: {path}") - def cellboxes_to_json(self): + def cellboxes_to_json(self) -> list[dict[str, Any]]: """ returns a list of dictionaries containing information about each cellbox in this Mesh. @@ -1514,7 +1530,7 @@ def cellboxes_to_json(self): cellboxes_json.append(cell) return cellboxes_json - def update_cellbox(self, index, values): + def update_cellbox(self, index: int, values: dict[str, Any]) -> None: """ method that adds values to the dict of a cellbox at certain index (to be used by the vessel perf. module to add the perf. metrics to the cellbox) diff --git a/meshiphi/mesh_generation/mesh.py b/meshiphi/mesh_generation/mesh.py index df1e09ad..81aa2274 100644 --- a/meshiphi/mesh_generation/mesh.py +++ b/meshiphi/mesh_generation/mesh.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from meshiphi.mesh_generation.cellbox import CellBox @@ -42,7 +42,7 @@ class Mesh: def __init__( self, boundary: Boundary, - cellboxes: list, + cellboxes: list[CellBox], neighbour_graph: NeighbourGraph, max_split_depth: int, ) -> None: @@ -59,11 +59,11 @@ def __init__( self.cellboxes = cellboxes self.neighbour_graph = neighbour_graph self.max_split_depth = max_split_depth - self.config: dict = {} + self.config: dict[str, Any] = {} # Functions for adding data to the Mesh - def add_data_points(self, data_points): + def add_data_points(self, data_points: Any) -> None: """ takes a dataframe containing geospatial-temporal located values and assigns them to cellboxes within this Mesh. @@ -91,16 +91,16 @@ def add_data_points(self, data_points): # Functions for outputting the Mesh - def set_config(self, config: dict) -> None: + def set_config(self, config: dict[str, Any]) -> None: self.config = config - def get_config(self): + def get_config(self) -> dict[str, Any]: return self.config - def set_cellboxes(self, cellboxes): + def set_cellboxes(self, cellboxes: list[CellBox]) -> None: self.cellboxes = cellboxes - def get_cellboxes(self): + def get_cellboxes(self) -> list[CellBox]: """ returns a list of dictionaries containing information about each cellbox in this Mesh. @@ -136,7 +136,7 @@ def get_cellboxes(self): return_cellboxes.append(cellbox) return return_cellboxes - def get_cellbox(self, long, lat): + def get_cellbox(self, long: float, lat: float) -> CellBox: """ Returns the CellBox which contains a point, given by parameters lat, long @@ -154,8 +154,8 @@ def get_cellbox(self, long, lat): selected_cell.append(cellbox) return selected_cell[0] - def get_bounds(self): + def get_bounds(self) -> Boundary: return self.boundary - def get_max_split_depth(self): + def get_max_split_depth(self) -> int: return self.max_split_depth diff --git a/meshiphi/mesh_generation/mesh_builder.py b/meshiphi/mesh_generation/mesh_builder.py index ef13ae81..4430ec5a 100644 --- a/meshiphi/mesh_generation/mesh_builder.py +++ b/meshiphi/mesh_generation/mesh_builder.py @@ -19,7 +19,7 @@ from __future__ import annotations import logging -from typing import Any +from typing import Any, cast import numpy as np from tqdm import tqdm @@ -45,7 +45,7 @@ class MeshBuilder: """ - def __init__(self, config: dict) -> None: + def __init__(self, config: dict[str, Any]) -> None: """ Constructs a Mesh from a given config file.\n @@ -141,7 +141,7 @@ def __init__(self, config: dict) -> None: self.mesh = Mesh(bounds, cellboxes, self.neighbour_graph, max_split_depth) self.mesh.set_config(config) - def initialize_meta_data(self, bounds, min_datapoints): + def initialize_meta_data(self, bounds: Boundary, min_datapoints: int) -> list[Metadata]: """ Creates a metadata object which holds information about the data sources within a cellbox. @@ -187,7 +187,9 @@ def initialize_meta_data(self, bounds, min_datapoints): return meta_data_list - def initialize_meta_data_subsets(self, bounds, meta_data_list): + def initialize_meta_data_subsets( + self, bounds: Boundary, meta_data_list: list[Metadata] + ) -> list[Metadata]: """ Updates cellbox metadata objects to include data subsets which contain datapoints from only within the cellbox boundaries (as opposed to the @@ -223,7 +225,7 @@ def initialize_meta_data_subsets(self, bounds, meta_data_list): return updated_meta_data_list - def check_value_fill_type(self, data_source): + def check_value_fill_type(self, data_source: dict[str, Any]) -> str | float: def is_float(element: Any) -> bool: if element is None: return False @@ -245,8 +247,10 @@ def is_float(element: Any) -> bool: ) return value_fill_type - def initialize_cellboxes(self, bounds, cell_width, cell_height): - cellboxes: list = [] + def initialize_cellboxes( + self, bounds: Boundary, cell_width: float, cell_height: float + ) -> list[CellBox]: + cellboxes: list[CellBox] = [] (bounds.get_lat_max() - bounds.get_lat_min()) / cell_height lat_range = np.arange(bounds.get_lat_min(), bounds.get_lat_max(), cell_height) @@ -275,7 +279,14 @@ def initialize_cellboxes(self, bounds, cell_width, cell_height): cellboxes.append(cellbox) return cellboxes - def add_dataloader(self, Dataloader, params, bounds=None, name="myDataLoader", min_dp=5): + def add_dataloader( + self, + Dataloader: Any, + params: dict[str, Any], + bounds: Boundary | None = None, + name: str = "myDataLoader", + min_dp: int = 5, + ) -> MeshBuilder: """ Adds a dataloader to a pre-existing mesh by adding to the metadata @@ -316,7 +327,9 @@ def add_dataloader(self, Dataloader, params, bounds=None, name="myDataLoader", m # Add new meta data to list of data sources per cellbox cellbox.set_data_source([*cellbox.get_data_source(), meta_data_obj]) - def validate_bounds(self, bounds, cell_width, cell_height): + return self + + def validate_bounds(self, bounds: Boundary, cell_width: float, cell_height: float) -> None: if (bounds.get_long_max() - bounds.get_long_min()) % 360 % cell_width != 0: raise ValueError( f"The defined longitude region <{bounds.get_long_min()} :{bounds.get_long_max()}> " @@ -329,7 +342,9 @@ def validate_bounds(self, bounds, cell_width, cell_height): f"is not divisable by the initial cell height <{cell_height}>" ) - def check_global_mesh(self, bounds, cellboxes, grid_width): + def check_global_mesh( + self, bounds: Boundary, cellboxes: list[CellBox], grid_width: int + ) -> bool: """ Checks if the mesh is a global one and connects the cellboxes at the minimum longtitude and max longtitude accordingly @@ -386,7 +401,7 @@ def check_global_mesh(self, bounds, cellboxes, grid_width): return is_global_mesh - def to_json(self): + def to_json(self) -> dict[str, Any]: """ Returns this Mesh converted to a JSON object. @@ -399,12 +414,13 @@ def to_json(self): "neighbour_graph": a graph representing the adjacency of CellBoxes within the Mesh } """ - output = {} + output: dict[str, Any] = {} output["config"] = {"mesh_info": self.config} output["cellboxes"] = self.mesh.get_cellboxes() output["neighbour_graph"] = self.neighbour_graph.get_graph() + return output - def split_and_replace(self, cellbox): + def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: """ Replaces a cellbox given by parameter 'cellbox' in this grid with 4 smaller cellboxes representing the four corners of the given cellbox. @@ -539,15 +555,17 @@ def split_and_replace(self, cellbox): # set the original splitted cellbox to None cellboxes[cellbox_indx] = None + return cast("list[CellBox]", split_cellboxes) + ############################## methods to fill the neighbour maps of the splitted cells ######################## def fill_se_map( self, - south_east_indx, - south_neighbour_indx, - east_neighbour_indx, - se_neighbour_map, - ): + south_east_indx: str, + south_neighbour_indx: list[Any], + east_neighbour_indx: list[Any], + se_neighbour_map: dict[int, list[Any]], + ) -> None: """ method that fills the South east neighbours """ @@ -577,11 +595,11 @@ def fill_se_map( def fill_ne_map( self, - north_east_indx, - north_neighbour_indx, - east_neighbour_indx, - ne_neighbour_map, - ): + north_east_indx: str, + north_neighbour_indx: list[Any], + east_neighbour_indx: list[Any], + ne_neighbour_map: dict[int, list[Any]], + ) -> None: """ method that fills the North east neighbours """ @@ -611,11 +629,11 @@ def fill_ne_map( def fill_nw_map( self, - north_west_indx, - north_neighbour_indx, - west_neighbour_indx, - nw_neighbour_map, - ): + north_west_indx: str, + north_neighbour_indx: list[Any], + west_neighbour_indx: list[Any], + nw_neighbour_map: dict[int, list[Any]], + ) -> None: """ method that fills the North west neighbours """ @@ -645,11 +663,11 @@ def fill_nw_map( def fill_sw_neighbour_map( self, - south_west_indx, - south_neighbour_indx, - west_neighbour_indx, - sw_neighbour_map, - ): + south_west_indx: str, + south_neighbour_indx: list[Any], + west_neighbour_indx: list[Any], + sw_neighbour_map: dict[int, list[Any]], + ) -> None: """ method that fills the South west neighbours """ @@ -677,7 +695,7 @@ def fill_sw_neighbour_map( ): sw_neighbour_map[-3].append(indx) - def split_to_depth(self, split_depth): + def split_to_depth(self, split_depth: int) -> None: """ splits all cellboxes in this grid until a maximum split depth is reached, or all cellboxes are homogeneous. @@ -767,7 +785,7 @@ def build_environmental_mesh(self) -> EnvironmentMesh: self.get_config(), ) - def get_config(self) -> dict: + def get_config(self) -> dict[str, Any]: """ returns the config """ diff --git a/meshiphi/mesh_generation/neighbour_graph.py b/meshiphi/mesh_generation/neighbour_graph.py index d589feb2..d81cb69f 100644 --- a/meshiphi/mesh_generation/neighbour_graph.py +++ b/meshiphi/mesh_generation/neighbour_graph.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any +from typing import Any, cast from meshiphi.mesh_generation.direction import Direction @@ -56,7 +56,7 @@ def from_json(cls, ng_json: dict[str, Any]) -> NeighbourGraph: obj.neighbour_graph = neighbour_graph return obj - def increment_ids(self, inc): + def increment_ids(self, inc: int) -> None: """ Increments all ID's within the neighbour_graph by a given int inc @@ -82,13 +82,13 @@ def get_graph(self) -> dict[str, Any]: """ return self.neighbour_graph - def update_neighbour(self, index, direction, neighbours): + def update_neighbour(self, index: str, direction: str, neighbours: list[Any]) -> None: """ updates the neighbour in a certain direction """ self.neighbour_graph[index][direction] = neighbours - def add_neighbour(self, index, direction, neighbour_indx): + def add_neighbour(self, index: str, direction: str, neighbour_indx: Any) -> None: """ adds a neighbour in a certain direction @@ -99,7 +99,7 @@ def add_neighbour(self, index, direction, neighbour_indx): """ self.neighbour_graph[index][direction].append(neighbour_indx) - def remove_node_and_update_neighbours(self, cellbox_index): + def remove_node_and_update_neighbours(self, cellbox_index: str) -> None: """ method that removes a node in the neighbour_graph at a given index. remove_node_from_neighbours should be called first. Args: @@ -112,13 +112,13 @@ def remove_node_and_update_neighbours(self, cellbox_index): self.neighbour_graph.pop(cellbox_index) - def get_neighbours(self, cellbox_indx, direction): + def get_neighbours(self, cellbox_indx: str, direction: str) -> list[Any]: """ returns neighbour in a certain direction """ - return self.neighbour_graph[cellbox_indx][direction] + return cast("list[Any]", self.neighbour_graph[cellbox_indx][direction]) - def add_node(self, index, neighbour_map): + def add_node(self, index: str, neighbour_map: dict[str, list[Any]]) -> None: """ method that adds a node to the neighbour_graph at a given index Args: @@ -127,7 +127,7 @@ def add_node(self, index, neighbour_map): """ self.neighbour_graph[index] = neighbour_map - def remove_node(self, cellbox_index): + def remove_node(self, cellbox_index: str) -> None: """ method that removes a node to the neighbour_graph at a given index Args: @@ -135,7 +135,13 @@ def remove_node(self, cellbox_index): """ self.neighbour_graph.pop(cellbox_index) - def update_neighbours(self, cellbox_indx, new_neighbours_indx, direction, cellboxes): + def update_neighbours( + self, + cellbox_indx: str, + new_neighbours_indx: list[Any], + direction: int, + cellboxes: list[Any], + ) -> None: """ method that updates the neighbour of a certain cellbox in a specific direction. It removes cellbox_indx from the neighbour_map of its neighbours in a specific direction and add new_neighbour_indx @@ -164,7 +170,7 @@ def update_neighbours(self, cellbox_indx, new_neighbours_indx, direction, cellbo if crossing_case != 0: self.neighbour_graph[indx][crossing_case].append(new_neighbours_indx[1]) - def remove_node_from_neighbours(self, cellbox_indx, direction): + def remove_node_from_neighbours(self, cellbox_indx: str, direction: int) -> None: """ method that goes through neighbours in a given direction and remove cellbox_index from their neighbour_maps Args: @@ -186,12 +192,12 @@ def remove_node_from_neighbours(self, cellbox_indx, direction): def update_corner_neighbours( self, - cellbox_indx, - north_west_indx, - north_east_indx, - south_west_indx, - south_east_indx, - ): + cellbox_indx: str, + north_west_indx: Any, + north_east_indx: Any, + south_west_indx: Any, + south_east_indx: Any, + ) -> None: """ method that updates the corner neighbours of cellbox_indx with the given indeces """ @@ -219,7 +225,7 @@ def update_corner_neighbours( south_west_indx ] - def get_neighbour_case_bounds(self, bounds_a, bounds_b): + def get_neighbour_case_bounds(self, bounds_a: Any, bounds_b: Any) -> int: """ Given two bounds (bounds_a, bounds_b) returns a case number representing where the two bounds are touching. @@ -248,48 +254,48 @@ def get_neighbour_case_bounds(self, bounds_a, bounds_b): lat_b = bounds_b.get_lat_min() if (long_a + bounds_a.get_width()) == long_b and (lat_a + bounds_a.get_height()) == lat_b: - return Direction.north_east + return cast("int", Direction.north_east) if ( (long_a + bounds_a.get_width() == long_b) and (lat_b < (lat_a + bounds_a.get_height())) and ((lat_b + bounds_b.get_height()) > lat_a) ): - return Direction.east + return cast("int", Direction.east) if (long_a + bounds_a.get_width()) == long_b and (lat_a == lat_b + bounds_b.get_height()): - return Direction.south_east + return cast("int", Direction.south_east) if ( ((lat_b + bounds_b.get_height()) == lat_a) and ((long_b + bounds_b.get_width()) > long_a) and (long_b < (long_a + bounds_a.get_width())) ): - return Direction.south + return cast("int", Direction.south) if long_a == (long_b + bounds_b.get_width()) and lat_a == (lat_b + bounds_b.get_height()): - return Direction.south_west + return cast("int", Direction.south_west) if ( (long_b + bounds_b.get_width() == long_a) and (lat_b < (lat_a + bounds_a.get_height())) and ((lat_b + bounds_b.get_height()) > lat_a) ): - return Direction.west + return cast("int", Direction.west) if long_a == (long_b + bounds_b.get_width()) and (lat_a + bounds_a.get_height() == lat_b): - return Direction.north_west + return cast("int", Direction.north_west) if ( (lat_b == (lat_a + bounds_a.get_height())) and ((long_b + bounds_b.get_width()) > long_a) and (long_b < (long_a + bounds_a.get_width())) ): - return Direction.north + return cast("int", Direction.north) return 0 # Cells are not neighbours. - def get_neighbour_case(self, cellbox_a, cellbox_b): + def get_neighbour_case(self, cellbox_a: Any, cellbox_b: Any) -> int: """ Given two cellboxes (cellbox_a, cellbox_b) returns a case number representing where the two cellboxes are touching. @@ -318,7 +324,7 @@ def get_neighbour_case(self, cellbox_a, cellbox_b): long_b = cellbox_b.get_bounds().get_long_min() lat_b = cellbox_b.get_bounds().get_lat_min() - def on_global_bound(cellbox_a, cellbox_b): + def on_global_bound(cellbox_a: Any, cellbox_b: Any) -> bool: """ Given two cellboxes (cellbox_a, cellbox_b) returns a boolean representing whether the two cellboxes are touching on the global bound (-180,180). @@ -330,8 +336,10 @@ def on_global_bound(cellbox_a, cellbox_b): Returns: bool: a boolean representing if the two cellboxes are touching on the global bound (-180,180). """ - return (long_a == -180 and cellbox_b.get_bounds().get_long_max() == 180) or ( - long_b == -180 and cellbox_a.get_bounds().get_long_max() == 180 + return cast( + "bool", + (long_a == -180 and cellbox_b.get_bounds().get_long_max() == 180) + or (long_b == -180 and cellbox_a.get_bounds().get_long_max() == 180), ) if self.is_global_mesh() and on_global_bound(cellbox_a, cellbox_b): @@ -340,53 +348,53 @@ def on_global_bound(cellbox_a, cellbox_b): if (long_a + cellbox_a.get_bounds().get_width()) == long_b and ( lat_a + cellbox_a.get_bounds().get_height() ) == lat_b: - return Direction.north_east + return cast("int", Direction.north_east) if ( (long_a + cellbox_a.get_bounds().get_width() == long_b) and (lat_b < (lat_a + cellbox_a.get_bounds().get_height())) and ((lat_b + cellbox_b.get_bounds().get_height()) > lat_a) ): - return Direction.east + return cast("int", Direction.east) if (long_a + cellbox_a.get_bounds().get_width()) == long_b and ( lat_a == lat_b + cellbox_b.get_bounds().get_height() ): - return Direction.south_east + return cast("int", Direction.south_east) if ( ((lat_b + cellbox_b.get_bounds().get_height()) == lat_a) and ((long_b + cellbox_b.get_bounds().get_width()) > long_a) and (long_b < (long_a + cellbox_a.get_bounds().get_width())) ): - return Direction.south + return cast("int", Direction.south) if long_a == (long_b + cellbox_b.get_bounds().get_width()) and lat_a == ( lat_b + cellbox_b.get_bounds().get_height() ): - return Direction.south_west + return cast("int", Direction.south_west) if ( (long_b + cellbox_b.get_bounds().get_width() == long_a) and (lat_b < (lat_a + cellbox_a.get_bounds().get_height())) and ((lat_b + cellbox_b.get_bounds().get_height()) > lat_a) ): - return Direction.west + return cast("int", Direction.west) if long_a == (long_b + cellbox_b.get_bounds().get_width()) and ( lat_a + cellbox_a.get_bounds().get_height() == lat_b ): - return Direction.north_west + return cast("int", Direction.north_west) if ( (lat_b == (lat_a + cellbox_a.get_bounds().get_height())) and ((long_b + cellbox_b.get_bounds().get_width()) > long_a) and (long_b < (long_a + cellbox_a.get_bounds().get_width())) ): - return Direction.north + return cast("int", Direction.north) return 0 # Cells are not neighbours. - def get_global_mesh_neighbour_case(self, cellbox_a, cellbox_b): + def get_global_mesh_neighbour_case(self, cellbox_a: Any, cellbox_b: Any) -> int: """ Given two cellboxes in a *global mesh* (cellbox_a, cellbox_b) returns a case number representing where the two cellboxes are touching. @@ -416,52 +424,52 @@ def get_global_mesh_neighbour_case(self, cellbox_a, cellbox_b): if (long_a + cellbox_a.get_bounds().get_width()) == abs(long_b) and ( lat_a + cellbox_a.get_bounds().get_height() ) == lat_b: - return Direction.north_east + return cast("int", Direction.north_east) if ( (long_a + cellbox_a.get_bounds().get_width() == abs(long_b)) and (lat_b < (lat_a + cellbox_a.get_bounds().get_height())) and ((lat_b + cellbox_b.get_bounds().get_height()) > lat_a) ): - return Direction.east + return cast("int", Direction.east) if (long_a + cellbox_a.get_bounds().get_width()) == abs(long_b) and ( lat_a == lat_b + cellbox_b.get_bounds().get_height() ): - return Direction.south_east + return cast("int", Direction.south_east) if ( ((lat_b + cellbox_b.get_bounds().get_height()) == lat_a) and ((long_b + cellbox_b.get_bounds().get_width()) > long_a) and (long_b < (long_a + cellbox_a.get_bounds().get_width())) ): - return Direction.south + return cast("int", Direction.south) if abs(long_a) == (long_b + cellbox_b.get_bounds().get_width()) and lat_a == ( lat_b + cellbox_b.get_bounds().get_height() ): - return Direction.south_west + return cast("int", Direction.south_west) if ( (long_b + cellbox_b.get_bounds().get_width() == abs(long_a)) and (lat_b < (lat_a + cellbox_a.get_bounds().get_height())) and ((lat_b + cellbox_b.get_bounds().get_height()) > lat_a) ): - return Direction.west + return cast("int", Direction.west) if abs(long_a) == (long_b + cellbox_b.get_bounds().get_width()) and ( lat_a + cellbox_a.get_bounds().get_height() == lat_b ): - return Direction.north_west + return cast("int", Direction.north_west) if ( (lat_b == (lat_a + cellbox_a.get_bounds().get_height())) and ((long_b + cellbox_b.get_bounds().get_width()) > long_a) and (long_b < (long_a + cellbox_a.get_bounds().get_width())) ): - return Direction.north + return cast("int", Direction.north) return 0 # Cells are not neighbours. - def remove_neighbour(self, index, direction, neighbour_index): + def remove_neighbour(self, index: str, direction: str, neighbour_index: Any) -> None: """ remove certain neighbour in a specific direction """ self.neighbour_graph[index][direction].remove(neighbour_index) - def initialise_neighbour_graph(self, cellboxes, grid_width): + def initialise_neighbour_graph(self, cellboxes: list[Any], grid_width: int) -> None: """ initialize the neighbour graph """ @@ -469,15 +477,15 @@ def initialise_neighbour_graph(self, cellboxes, grid_width): cellbox_indx = cellboxes.index(cellbox) neighbour_map = self.initialise_map(cellbox_indx, grid_width, len(cellboxes)) - self.add_node(cellbox_indx, neighbour_map) + self.add_node(str(cellbox_indx), neighbour_map) def initialise_map( self, cellbox_indx: int, grid_width: int, cellboxes_length: int - ) -> dict[int, list[int]]: + ) -> dict[str, list[Any]]: """ initialse the neighbour map of a cellbox with the given cellbox_index """ - neighbour_map: dict[int, list[int]] = { + neighbour_map: dict[Any, list[Any]] = { Direction.north_east: [], Direction.east: [], Direction.south_east: [], @@ -517,11 +525,11 @@ def initialise_map( neighbour_map[Direction.south].append(int(cellbox_indx - grid_width)) return neighbour_map - def set_global_mesh(self, is_global): + def set_global_mesh(self, is_global: bool) -> None: self._is_global_mesh = is_global - def is_global_mesh(self): + def is_global_mesh(self) -> bool: return self._is_global_mesh - def get_neighbour_map(self, _id): - return self.neighbour_graph[_id] + def get_neighbour_map(self, _id: str) -> dict[str, list[Any]]: + return cast("dict[str, list[Any]]", self.neighbour_graph[_id]) From 210e5b96b38197fbea1eec7bafb6b1ef4a4f7701 Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Wed, 14 Jan 2026 17:34:55 +0000 Subject: [PATCH 06/15] further --strict changes --- meshiphi/dataloaders/scalar/amsr.py | 22 +++-- .../scalar/ecmwf_sig_wave_height.py | 40 +++++--- meshiphi/dataloaders/scalar/scalar_grf.py | 32 ++++--- meshiphi/dataloaders/scalar/shape.py | 78 +++++++++------ meshiphi/dataloaders/vector/baltic_current.py | 11 ++- meshiphi/dataloaders/vector/duacs_current.py | 12 ++- .../vector/era5_wave_direction_vector.py | 12 ++- meshiphi/dataloaders/vector/era5_wind.py | 12 ++- .../dataloaders/vector/north_sea_current.py | 11 ++- meshiphi/dataloaders/vector/oras5_current.py | 11 ++- meshiphi/dataloaders/vector/sose.py | 12 ++- meshiphi/dataloaders/vector/vector_csv.py | 11 ++- meshiphi/dataloaders/vector/vector_grf.py | 21 ++++- meshiphi/dataloaders/vector/vector_shape.py | 94 +++++++++++++------ 14 files changed, 261 insertions(+), 118 deletions(-) diff --git a/meshiphi/dataloaders/scalar/amsr.py b/meshiphi/dataloaders/scalar/amsr.py index 94e44c7b..faee3160 100644 --- a/meshiphi/dataloaders/scalar/amsr.py +++ b/meshiphi/dataloaders/scalar/amsr.py @@ -1,13 +1,19 @@ +from __future__ import annotations + import logging from datetime import datetime +from typing import TYPE_CHECKING, Any import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class AMSRDataLoader(ScalarDataLoader): - def add_default_params(self, params): +class AMSRDataLoader(ScalarDataLoader): # type: ignore[misc] + def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Translates 'hemisphere' parameter into values of in_proj and out_proj that pyProj can understand. Also defines x_col and y_col for AMSR data @@ -38,7 +44,7 @@ def add_default_params(self, params): return params - def import_data(self, bounds): + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a AMSR NetCDF file, or folder of files. Drops unnecessary column 'polar_stereographic', and renames variable @@ -53,7 +59,7 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'SIC' """ - def retrieve_date(filename): + def retrieve_date(filename: str) -> str: """ Get date from filename in format: asi-AMSR2-s6250--v.5.4.nc @@ -61,7 +67,7 @@ def retrieve_date(filename): date = filename.split("-")[-2] return f"{date[:4]}-{date[4:6]}-{date[6:]}" - def retrieve_data(filename, date): + def retrieve_data(filename: str, date: str) -> xr.Dataset: """ Read in data as xr.Dataset, create time coordinate """ @@ -69,10 +75,10 @@ def retrieve_data(filename, date): # Add date to data return data.assign_coords(time=date) - data_array = [] + data_array: list[xr.Dataset] = [] if self.files is None: raise ValueError("files parameter is required for AMSRDataLoader") - relevant_files = [] + relevant_files: list[str] = [] # For each file found from config for file in self.files: # If date within boundary @@ -100,7 +106,7 @@ def retrieve_data(filename, date): data = data.rename({"z": "SIC"}) # Limit self.files to only those actually used - self.files = relevant_files + self.files: list[str] = relevant_files # TODO Limit data range before reprojection diff --git a/meshiphi/dataloaders/scalar/ecmwf_sig_wave_height.py b/meshiphi/dataloaders/scalar/ecmwf_sig_wave_height.py index 9e6111fe..c1298d4d 100644 --- a/meshiphi/dataloaders/scalar/ecmwf_sig_wave_height.py +++ b/meshiphi/dataloaders/scalar/ecmwf_sig_wave_height.py @@ -1,14 +1,20 @@ +from __future__ import annotations + import logging import os from datetime import datetime, timedelta +from typing import TYPE_CHECKING import xarray as xr from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class ECMWFSigWaveHeightDataLoader(ScalarDataLoader): - def import_data(self, bounds): +class ECMWFSigWaveHeightDataLoader(ScalarDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a ECMWF GRIB2 file, or folder of files. Extracts Significant wave height @@ -22,12 +28,12 @@ def import_data(self, bounds): Dataset has coordinates 'lat', 'long', and variable 'SIC' """ - def extract_base_date(filename): + def extract_base_date(filename: str) -> datetime: filename = os.path.basename(filename) base_date = filename.split("-")[0] return datetime.strptime(base_date, "%Y%m%d%H0000") - def calculate_timestamp(filename): + def calculate_timestamp(filename: str) -> datetime: """ Get date from filename in format: YYYYMMDDHH0000-wave-fc.grib2 @@ -35,13 +41,15 @@ def calculate_timestamp(filename): # Get filename independant of directory filename = os.path.basename(filename) # Get date of file and forecast timedelta - base_date, forecast_len, _, _ = filename.split("-") + base_date_str, forecast_len_str, _, _ = filename.split("-") # Turn forecast len into number of hours - forecast_len = int(forecast_len[:-1]) + forecast_len_hours = int(forecast_len_str[:-1]) # Calculate - return datetime.strptime(base_date, "%Y%m%d%H0000") + timedelta(hours=forecast_len) + return datetime.strptime(base_date_str, "%Y%m%d%H0000") + timedelta( + hours=forecast_len_hours + ) - def retrieve_data(filename, timestamp): + def retrieve_data(filename: str, timestamp: datetime) -> xr.Dataset: """ Read in data as xr.Dataset, create time coordinate """ @@ -56,8 +64,8 @@ def retrieve_data(filename, timestamp): # Go through reverse alphabetical list until date < start_date found if self.files is None: raise ValueError("files parameter is required for ECMWFSigWaveHeightDataLoader") - self.files = sorted(self.files, reverse=True) - for file in self.files: + sorted_files: list[str] = sorted(self.files, reverse=True) + for file in sorted_files: base_date = extract_base_date(file) if base_date < start_time: closest_date = base_date @@ -66,12 +74,14 @@ def retrieve_data(filename, timestamp): else: raise FileNotFoundError("No file found prior to start time") # Limit files to be read in - self.files = [file for file in self.files if extract_base_date(file) == closest_date] + filtered_files: list[str] = [ + file for file in sorted_files if extract_base_date(file) == closest_date + ] - data_array = [] - relevant_files = [] + data_array: list[xr.Dataset] = [] + relevant_files: list[str] = [] # Limit data to forecasts that cover time boundary - for file in self.files: + for file in filtered_files: timestamp = calculate_timestamp(file) print(start_time, timestamp, end_time) if start_time <= timestamp <= end_time: @@ -93,6 +103,6 @@ def retrieve_data(filename, timestamp): data = data.reindex(lat=data.lat[::-1]) data = data.reset_coords(drop=True) # Limit self.files to only those actually used - self.files = relevant_files + self.files: list[str] = relevant_files return data diff --git a/meshiphi/dataloaders/scalar/scalar_grf.py b/meshiphi/dataloaders/scalar/scalar_grf.py index d72c4b0b..8a052250 100644 --- a/meshiphi/dataloaders/scalar/scalar_grf.py +++ b/meshiphi/dataloaders/scalar/scalar_grf.py @@ -1,12 +1,20 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, cast + import numpy as np import pandas as pd +import xarray as xr # noqa: TC002 from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader from meshiphi.utils import gaussian_random_field +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class ScalarGRFDataLoader(ScalarDataLoader): - def add_default_params(self, params): +class ScalarGRFDataLoader(ScalarDataLoader): # type: ignore[misc] + def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for abstract GRF dataloaders, starting by including defaults for scalar dataloaders. @@ -60,7 +68,7 @@ def add_default_params(self, params): return params - def import_data(self, bounds): + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Creates data in the form of a Gaussian Random Field @@ -76,7 +84,7 @@ def import_data(self, bounds): """ - def grf_to_binary(grf, threshold): + def grf_to_binary(grf: np.ndarray[Any, Any], threshold: float) -> np.ndarray[Any, Any]: """ Creates a mask out of a GRF if params specify a binary output """ @@ -84,9 +92,11 @@ def grf_to_binary(grf, threshold): grf[grf >= threshold] = 1.0 grf[grf < threshold] = 0.0 # Cast 0/1 to False/True - return grf is True + return grf.astype(bool) - def grf_to_scalar(grf, threshold, min_val, max_val): + def grf_to_scalar( + grf: np.ndarray[Any, Any], threshold: list[float], min_val: float, max_val: float + ) -> np.ndarray[Any, Any]: # Bound GRF to threshold limits grf[grf < threshold[0]] = threshold[0] grf[grf >= threshold[1]] = threshold[1] @@ -94,7 +104,7 @@ def grf_to_scalar(grf, threshold, min_val, max_val): grf = grf - np.min(grf) grf = grf / (np.max(grf) - np.min(grf)) # Scale to max/min - return grf * (max_val - min_val) + min_val + return cast("np.ndarray[Any, Any]", grf * (max_val - min_val) + min_val) # Set seed for generation. If not specified, will be 'random' np.random.seed(self.seed) @@ -119,11 +129,9 @@ def grf_to_scalar(grf, threshold, min_val, max_val): raise ValueError("threshold, min, and max parameters are required for scalar GRF") grf = grf_to_scalar(grf, self.threshold, self.min, self.max) # Scale with multiplier and offset - if self.multiplier is None: - self.multiplier = 1.0 - if self.offset is None: - self.offset = 0.0 - grf = self.multiplier * grf + self.offset + multiplier: float = self.multiplier if self.multiplier is not None else 1.0 + offset: float = self.offset if self.offset is not None else 0.0 + grf = multiplier * grf + offset # Set up domain of field lat_array = np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.size) diff --git a/meshiphi/dataloaders/scalar/shape.py b/meshiphi/dataloaders/scalar/shape.py index b2351319..b584f663 100644 --- a/meshiphi/dataloaders/scalar/shape.py +++ b/meshiphi/dataloaders/scalar/shape.py @@ -1,15 +1,22 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING, Any import numpy as np import pandas as pd +import xarray as xr # noqa: TC002 from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + logger = logging.getLogger(__name__) -class ShapeDataLoader(ScalarDataLoader): - def add_default_params(self, params): +class ShapeDataLoader(ScalarDataLoader): # type: ignore[misc] + def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for abstract shape dataloaders, starting by including defaults for scalar dataloaders. @@ -62,7 +69,7 @@ def add_default_params(self, params): return params - def import_data(self, bounds): + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Generates data in the form of an abstract shape, such as circle, checkerboard, or gradient. This method acts like a factory in that @@ -94,7 +101,7 @@ def import_data(self, bounds): # No need to trim data, as was defined by bounds return data.set_index(["lat", "long"]).to_xarray() - def gen_circle(self, bounds): + def gen_circle(self, bounds: Boundary) -> pd.DataFrame: """ Generates a circle within bounds of lat/long min/max. Circle centre and radius can be defined in the config, as well as @@ -111,10 +118,11 @@ def gen_circle(self, bounds): # Set centre as centre of data_grid if none specified if self.radius is None: raise ValueError("radius parameter is required for circle") - if self.centre is None: - self.centre = (None, None) - c_y = self.lat[int(self.ny / 2)] if self.centre[0] is None else self.centre[0] - c_x = self.long[int(self.nx / 2)] if self.centre[1] is None else self.centre[1] + centre: tuple[float | None, float | None] = ( + self.centre if hasattr(self, "centre") else (None, None) + ) + c_y = self.lat[int(self.ny / 2)] if centre[0] is None else centre[0] + c_x = self.long[int(self.nx / 2)] if centre[1] is None else centre[1] # Create vectors for row and col idx's y = np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.ny).reshape(-1, 1) @@ -146,13 +154,14 @@ def gen_circle(self, bounds): # Change boolean values to int dummy_df["dummy_data"] = dummy_df["dummy_data"].astype(int) # Multiply by scaling factor if present - if self.multiplier is None: - self.multiplier = 1.0 - dummy_df["dummy_data"] = dummy_df["dummy_data"] * self.multiplier + multiplier: float = ( + self.multiplier if hasattr(self, "multiplier") and self.multiplier is not None else 1.0 + ) + dummy_df["dummy_data"] = dummy_df["dummy_data"] * multiplier return dummy_df - def gen_gradient(self, bounds): + def gen_gradient(self, bounds: Boundary) -> pd.DataFrame: """ Generates a gradient within bounds of lat/long min/max. Gradient direction can be defined in the config, as well as @@ -168,9 +177,10 @@ def gen_gradient(self, bounds): logger.info("\tCreating gradient of values") # Create 1D gradient - if self.vertical is None: - self.vertical = True # Default to vertical gradient - gradient = np.linspace(0, 1, self.ny) if self.vertical else np.linspace(0, 1, self.nx) + vertical: bool = ( + self.vertical if hasattr(self, "vertical") and self.vertical is not None else True + ) + gradient = np.linspace(0, 1, self.ny) if vertical else np.linspace(0, 1, self.nx) dummy_df = pd.DataFrame(columns=["lat", "long", "dummy_data"]) logger.info("- Generating dataset") @@ -178,7 +188,7 @@ def gen_gradient(self, bounds): for i in range(self.ny): for j in range(self.nx): # Change dummy data depending on which axis to gradient - datum = gradient[i] if self.vertical else gradient[j] + datum = gradient[i] if vertical else gradient[j] # Create a new row, adding datum value row = pd.DataFrame( data={ @@ -192,11 +202,14 @@ def gen_gradient(self, bounds): dummy_df = row if dummy_df.empty else pd.concat([dummy_df, row], ignore_index=True) # Multiply by scaling factor if present - dummy_df["dummy_data"] = dummy_df["dummy_data"] * self.multiplier + multiplier_grad: float = ( + self.multiplier if hasattr(self, "multiplier") and self.multiplier is not None else 1.0 + ) + dummy_df["dummy_data"] = dummy_df["dummy_data"] * multiplier_grad return dummy_df - def gen_checkerboard(self, bounds): + def gen_checkerboard(self, bounds: Boundary) -> pd.DataFrame: """ Generates a checkerboard pattern within bounds of lat/long min/max Square size can be defined in the config, as well as resolution of @@ -214,12 +227,13 @@ def gen_checkerboard(self, bounds): logger.info("- Creating series of 0's and 1's for lat/long") # Create checkerboard pattern - if self.gridsize is None: - self.gridsize = (1, 1) - if isinstance(self.gridsize, int): - gridsize_tuple = (self.gridsize, self.gridsize) + gridsize: int | tuple[float, float] = ( + self.gridsize if hasattr(self, "gridsize") and self.gridsize is not None else (1, 1) + ) + if isinstance(gridsize, int): + gridsize_tuple: tuple[float, float] = (gridsize, gridsize) else: - gridsize_tuple = self.gridsize + gridsize_tuple = gridsize # Create horizontal stripes of 0's and 1's, stripe size defined by gridsize horizontal = np.floor((self.lat - bounds.get_lat_min()) / gridsize_tuple[1]) % 2 # Create vertical stripes of 0's and 1's, stripe size defined by gridsize @@ -245,7 +259,7 @@ def gen_checkerboard(self, bounds): return dummy_df - def gen_rectangle(self, bounds): + def gen_rectangle(self, bounds: Boundary) -> pd.DataFrame: """ Generates a rectangle within bounds of lat/long min/max. Side lengths and centroid can be defined in the config, as well as @@ -262,10 +276,11 @@ def gen_rectangle(self, bounds): # Set centre as centre of data_grid if none specified if self.width is None or self.height is None: raise ValueError("width and height parameters are required for rectangle") - if self.centre is None: - self.centre = (None, None) - c_y = self.lat[int(self.ny / 2)] if self.centre[0] is None else self.centre[0] - c_x = self.long[int(self.nx / 2)] if self.centre[1] is None else self.centre[1] + centre_rect: tuple[float | None, float | None] = ( + self.centre if hasattr(self, "centre") else (None, None) + ) + c_y = self.lat[int(self.ny / 2)] if centre_rect[0] is None else centre_rect[0] + c_x = self.long[int(self.nx / 2)] if centre_rect[1] is None else centre_rect[1] # Create vectors for row and col idx's y = np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.ny).reshape(-1, 1) @@ -298,8 +313,9 @@ def gen_rectangle(self, bounds): # Change boolean values to int dummy_df["dummy_data"] = dummy_df["dummy_data"].astype(int) # Multiply by scaling factor if present - if self.multiplier is None: - self.multiplier = 1.0 - dummy_df["dummy_data"] = dummy_df["dummy_data"] * self.multiplier + multiplier_rect: float = ( + self.multiplier if hasattr(self, "multiplier") and self.multiplier is not None else 1.0 + ) + dummy_df["dummy_data"] = dummy_df["dummy_data"] * multiplier_rect return dummy_df diff --git a/meshiphi/dataloaders/vector/baltic_current.py b/meshiphi/dataloaders/vector/baltic_current.py index f7606502..47360564 100644 --- a/meshiphi/dataloaders/vector/baltic_current.py +++ b/meshiphi/dataloaders/vector/baltic_current.py @@ -1,10 +1,17 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import xarray as xr from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class BalticCurrentDataLoader(VectorDataLoader): - def import_data(self, bounds): +class BalticCurrentDataLoader(VectorDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in current data from a copernicus baltic sea physics reanalysis NetCDF file. Renames coordinates to 'lat' and 'long', and renames variable to diff --git a/meshiphi/dataloaders/vector/duacs_current.py b/meshiphi/dataloaders/vector/duacs_current.py index b5f32bb1..08549392 100644 --- a/meshiphi/dataloaders/vector/duacs_current.py +++ b/meshiphi/dataloaders/vector/duacs_current.py @@ -1,13 +1,19 @@ +from __future__ import annotations + from datetime import datetime from os.path import basename +from typing import TYPE_CHECKING import xarray as xr from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class DuacsCurrentDataLoader(VectorDataLoader): - def import_data(self, bounds): +class DuacsCurrentDataLoader(VectorDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a DUACS altimeter derived current NetCDF file. Renames coordinates to 'lat' and 'long', and renames variable to @@ -27,7 +33,7 @@ def import_data(self, bounds): # Reduce files to those within date range if self.files is None: raise ValueError("files parameter is required for DuacsCurrentDataLoader") - self.files = [ + self.files: list[str] = [ file for file in self.files if time_range[0] diff --git a/meshiphi/dataloaders/vector/era5_wave_direction_vector.py b/meshiphi/dataloaders/vector/era5_wave_direction_vector.py index b897e2c8..97c83702 100644 --- a/meshiphi/dataloaders/vector/era5_wave_direction_vector.py +++ b/meshiphi/dataloaders/vector/era5_wave_direction_vector.py @@ -1,14 +1,20 @@ +from __future__ import annotations + from datetime import datetime from os.path import basename +from typing import TYPE_CHECKING import numpy as np import xarray as xr from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class ERA5WaveDirectionLoader(VectorDataLoader): - def import_data(self, bounds): +class ERA5WaveDirectionLoader(VectorDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in wave direction data from a ERA5 NetCDF file. Renames coordinates to 'lat' and 'long' and calculates unit vector @@ -28,7 +34,7 @@ def import_data(self, bounds): # Reduce files to those within date range if self.files is None: raise ValueError("files parameter is required for ERA5WaveDirectionLoader") - self.files = [ + self.files: list[str] = [ file for file in self.files if time_range[0] diff --git a/meshiphi/dataloaders/vector/era5_wind.py b/meshiphi/dataloaders/vector/era5_wind.py index 2000dfe1..d78479c9 100644 --- a/meshiphi/dataloaders/vector/era5_wind.py +++ b/meshiphi/dataloaders/vector/era5_wind.py @@ -1,13 +1,19 @@ +from __future__ import annotations + from datetime import datetime from os.path import basename +from typing import TYPE_CHECKING import xarray as xr from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class ERA5WindDataLoader(VectorDataLoader): - def import_data(self, bounds): +class ERA5WindDataLoader(VectorDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in wind data from a ERA5 NetCDF file. Renames coordinates to 'lat' and 'long' @@ -26,7 +32,7 @@ def import_data(self, bounds): # Reduce files to those within date range if self.files is None: raise ValueError("files parameter is required for ERA5WindDataLoader") - self.files = [ + self.files: list[str] = [ file for file in self.files if time_range[0] diff --git a/meshiphi/dataloaders/vector/north_sea_current.py b/meshiphi/dataloaders/vector/north_sea_current.py index 851f55f3..3143bbe6 100644 --- a/meshiphi/dataloaders/vector/north_sea_current.py +++ b/meshiphi/dataloaders/vector/north_sea_current.py @@ -1,10 +1,17 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import xarray as xr from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class NorthSeaCurrentDataLoader(VectorDataLoader): - def import_data(self, bounds): +class NorthSeaCurrentDataLoader(VectorDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a BSOSE Depth NetCDF file. Renames coordinates to 'lat', 'long', 'time', and renames variable to diff --git a/meshiphi/dataloaders/vector/oras5_current.py b/meshiphi/dataloaders/vector/oras5_current.py index 774a7491..04069a9d 100644 --- a/meshiphi/dataloaders/vector/oras5_current.py +++ b/meshiphi/dataloaders/vector/oras5_current.py @@ -1,11 +1,18 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import xarray as xr from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + # TODO Read in 2 files, combine to one object -class ORAS5CurrentDataLoader(VectorDataLoader): - def import_data(self, bounds): +class ORAS5CurrentDataLoader(VectorDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a ORAS5 Depth NetCDF files. Renames coordinates to 'lat' and 'long', and renames variable to diff --git a/meshiphi/dataloaders/vector/sose.py b/meshiphi/dataloaders/vector/sose.py index 30473425..04cc8a57 100644 --- a/meshiphi/dataloaders/vector/sose.py +++ b/meshiphi/dataloaders/vector/sose.py @@ -1,10 +1,18 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pandas as pd # noqa: TC002 import xarray as xr from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class SOSEDataLoader(VectorDataLoader): - def import_data(self, bounds): +class SOSEDataLoader(VectorDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Reads in data from a SOSE Currents NetCDF file. Renames coordinates to 'lat' and 'long', and renames variable to diff --git a/meshiphi/dataloaders/vector/vector_csv.py b/meshiphi/dataloaders/vector/vector_csv.py index dd849db1..13d8a2e5 100644 --- a/meshiphi/dataloaders/vector/vector_csv.py +++ b/meshiphi/dataloaders/vector/vector_csv.py @@ -1,10 +1,17 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import pandas as pd from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class VectorCSVDataLoader(VectorDataLoader): - def import_data(self, bounds): +class VectorCSVDataLoader(VectorDataLoader): # type: ignore[misc] + def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Reads in data from a CSV file. diff --git a/meshiphi/dataloaders/vector/vector_grf.py b/meshiphi/dataloaders/vector/vector_grf.py index 0f73f307..8b9d9665 100644 --- a/meshiphi/dataloaders/vector/vector_grf.py +++ b/meshiphi/dataloaders/vector/vector_grf.py @@ -1,12 +1,20 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + import numpy as np import pandas as pd +import xarray as xr # noqa: TC002 from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader from meshiphi.utils import gaussian_random_field +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + -class VectorGRFDataLoader(VectorDataLoader): - def add_default_params(self, params): +class VectorGRFDataLoader(VectorDataLoader): # type: ignore[misc] + def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for abstract GRF dataloaders, starting by including defaults for scalar dataloaders. @@ -48,7 +56,7 @@ def add_default_params(self, params): return params - def import_data(self, bounds): + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Creates data in the form of a Gaussian Random Field @@ -64,7 +72,12 @@ def import_data(self, bounds): """ - def grf_to_vector(magnitudes, directions, min_val, max_val): + def grf_to_vector( + magnitudes: np.ndarray[Any, Any], + directions: np.ndarray[Any, Any], + min_val: float, + max_val: float, + ) -> tuple[np.ndarray[Any, Any], np.ndarray[Any, Any]]: # Scale to max/min magnitudes = magnitudes * (max_val - min_val) + min_val diff --git a/meshiphi/dataloaders/vector/vector_shape.py b/meshiphi/dataloaders/vector/vector_shape.py index d83ec386..a1d1b7be 100644 --- a/meshiphi/dataloaders/vector/vector_shape.py +++ b/meshiphi/dataloaders/vector/vector_shape.py @@ -1,15 +1,22 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING, Any import numpy as np import pandas as pd +import xarray as xr # noqa: TC002 from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + logger = logging.getLogger(__name__) -class VectorShapeDataLoader(VectorDataLoader): - def add_default_params(self, params): +class VectorShapeDataLoader(VectorDataLoader): # type: ignore[misc] + def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for vector shape dataloaders @@ -59,7 +66,7 @@ def add_default_params(self, params): return params - def import_data(self, bounds): + def import_data(self, bounds: Boundary) -> xr.Dataset: """ Generates data in the form of an abstract shape, such as circle, or gradient. This method acts like a factory in that it simply @@ -88,7 +95,7 @@ def import_data(self, bounds): # No need to trim data, as was defined by bounds return data.set_index(["lat", "long"]).to_xarray() - def gen_gradient(self, bounds): + def gen_gradient(self, bounds: Boundary) -> pd.DataFrame: """ Generates a gradient within bounds of lat/long min/max. Gradient direction can be defined in the config, as well as @@ -104,9 +111,10 @@ def gen_gradient(self, bounds): logger.info("\tCreating gradient of values") # Create 1D gradient - if self.vertical is None: - self.vertical = True # Default to vertical gradient - gradient = np.linspace(0, 1, self.ny) if self.vertical else np.linspace(0, 1, self.nx) + vertical: bool = ( + self.vertical if hasattr(self, "vertical") and self.vertical is not None else True + ) + gradient = np.linspace(0, 1, self.ny) if vertical else np.linspace(0, 1, self.nx) dummy_df = pd.DataFrame(columns=["lat", "long", "dummy_data_u", "dummy_data_v"]) logger.info("- Generating vector dataset") @@ -114,9 +122,9 @@ def gen_gradient(self, bounds): for i in range(self.ny): for j in range(self.nx): # Change dummy data depending on which axis to gradient - datum = gradient[i] if self.vertical else gradient[j] + datum = gradient[i] if vertical else gradient[j] # Create a new row, adding datum value - if self.vertical: + if vertical: row = pd.DataFrame( data={ "lat": self.lat[i], @@ -140,16 +148,22 @@ def gen_gradient(self, bounds): dummy_df = row if dummy_df.empty else pd.concat([dummy_df, row], ignore_index=True) # Multiply by scaling factor if present - if self.multiplier_u is None: - self.multiplier_u = 1.0 - if self.multiplier_v is None: - self.multiplier_v = 1.0 - dummy_df["dummy_data_u"] = dummy_df["dummy_data_u"] * self.multiplier_u - dummy_df["dummy_data_v"] = dummy_df["dummy_data_v"] * self.multiplier_v + multiplier_u: float = ( + self.multiplier_u + if hasattr(self, "multiplier_u") and self.multiplier_u is not None + else 1.0 + ) + multiplier_v: float = ( + self.multiplier_v + if hasattr(self, "multiplier_v") and self.multiplier_v is not None + else 1.0 + ) + dummy_df["dummy_data_u"] = dummy_df["dummy_data_u"] * multiplier_u + dummy_df["dummy_data_v"] = dummy_df["dummy_data_v"] * multiplier_v return dummy_df - def gen_circle(self, bounds): + def gen_circle(self, bounds: Boundary) -> pd.DataFrame: """ Generates a circle within bounds of lat/long min/max. Circle centre and radius can be defined in the config, as well as @@ -166,10 +180,11 @@ def gen_circle(self, bounds): # Set centre as centre of data_grid if none specified if self.radius is None: raise ValueError("radius parameter is required for vector_circle") - if self.centre is None: - self.centre = (None, None) - c_y = self.lat[int(self.ny / 2)] if self.centre[0] is None else self.centre[0] - c_x = self.long[int(self.nx / 2)] if self.centre[1] is None else self.centre[1] + centre: tuple[float | None, float | None] = ( + self.centre if hasattr(self, "centre") else (None, None) + ) + c_y = self.lat[int(self.ny / 2)] if centre[0] is None else centre[0] + c_x = self.long[int(self.nx / 2)] if centre[1] is None else centre[1] # Create vectors for row and col indices y = np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.ny).reshape(-1, 1) @@ -203,12 +218,22 @@ def gen_circle(self, bounds): dummy_df["dummy_data_u"] = dummy_df["dummy_data_u"].astype(int) dummy_df["dummy_data_v"] = dummy_df["dummy_data_v"].astype(int) # Multiply by scaling factor if present - dummy_df["dummy_data_u"] = dummy_df["dummy_data_u"] * self.multiplier_u - dummy_df["dummy_data_v"] = dummy_df["dummy_data_v"] * self.multiplier_v + multiplier_u_circ: float = ( + self.multiplier_u + if hasattr(self, "multiplier_u") and self.multiplier_u is not None + else 1.0 + ) + multiplier_v_circ: float = ( + self.multiplier_v + if hasattr(self, "multiplier_v") and self.multiplier_v is not None + else 1.0 + ) + dummy_df["dummy_data_u"] = dummy_df["dummy_data_u"] * multiplier_u_circ + dummy_df["dummy_data_v"] = dummy_df["dummy_data_v"] * multiplier_v_circ return dummy_df - def gen_rectangle(self, bounds): + def gen_rectangle(self, bounds: Boundary) -> pd.DataFrame: """ Generates a rectangle within bounds of lat/long min/max. Side lengths and centroid can be defined in the config, as well as @@ -225,10 +250,11 @@ def gen_rectangle(self, bounds): # Set centre as centre of data_grid if none specified if self.width is None or self.height is None: raise ValueError("width and height parameters are required for vector_rectangle") - if self.centre is None: - self.centre = (None, None) - c_y = self.lat[int(self.ny / 2)] if self.centre[0] is None else self.centre[0] - c_x = self.long[int(self.nx / 2)] if self.centre[1] is None else self.centre[1] + centre_rect: tuple[float | None, float | None] = ( + self.centre if hasattr(self, "centre") else (None, None) + ) + c_y = self.lat[int(self.ny / 2)] if centre_rect[0] is None else centre_rect[0] + c_x = self.long[int(self.nx / 2)] if centre_rect[1] is None else centre_rect[1] # Create vectors for row and col indices y = np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.ny).reshape(-1, 1) @@ -263,7 +289,17 @@ def gen_rectangle(self, bounds): dummy_df["dummy_data_u"] = dummy_df["dummy_data_u"].astype(int) dummy_df["dummy_data_v"] = dummy_df["dummy_data_v"].astype(int) # Multiply by scaling factor if present - dummy_df["dummy_data_u"] = dummy_df["dummy_data_u"] * self.multiplier_u - dummy_df["dummy_data_v"] = dummy_df["dummy_data_v"] * self.multiplier_v + multiplier_u_rect: float = ( + self.multiplier_u + if hasattr(self, "multiplier_u") and self.multiplier_u is not None + else 1.0 + ) + multiplier_v_rect: float = ( + self.multiplier_v + if hasattr(self, "multiplier_v") and self.multiplier_v is not None + else 1.0 + ) + dummy_df["dummy_data_u"] = dummy_df["dummy_data_u"] * multiplier_u_rect + dummy_df["dummy_data_v"] = dummy_df["dummy_data_v"] * multiplier_v_rect return dummy_df From cbc04455d0f1db4abb2e8a84da20d3a8c53c9322 Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Wed, 14 Jan 2026 17:47:24 +0000 Subject: [PATCH 07/15] further --strict types added --- meshiphi/utils.py | 93 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 70 insertions(+), 23 deletions(-) diff --git a/meshiphi/utils.py b/meshiphi/utils.py index 9782edf7..1f8d425c 100644 --- a/meshiphi/utils.py +++ b/meshiphi/utils.py @@ -2,24 +2,47 @@ Miscellaneous utility functions that may be of use throughout MeshiPhi """ +from __future__ import annotations + import logging import time import tracemalloc from calendar import monthrange from datetime import datetime, timedelta from functools import wraps +from typing import TYPE_CHECKING, Any, Callable, Generator, TypeVar, overload import numpy as np +import numpy.typing as npt try: from scipy.fft import fftshift except ImportError: from scipy.fftpack import fftshift +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + logger = logging.getLogger(__name__) +F = TypeVar("F", bound=Callable[..., Any]) + + +@overload +def longitude_domain(long: float) -> float: ... + + +@overload +def longitude_domain(long: list[float]) -> list[float]: ... + + +@overload +def longitude_domain(long: npt.NDArray[np.floating[Any]]) -> npt.NDArray[np.floating[Any]]: ... + -def longitude_domain(long): +def longitude_domain( + long: float | list[float] | npt.NDArray[np.floating[Any]], +) -> float | list[float] | npt.NDArray[np.floating[Any]]: """ Converts any longitude degree value into one between -180:180 """ @@ -35,20 +58,22 @@ def longitude_domain(long): return (long + 180) % 360 - 180 -def longitude_distance(long_a, long_b): +def longitude_distance(long_a: float, long_b: float) -> float: """ Calculates the angular distance between two longitude values """ - long_dist = np.abs(longitude_domain(long_b) - longitude_domain(long_a)) - long_dist = np.mod(long_dist, 360) + long_dist = float(np.abs(longitude_domain(long_b) - longitude_domain(long_a))) + long_dist = float(np.mod(long_dist, 360)) if long_dist > 180: return 360 - long_dist return long_dist -def frac_of_month(year, month, start_date=None, end_date=None): +def frac_of_month( + year: int, month: int, start_date: datetime | None = None, end_date: datetime | None = None +) -> float: # Determine the number of days in the month specified days_in_month = monthrange(year, month)[1] # If not specified, default to beginning/end of month @@ -66,22 +91,22 @@ def frac_of_month(year, month, start_date=None, end_date=None): return days_overlap / days_in_month -def boundary_to_coords(bounds): +def boundary_to_coords(bounds: Boundary) -> tuple[tuple[float, float], tuple[float, float]]: min_coords = (bounds.get_lat_min(), bounds.get_long_min()) max_coords = (bounds.get_lat_max(), bounds.get_long_max()) return min_coords, max_coords -def str_to_datetime(date_str): +def str_to_datetime(date_str: str) -> datetime: return datetime.strptime(date_str, "%Y-%m-%d") -def date_range(start_date, end_date): +def date_range(start_date: datetime, end_date: datetime) -> Generator[datetime, None, None]: for n in range(int((end_date - start_date).days)): yield start_date + timedelta(n) -def convert_decimal_days(decimal_days, mins=False): +def convert_decimal_days(decimal_days: float, mins: bool = False) -> str: """ Convert decimal days to more readable Days, Hours and (optionally) Minutes Args: @@ -109,7 +134,27 @@ def convert_decimal_days(decimal_days, mins=False): return new_time -def round_to_sigfig(x, sigfig=5): +@overload +def round_to_sigfig(x: int, sigfig: int = 5) -> int: ... + + +@overload +def round_to_sigfig(x: float, sigfig: int = 5) -> float: ... + + +@overload +def round_to_sigfig(x: list[float], sigfig: int = 5) -> list[float]: ... + + +@overload +def round_to_sigfig( + x: npt.NDArray[np.floating[Any]], sigfig: int = 5 +) -> npt.NDArray[np.floating[Any]]: ... + + +def round_to_sigfig( + x: int | float | list[float] | npt.NDArray[np.floating[Any]], sigfig: int = 5 +) -> int | float | list[float] | npt.NDArray[np.floating[Any]]: """ Rounds numbers to some number of significant figures @@ -156,7 +201,7 @@ def round_to_sigfig(x, sigfig=5): return rounded -def divergence(flow): +def divergence(flow: npt.NDArray[np.floating[Any]]) -> npt.NDArray[np.floating[Any]]: flow = np.swapaxes(flow, 0, 1) Fx, Fy = flow[:, :, 0], flow[:, :, 1] dFx_dx = np.gradient(Fx, axis=0) @@ -164,7 +209,7 @@ def divergence(flow): return dFx_dx + dFy_dy -def curl(flow): +def curl(flow: npt.NDArray[np.floating[Any]]) -> npt.NDArray[np.floating[Any]]: flow = np.swapaxes(flow, 0, 1) Fx, Fy = flow[:, :, 0], flow[:, :, 1] dFx_dy = np.gradient(Fx, axis=1) @@ -173,7 +218,7 @@ def curl(flow): # GRF functions -def fftind(size): +def fftind(size: int) -> npt.NDArray[np.int_]: """ Creates a numpy array of shifted Fourier coordinates. @@ -194,7 +239,7 @@ def fftind(size): return fftshift(k_ind) -def gaussian_random_field(size, alpha): +def gaussian_random_field(size: int, alpha: float) -> npt.NDArray[np.floating[Any]]: """ Creates a gaussian random field with normal (circular) distribution Code from https://github.com/bsciolla/gaussian-random-fields/blob/master/gaussian_random_fields.py @@ -231,9 +276,9 @@ def gaussian_random_field(size, alpha): return grf / (np.max(grf) - np.min(grf)) -def memory_trace(func): +def memory_trace(func: F) -> F: @wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: tracemalloc.start(20) res = func(*args, **kwargs) snapshot = tracemalloc.take_snapshot() @@ -244,23 +289,25 @@ def wrapper(*args, **kwargs): logging.info("\n".join(stat.traceback.format())) return res - return wrapper + return wrapper # type: ignore[return-value] -def timed_call(func): +def timed_call(func: F) -> F: @wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: start = time.perf_counter() res = func(*args, **kwargs) end = time.perf_counter() logger.info(f"Timed call to {func.__name__} took {end - start:02f} seconds") return res - return wrapper + return wrapper # type: ignore[return-value] # CLI utilities -def setup_logging(func, log_format="[%(asctime)-17s :%(levelname)-8s] - %(message)s"): +def setup_logging( + func: F, log_format: str = "[%(asctime)-17s :%(levelname)-8s] - %(message)s" +) -> F: """Wraps a CLI endpoint and sets up logging for it This is probably not the smoothest implementation, but it's an educational @@ -275,7 +322,7 @@ def setup_logging(func, log_format="[%(asctime)-17s :%(levelname)-8s] - %(messag """ @wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: parsed_args = func(*args, **kwargs) level = logging.INFO @@ -296,4 +343,4 @@ def wrapper(*args, **kwargs): logging.getLogger("urllib3").setLevel(logging.WARNING) return parsed_args - return wrapper + return wrapper # type: ignore[return-value] From 011699c5f05bd44b4af6d53eeb660a55da7ef3b5 Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Wed, 14 Jan 2026 18:34:11 +0000 Subject: [PATCH 08/15] yet more --strict types --- meshiphi/cli.py | 2 +- meshiphi/dataloaders/factory.py | 11 +- meshiphi/dataloaders/lut/abstract_lut.py | 4 +- meshiphi/dataloaders/lut/density.py | 2 +- meshiphi/dataloaders/lut/lut_csv.py | 2 +- meshiphi/dataloaders/lut/lut_geojson.py | 2 +- meshiphi/dataloaders/lut/lut_shapefile.py | 2 +- meshiphi/dataloaders/lut/scotland_ncmpa.py | 2 +- meshiphi/dataloaders/lut/thickness.py | 2 +- .../dataloaders/scalar/abstract_scalar.py | 28 +-- meshiphi/dataloaders/scalar/amsr.py | 2 +- meshiphi/dataloaders/scalar/baltic_sea_ice.py | 2 +- meshiphi/dataloaders/scalar/bsose_depth.py | 2 +- meshiphi/dataloaders/scalar/bsose_sea_ice.py | 2 +- .../scalar/ecmwf_sig_wave_height.py | 2 +- .../scalar/era5_max_wave_height.py | 2 +- .../scalar/era5_mean_wave_direction.py | 2 +- .../scalar/era5_sig_wave_height.py | 2 +- .../dataloaders/scalar/era5_wave_period.py | 2 +- meshiphi/dataloaders/scalar/era5_wind_dir.py | 2 +- meshiphi/dataloaders/scalar/era5_wind_mag.py | 2 +- meshiphi/dataloaders/scalar/gebco.py | 2 +- meshiphi/dataloaders/scalar/icenet.py | 2 +- meshiphi/dataloaders/scalar/modis.py | 2 +- meshiphi/dataloaders/scalar/scalar_csv.py | 2 +- meshiphi/dataloaders/scalar/scalar_grf.py | 6 +- meshiphi/dataloaders/scalar/shape.py | 8 +- meshiphi/dataloaders/scalar/visual_iced.py | 4 +- .../dataloaders/vector/abstract_vector.py | 26 +-- meshiphi/dataloaders/vector/baltic_current.py | 2 +- meshiphi/dataloaders/vector/duacs_current.py | 2 +- .../vector/era5_wave_direction_vector.py | 2 +- meshiphi/dataloaders/vector/era5_wind.py | 2 +- .../dataloaders/vector/north_sea_current.py | 2 +- meshiphi/dataloaders/vector/oras5_current.py | 2 +- meshiphi/dataloaders/vector/sose.py | 2 +- meshiphi/dataloaders/vector/vector_csv.py | 2 +- meshiphi/dataloaders/vector/vector_grf.py | 6 +- meshiphi/dataloaders/vector/vector_shape.py | 8 +- meshiphi/mesh_generation/cellbox.py | 29 ++-- meshiphi/mesh_generation/mesh_builder.py | 161 +++++++++--------- meshiphi/mesh_generation/neighbour_graph.py | 56 +++--- meshiphi/mesh_plotting/mesh_plotter.py | 12 +- .../mesh_validation/full_mesh_validator.py | 15 +- meshiphi/mesh_validation/mesh_comparator.py | 24 ++- meshiphi/mesh_validation/mesh_validator.py | 26 +-- meshiphi/mesh_validation/sampler.py | 6 +- meshiphi/test_automation/test_automater.py | 81 ++++++--- meshiphi/utils.py | 16 +- pyproject.toml | 4 +- 50 files changed, 331 insertions(+), 260 deletions(-) diff --git a/meshiphi/cli.py b/meshiphi/cli.py index b663d71e..05d3fd05 100644 --- a/meshiphi/cli.py +++ b/meshiphi/cli.py @@ -220,7 +220,7 @@ def meshiphi_test_cli() -> None: """ @setup_logging - def get_test_automater_args(): + def get_test_automater_args() -> argparse.Namespace: ap = argparse.ArgumentParser() # Add one or two arguments as branches # If one, compare current branch to specified one diff --git a/meshiphi/dataloaders/factory.py b/meshiphi/dataloaders/factory.py index c29208e1..bc130206 100644 --- a/meshiphi/dataloaders/factory.py +++ b/meshiphi/dataloaders/factory.py @@ -2,6 +2,10 @@ import os from glob import glob +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary from meshiphi.dataloaders.lut.density import DensityDataLoader from meshiphi.dataloaders.lut.lut_csv import LutCSV @@ -52,7 +56,7 @@ class DataLoaderFactory: """ @staticmethod - def get_dataloader(name, bounds, params, min_dp=5): + def get_dataloader(name: str, bounds: Boundary, params: dict[str, Any], min_dp: int = 5) -> Any: """ Creates appropriate dataloader object based on name @@ -79,7 +83,7 @@ def get_dataloader(name, bounds, params, min_dp=5): # Cast name to lowercase to make case insensitive name = name.lower() # Translate 'file' or 'folder' into 'files' key - params = DataLoaderFactory.translate_file_input(params) + DataLoaderFactory.translate_file_input(params) # Add loader name to params params["dataloader_name"] = name @@ -152,7 +156,7 @@ def get_dataloader(name, bounds, params, min_dp=5): return data_loader(bounds, params) @staticmethod - def translate_file_input(params): + def translate_file_input(params: dict[str, Any]) -> None: """ Allows flexible file specification in params. Translates 'file' or 'folder' into 'files' @@ -168,4 +172,3 @@ def translate_file_input(params): folder = os.path.join(params["folder"], "") # Adds trailing slash if non-existent params["files"] = sorted(glob(folder + "*")) del params["folder"] - return params diff --git a/meshiphi/dataloaders/lut/abstract_lut.py b/meshiphi/dataloaders/lut/abstract_lut.py index 17c1d3b8..4a9d1883 100644 --- a/meshiphi/dataloaders/lut/abstract_lut.py +++ b/meshiphi/dataloaders/lut/abstract_lut.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) -class LutDataLoader(DataLoaderInterface): # type: ignore[misc] +class LutDataLoader(DataLoaderInterface): """ Abstract class for all LookUp Table Datasets. """ @@ -228,7 +228,7 @@ def trim_datapoints(self, bounds: Boundary, data: Any = None) -> pd.DataFrame: data = self.data # Limit time to boundary if "time" in data.index.names: - data = data.loc[bounds.get_time_min() : bounds.get_time_max()] + data = data.loc[bounds.get_time_min() : bounds.get_time_max()] # type: ignore[misc] # Find intersection of each polygon to the boundary bounds_polygon = bounds.to_polygon() diff --git a/meshiphi/dataloaders/lut/density.py b/meshiphi/dataloaders/lut/density.py index 90fdc075..5019261e 100644 --- a/meshiphi/dataloaders/lut/density.py +++ b/meshiphi/dataloaders/lut/density.py @@ -11,7 +11,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class DensityDataLoader(LutDataLoader): # type: ignore[misc] +class DensityDataLoader(LutDataLoader): def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Creates a simulated dataset of sea ice density based on diff --git a/meshiphi/dataloaders/lut/lut_csv.py b/meshiphi/dataloaders/lut/lut_csv.py index eb0664bd..e1de9871 100644 --- a/meshiphi/dataloaders/lut/lut_csv.py +++ b/meshiphi/dataloaders/lut/lut_csv.py @@ -9,7 +9,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class LutCSV(LutDataLoader): # type: ignore[misc] +class LutCSV(LutDataLoader): def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Import a list of .csv files, assign regions a value specified in diff --git a/meshiphi/dataloaders/lut/lut_geojson.py b/meshiphi/dataloaders/lut/lut_geojson.py index 9113148f..1c914d28 100644 --- a/meshiphi/dataloaders/lut/lut_geojson.py +++ b/meshiphi/dataloaders/lut/lut_geojson.py @@ -11,7 +11,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class LutGeoJSON(LutDataLoader): # type: ignore[misc] +class LutGeoJSON(LutDataLoader): def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for LUT dataloader. Only unique addition over diff --git a/meshiphi/dataloaders/lut/lut_shapefile.py b/meshiphi/dataloaders/lut/lut_shapefile.py index cd437779..7d0d9850 100644 --- a/meshiphi/dataloaders/lut/lut_shapefile.py +++ b/meshiphi/dataloaders/lut/lut_shapefile.py @@ -11,7 +11,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class LutShapefile(LutDataLoader): # type: ignore[misc] +class LutShapefile(LutDataLoader): def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for LUT dataloader. Only unique addition over diff --git a/meshiphi/dataloaders/lut/scotland_ncmpa.py b/meshiphi/dataloaders/lut/scotland_ncmpa.py index 291e1196..16c596be 100644 --- a/meshiphi/dataloaders/lut/scotland_ncmpa.py +++ b/meshiphi/dataloaders/lut/scotland_ncmpa.py @@ -8,7 +8,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class ScotlandNCMPA(LutDataLoader): # type: ignore[misc] +class ScotlandNCMPA(LutDataLoader): def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Creates a simulated dataset of sea ice thickness based on diff --git a/meshiphi/dataloaders/lut/thickness.py b/meshiphi/dataloaders/lut/thickness.py index 4a207008..5ee180d2 100644 --- a/meshiphi/dataloaders/lut/thickness.py +++ b/meshiphi/dataloaders/lut/thickness.py @@ -39,7 +39,7 @@ } -class ThicknessDataLoader(LutDataLoader): # type: ignore[misc] +class ThicknessDataLoader(LutDataLoader): class Region: """ Data storage object with region boundary as polygon, seasonal diff --git a/meshiphi/dataloaders/scalar/abstract_scalar.py b/meshiphi/dataloaders/scalar/abstract_scalar.py index 142c2fdc..2044c0bd 100644 --- a/meshiphi/dataloaders/scalar/abstract_scalar.py +++ b/meshiphi/dataloaders/scalar/abstract_scalar.py @@ -462,7 +462,7 @@ def trim_datapoints_from_xr(data: xr.Dataset, bounds: Boundary) -> xr.Dataset: return trim_datapoints_from_xr(data, bounds) raise TypeError(f"Unsupported data type: {type(data)}") - def get_value( + def get_value( # type: ignore[override] self, bounds: Boundary, data: xr.Dataset | pd.DataFrame | None = None, @@ -557,7 +557,7 @@ def get_value_from_xr( np.float64: Aggregated value """ # Extract values to be worked on by numpy functions - dps = dps.values + dps = dps.values # type: ignore[assignment] logger.debug( f"\t{len(dps)} datapoints found for attribute '{self.data_name}' within bounds '{bounds}'" ) @@ -878,7 +878,7 @@ def reproject_xr( return data.sortby("lat", ascending=True).sortby("long", ascending=True) # If want accurate results df = data.to_dataframe().reset_index().dropna() - return reproject_df(df, in_proj, out_proj, x_col, y_col) + return reproject_df(df, in_proj, out_proj, x_col, y_col) # type: ignore[no-any-return] # If no reprojection to do if in_proj == out_proj: @@ -933,24 +933,24 @@ def downsample_xr(data: xr.Dataset, ds: list[int], agg_type: str) -> xr.Dataset: """ if agg_type == "MIN": # Returns min of bin - data = data.coarsen(lat=ds[1], boundary="pad").min() - data = data.coarsen(long=ds[0], boundary="pad").min() + data = data.coarsen(lat=ds[1], boundary="pad").min() # type: ignore[attr-defined] + data = data.coarsen(long=ds[0], boundary="pad").min() # type: ignore[attr-defined] elif agg_type == "MAX": # Returns max of bin - data = data.coarsen(lat=ds[1], boundary="pad").max() - data = data.coarsen(long=ds[0], boundary="pad").max() + data = data.coarsen(lat=ds[1], boundary="pad").max() # type: ignore[attr-defined] + data = data.coarsen(long=ds[0], boundary="pad").max() # type: ignore[attr-defined] elif agg_type == "MEAN": # Returns mean of bin - data = data.coarsen(lat=ds[1], boundary="pad").mean() - data = data.coarsen(long=ds[0], boundary="pad").mean() + data = data.coarsen(lat=ds[1], boundary="pad").mean() # type: ignore[attr-defined] + data = data.coarsen(long=ds[0], boundary="pad").mean() # type: ignore[attr-defined] elif agg_type == "MEDIAN": # Returns median of bin - data = data.coarsen(lat=ds[1], boundary="pad").median() - data = data.coarsen(long=ds[0], boundary="pad").median() + data = data.coarsen(lat=ds[1], boundary="pad").median() # type: ignore[attr-defined] + data = data.coarsen(long=ds[0], boundary="pad").median() # type: ignore[attr-defined] elif agg_type == "STD": # Returns std_dev of range - data = data.coarsen(lat=ds[1], boundary="pad").std() - data = data.coarsen(long=ds[0], boundary="pad").std() + data = data.coarsen(lat=ds[1], boundary="pad").std() # type: ignore[attr-defined] + data = data.coarsen(long=ds[0], boundary="pad").std() # type: ignore[attr-defined] elif agg_type == "COUNT": # Returns every first element in bin data = data.thin(lat=ds[1]) @@ -1041,7 +1041,7 @@ def get_data_name_from_xr(data: xr.Dataset) -> str: if len(name) != 1: raise ValueError( f"More than 1 data column detected, cannot retrieve data \ - name! Found columns: {','.join(name)}" + name! Found columns: {','.join(name)}" # type: ignore[arg-type] ) return str(name[0]) diff --git a/meshiphi/dataloaders/scalar/amsr.py b/meshiphi/dataloaders/scalar/amsr.py index faee3160..be0fa267 100644 --- a/meshiphi/dataloaders/scalar/amsr.py +++ b/meshiphi/dataloaders/scalar/amsr.py @@ -12,7 +12,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class AMSRDataLoader(ScalarDataLoader): # type: ignore[misc] +class AMSRDataLoader(ScalarDataLoader): def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Translates 'hemisphere' parameter into values of in_proj and out_proj diff --git a/meshiphi/dataloaders/scalar/baltic_sea_ice.py b/meshiphi/dataloaders/scalar/baltic_sea_ice.py index 992549d0..fe3f887b 100644 --- a/meshiphi/dataloaders/scalar/baltic_sea_ice.py +++ b/meshiphi/dataloaders/scalar/baltic_sea_ice.py @@ -10,7 +10,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class BalticSeaIceDataLoader(ScalarDataLoader): # type: ignore[misc] +class BalticSeaIceDataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a Baltic Sea Ice NetCDF file. diff --git a/meshiphi/dataloaders/scalar/bsose_depth.py b/meshiphi/dataloaders/scalar/bsose_depth.py index 8c5c650c..c43ffdd5 100644 --- a/meshiphi/dataloaders/scalar/bsose_depth.py +++ b/meshiphi/dataloaders/scalar/bsose_depth.py @@ -10,7 +10,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class BSOSEDepthDataLoader(ScalarDataLoader): # type: ignore[misc] +class BSOSEDepthDataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a BSOSE Depth NetCDF file. diff --git a/meshiphi/dataloaders/scalar/bsose_sea_ice.py b/meshiphi/dataloaders/scalar/bsose_sea_ice.py index 88216be1..eec9c581 100644 --- a/meshiphi/dataloaders/scalar/bsose_sea_ice.py +++ b/meshiphi/dataloaders/scalar/bsose_sea_ice.py @@ -11,7 +11,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class BSOSESeaIceDataLoader(ScalarDataLoader): # type: ignore[misc] +class BSOSESeaIceDataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a BSOSE Sea Ice NetCDF file. diff --git a/meshiphi/dataloaders/scalar/ecmwf_sig_wave_height.py b/meshiphi/dataloaders/scalar/ecmwf_sig_wave_height.py index c1298d4d..71be9051 100644 --- a/meshiphi/dataloaders/scalar/ecmwf_sig_wave_height.py +++ b/meshiphi/dataloaders/scalar/ecmwf_sig_wave_height.py @@ -13,7 +13,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class ECMWFSigWaveHeightDataLoader(ScalarDataLoader): # type: ignore[misc] +class ECMWFSigWaveHeightDataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a ECMWF GRIB2 file, or folder of files. diff --git a/meshiphi/dataloaders/scalar/era5_max_wave_height.py b/meshiphi/dataloaders/scalar/era5_max_wave_height.py index 1b7e0d76..bbf4e4b8 100644 --- a/meshiphi/dataloaders/scalar/era5_max_wave_height.py +++ b/meshiphi/dataloaders/scalar/era5_max_wave_height.py @@ -12,7 +12,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class ERA5MaxWaveHeightDataLoader(ScalarDataLoader): # type: ignore[misc] +class ERA5MaxWaveHeightDataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from an ERA5 NetCDF file. diff --git a/meshiphi/dataloaders/scalar/era5_mean_wave_direction.py b/meshiphi/dataloaders/scalar/era5_mean_wave_direction.py index 99fdb8f6..c02d5b8c 100644 --- a/meshiphi/dataloaders/scalar/era5_mean_wave_direction.py +++ b/meshiphi/dataloaders/scalar/era5_mean_wave_direction.py @@ -12,7 +12,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class ERA5MeanWaveDirDataLoader(ScalarDataLoader): # type: ignore[misc] +class ERA5MeanWaveDirDataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from an ERA5 NetCDF file. diff --git a/meshiphi/dataloaders/scalar/era5_sig_wave_height.py b/meshiphi/dataloaders/scalar/era5_sig_wave_height.py index b4338d7e..360b3a5d 100644 --- a/meshiphi/dataloaders/scalar/era5_sig_wave_height.py +++ b/meshiphi/dataloaders/scalar/era5_sig_wave_height.py @@ -12,7 +12,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class ERA5SigWaveHeightDataLoader(ScalarDataLoader): # type: ignore[misc] +class ERA5SigWaveHeightDataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from an ERA5 NetCDF file. diff --git a/meshiphi/dataloaders/scalar/era5_wave_period.py b/meshiphi/dataloaders/scalar/era5_wave_period.py index 05502892..67f2205c 100644 --- a/meshiphi/dataloaders/scalar/era5_wave_period.py +++ b/meshiphi/dataloaders/scalar/era5_wave_period.py @@ -12,7 +12,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class ERA5WavePeriodDataLoader(ScalarDataLoader): # type: ignore[misc] +class ERA5WavePeriodDataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from an ERA5 NetCDF file. diff --git a/meshiphi/dataloaders/scalar/era5_wind_dir.py b/meshiphi/dataloaders/scalar/era5_wind_dir.py index 1c1adb7e..e9db8793 100644 --- a/meshiphi/dataloaders/scalar/era5_wind_dir.py +++ b/meshiphi/dataloaders/scalar/era5_wind_dir.py @@ -13,7 +13,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class ERA5WindDirDataLoader(ScalarDataLoader): # type: ignore[misc] +class ERA5WindDirDataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from an ERA5 NetCDF file. diff --git a/meshiphi/dataloaders/scalar/era5_wind_mag.py b/meshiphi/dataloaders/scalar/era5_wind_mag.py index 1514aee4..a0bb4407 100644 --- a/meshiphi/dataloaders/scalar/era5_wind_mag.py +++ b/meshiphi/dataloaders/scalar/era5_wind_mag.py @@ -13,7 +13,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class ERA5WindMagDataLoader(ScalarDataLoader): # type: ignore[misc] +class ERA5WindMagDataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from an ERA5 NetCDF file. diff --git a/meshiphi/dataloaders/scalar/gebco.py b/meshiphi/dataloaders/scalar/gebco.py index 92f23682..4f63555d 100644 --- a/meshiphi/dataloaders/scalar/gebco.py +++ b/meshiphi/dataloaders/scalar/gebco.py @@ -10,7 +10,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class GEBCODataLoader(ScalarDataLoader): # type: ignore[misc] +class GEBCODataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from GEBCO NetCDF files. Renames coordinates to diff --git a/meshiphi/dataloaders/scalar/icenet.py b/meshiphi/dataloaders/scalar/icenet.py index aaa3c8e9..42d7c72d 100644 --- a/meshiphi/dataloaders/scalar/icenet.py +++ b/meshiphi/dataloaders/scalar/icenet.py @@ -15,7 +15,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class IceNetDataLoader(ScalarDataLoader): # type: ignore[misc] +class IceNetDataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Reads in data from a IceNet 2 NetCDF file. diff --git a/meshiphi/dataloaders/scalar/modis.py b/meshiphi/dataloaders/scalar/modis.py index 2d5e5ddd..899d394e 100644 --- a/meshiphi/dataloaders/scalar/modis.py +++ b/meshiphi/dataloaders/scalar/modis.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -class MODISDataLoader(ScalarDataLoader): # type: ignore[misc] +class MODISDataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a MODIS NetCDF file. diff --git a/meshiphi/dataloaders/scalar/scalar_csv.py b/meshiphi/dataloaders/scalar/scalar_csv.py index dfa61b09..78a5254f 100644 --- a/meshiphi/dataloaders/scalar/scalar_csv.py +++ b/meshiphi/dataloaders/scalar/scalar_csv.py @@ -10,7 +10,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class ScalarCSVDataLoader(ScalarDataLoader): # type: ignore[misc] +class ScalarCSVDataLoader(ScalarDataLoader): def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Reads in data from a CSV file. diff --git a/meshiphi/dataloaders/scalar/scalar_grf.py b/meshiphi/dataloaders/scalar/scalar_grf.py index 8a052250..38bc2b55 100644 --- a/meshiphi/dataloaders/scalar/scalar_grf.py +++ b/meshiphi/dataloaders/scalar/scalar_grf.py @@ -13,7 +13,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class ScalarGRFDataLoader(ScalarDataLoader): # type: ignore[misc] +class ScalarGRFDataLoader(ScalarDataLoader): def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for abstract GRF dataloaders, starting by @@ -127,7 +127,7 @@ def grf_to_scalar( else: if self.threshold is None or self.min is None or self.max is None: raise ValueError("threshold, min, and max parameters are required for scalar GRF") - grf = grf_to_scalar(grf, self.threshold, self.min, self.max) + grf = grf_to_scalar(grf, self.threshold, self.min, self.max) # type: ignore[arg-type] # Scale with multiplier and offset multiplier: float = self.multiplier if self.multiplier is not None else 1.0 offset: float = self.offset if self.offset is not None else 0.0 @@ -147,4 +147,4 @@ def grf_to_scalar( # Cast to dataframe data = pd.DataFrame(rows).set_index(["lat", "long"]) # Set to xarray dataset - return data.to_xarray() + return data.to_xarray() # type: ignore[no-any-return] diff --git a/meshiphi/dataloaders/scalar/shape.py b/meshiphi/dataloaders/scalar/shape.py index b584f663..06347bf8 100644 --- a/meshiphi/dataloaders/scalar/shape.py +++ b/meshiphi/dataloaders/scalar/shape.py @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) -class ShapeDataLoader(ScalarDataLoader): # type: ignore[misc] +class ShapeDataLoader(ScalarDataLoader): def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for abstract shape dataloaders, starting by @@ -99,7 +99,7 @@ def import_data(self, bounds: Boundary) -> xr.Dataset: raise ValueError(f"Unknown abstract shape type: {self.dataloader_name}") # No need to trim data, as was defined by bounds - return data.set_index(["lat", "long"]).to_xarray() + return data.set_index(["lat", "long"]).to_xarray() # type: ignore[no-any-return] def gen_circle(self, bounds: Boundary) -> pd.DataFrame: """ @@ -119,7 +119,7 @@ def gen_circle(self, bounds: Boundary) -> pd.DataFrame: if self.radius is None: raise ValueError("radius parameter is required for circle") centre: tuple[float | None, float | None] = ( - self.centre if hasattr(self, "centre") else (None, None) + self.centre if hasattr(self, "centre") else (None, None) # type: ignore[assignment] ) c_y = self.lat[int(self.ny / 2)] if centre[0] is None else centre[0] c_x = self.long[int(self.nx / 2)] if centre[1] is None else centre[1] @@ -277,7 +277,7 @@ def gen_rectangle(self, bounds: Boundary) -> pd.DataFrame: if self.width is None or self.height is None: raise ValueError("width and height parameters are required for rectangle") centre_rect: tuple[float | None, float | None] = ( - self.centre if hasattr(self, "centre") else (None, None) + self.centre if hasattr(self, "centre") else (None, None) # type: ignore[assignment] ) c_y = self.lat[int(self.ny / 2)] if centre_rect[0] is None else centre_rect[0] c_x = self.long[int(self.nx / 2)] if centre_rect[1] is None else centre_rect[1] diff --git a/meshiphi/dataloaders/scalar/visual_iced.py b/meshiphi/dataloaders/scalar/visual_iced.py index 02ba184b..044364eb 100644 --- a/meshiphi/dataloaders/scalar/visual_iced.py +++ b/meshiphi/dataloaders/scalar/visual_iced.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -class VisualIcedDataLoader(ScalarDataLoader): # type: ignore[misc] +class VisualIcedDataLoader(ScalarDataLoader): def import_data(self, _bounds: Boundary) -> xr.Dataset: """ Reads in data from Visual_Ice NetCDF files. Renames coordinates to @@ -87,4 +87,4 @@ def import_from_tiff(self, xarray_dataset: xr.Dataset) -> xr.Dataset: # convert back to xarray vi_dataframe = vi_dataframe.set_index(["x", "y"]) - return vi_dataframe.to_xarray() + return vi_dataframe.to_xarray() # type: ignore[no-any-return] diff --git a/meshiphi/dataloaders/vector/abstract_vector.py b/meshiphi/dataloaders/vector/abstract_vector.py index 4e17afb3..8c833a2f 100644 --- a/meshiphi/dataloaders/vector/abstract_vector.py +++ b/meshiphi/dataloaders/vector/abstract_vector.py @@ -645,7 +645,7 @@ def get_hom_condition( if isinstance(self.data, pd.core.frame.DataFrame): num_dp = len(self.trim_datapoints(bounds)) elif isinstance(self.data, xr.core.dataset.Dataset): - num_dp = min(self.trim_datapoints(bounds).count().values()) + num_dp = min(self.trim_datapoints(bounds).count().values()) # type: ignore[assignment] # Set default homogeneity hom_type = "CLR" @@ -813,7 +813,7 @@ def reproject_xr( return data.sortby("lat", ascending=True).sortby("long", ascending=True) # If want accurate results df = data.to_dataframe().reset_index().dropna() - return reproject_df(df, in_proj, out_proj, x_col, y_col) + return reproject_df(df, in_proj, out_proj, x_col, y_col) # type: ignore[no-any-return] # If no reprojection to do if in_proj == out_proj: @@ -868,24 +868,24 @@ def downsample_xr(data: xr.Dataset, ds: list[int], agg_type: str) -> xr.Dataset: """ if agg_type == "MIN": # Returns min of bin - data = data.coarsen(lat=ds[1], boundary="pad").min() - data = data.coarsen(long=ds[0], boundary="pad").min() + data = data.coarsen(lat=ds[1], boundary="pad").min() # type: ignore[attr-defined] + data = data.coarsen(long=ds[0], boundary="pad").min() # type: ignore[attr-defined] elif agg_type == "MAX": # Returns max of bin - data = data.coarsen(lat=ds[1], boundary="pad").max() - data = data.coarsen(long=ds[0], boundary="pad").max() + data = data.coarsen(lat=ds[1], boundary="pad").max() # type: ignore[attr-defined] + data = data.coarsen(long=ds[0], boundary="pad").max() # type: ignore[attr-defined] elif agg_type == "MEAN": # Returns mean of bin - data = data.coarsen(lat=ds[1], boundary="pad").mean() - data = data.coarsen(long=ds[0], boundary="pad").mean() + data = data.coarsen(lat=ds[1], boundary="pad").mean() # type: ignore[attr-defined] + data = data.coarsen(long=ds[0], boundary="pad").mean() # type: ignore[attr-defined] elif agg_type == "MEDIAN": # Returns median of bin - data = data.coarsen(lat=ds[1], boundary="pad").median() - data = data.coarsen(long=ds[0], boundary="pad").median() + data = data.coarsen(lat=ds[1], boundary="pad").median() # type: ignore[attr-defined] + data = data.coarsen(long=ds[0], boundary="pad").median() # type: ignore[attr-defined] elif agg_type == "STD": # Returns std_dev of range - data = data.coarsen(lat=ds[1], boundary="pad").std() - data = data.coarsen(long=ds[0], boundary="pad").std() + data = data.coarsen(lat=ds[1], boundary="pad").std() # type: ignore[attr-defined] + data = data.coarsen(long=ds[0], boundary="pad").std() # type: ignore[attr-defined] elif agg_type == "COUNT": # Returns every first element in bin data = data.thin(lat=ds[1]) @@ -950,7 +950,7 @@ def get_data_names_from_xr(data: xr.Dataset) -> str: # Extract data variables from xr.Dataset data_names = list(data.keys()) # Turn into comma seperated string and return - return ",".join(data_names) + return ",".join(data_names) # type: ignore[arg-type] logger.debug(f"\tRetrieving data name from {type(self.data)}") # Choose method of extraction based on data type diff --git a/meshiphi/dataloaders/vector/baltic_current.py b/meshiphi/dataloaders/vector/baltic_current.py index 47360564..6dec5fd7 100644 --- a/meshiphi/dataloaders/vector/baltic_current.py +++ b/meshiphi/dataloaders/vector/baltic_current.py @@ -10,7 +10,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class BalticCurrentDataLoader(VectorDataLoader): # type: ignore[misc] +class BalticCurrentDataLoader(VectorDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in current data from a copernicus baltic sea physics reanalysis NetCDF file. diff --git a/meshiphi/dataloaders/vector/duacs_current.py b/meshiphi/dataloaders/vector/duacs_current.py index 08549392..cbc70edb 100644 --- a/meshiphi/dataloaders/vector/duacs_current.py +++ b/meshiphi/dataloaders/vector/duacs_current.py @@ -12,7 +12,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class DuacsCurrentDataLoader(VectorDataLoader): # type: ignore[misc] +class DuacsCurrentDataLoader(VectorDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a DUACS altimeter derived current NetCDF file. diff --git a/meshiphi/dataloaders/vector/era5_wave_direction_vector.py b/meshiphi/dataloaders/vector/era5_wave_direction_vector.py index 97c83702..9174a4e2 100644 --- a/meshiphi/dataloaders/vector/era5_wave_direction_vector.py +++ b/meshiphi/dataloaders/vector/era5_wave_direction_vector.py @@ -13,7 +13,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class ERA5WaveDirectionLoader(VectorDataLoader): # type: ignore[misc] +class ERA5WaveDirectionLoader(VectorDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in wave direction data from a ERA5 NetCDF file. diff --git a/meshiphi/dataloaders/vector/era5_wind.py b/meshiphi/dataloaders/vector/era5_wind.py index d78479c9..cafb703e 100644 --- a/meshiphi/dataloaders/vector/era5_wind.py +++ b/meshiphi/dataloaders/vector/era5_wind.py @@ -12,7 +12,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class ERA5WindDataLoader(VectorDataLoader): # type: ignore[misc] +class ERA5WindDataLoader(VectorDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in wind data from a ERA5 NetCDF file. diff --git a/meshiphi/dataloaders/vector/north_sea_current.py b/meshiphi/dataloaders/vector/north_sea_current.py index 3143bbe6..f6dcb510 100644 --- a/meshiphi/dataloaders/vector/north_sea_current.py +++ b/meshiphi/dataloaders/vector/north_sea_current.py @@ -10,7 +10,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class NorthSeaCurrentDataLoader(VectorDataLoader): # type: ignore[misc] +class NorthSeaCurrentDataLoader(VectorDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a BSOSE Depth NetCDF file. diff --git a/meshiphi/dataloaders/vector/oras5_current.py b/meshiphi/dataloaders/vector/oras5_current.py index 04069a9d..bc121f06 100644 --- a/meshiphi/dataloaders/vector/oras5_current.py +++ b/meshiphi/dataloaders/vector/oras5_current.py @@ -11,7 +11,7 @@ # TODO Read in 2 files, combine to one object -class ORAS5CurrentDataLoader(VectorDataLoader): # type: ignore[misc] +class ORAS5CurrentDataLoader(VectorDataLoader): def import_data(self, bounds: Boundary) -> xr.Dataset: """ Reads in data from a ORAS5 Depth NetCDF files. diff --git a/meshiphi/dataloaders/vector/sose.py b/meshiphi/dataloaders/vector/sose.py index 04cc8a57..a965cb74 100644 --- a/meshiphi/dataloaders/vector/sose.py +++ b/meshiphi/dataloaders/vector/sose.py @@ -11,7 +11,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class SOSEDataLoader(VectorDataLoader): # type: ignore[misc] +class SOSEDataLoader(VectorDataLoader): def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Reads in data from a SOSE Currents NetCDF file. diff --git a/meshiphi/dataloaders/vector/vector_csv.py b/meshiphi/dataloaders/vector/vector_csv.py index 13d8a2e5..823eca51 100644 --- a/meshiphi/dataloaders/vector/vector_csv.py +++ b/meshiphi/dataloaders/vector/vector_csv.py @@ -10,7 +10,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class VectorCSVDataLoader(VectorDataLoader): # type: ignore[misc] +class VectorCSVDataLoader(VectorDataLoader): def import_data(self, bounds: Boundary) -> pd.DataFrame: """ Reads in data from a CSV file. diff --git a/meshiphi/dataloaders/vector/vector_grf.py b/meshiphi/dataloaders/vector/vector_grf.py index 8b9d9665..86d8518e 100644 --- a/meshiphi/dataloaders/vector/vector_grf.py +++ b/meshiphi/dataloaders/vector/vector_grf.py @@ -13,7 +13,7 @@ from meshiphi.mesh_generation.boundary import Boundary -class VectorGRFDataLoader(VectorDataLoader): # type: ignore[misc] +class VectorGRFDataLoader(VectorDataLoader): def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for abstract GRF dataloaders, starting by @@ -102,7 +102,7 @@ def grf_to_vector( directions = gaussian_random_field(self.size, self.alpha) directions = np.radians(360 * directions) - vec_x, vec_y = grf_to_vector(magnitudes, directions, self.min, self.max) + vec_x, vec_y = grf_to_vector(magnitudes, directions, self.min, self.max) # type: ignore[arg-type] # Set up domain of field lat_array = np.linspace(bounds.get_lat_min(), bounds.get_lat_max(), self.size) @@ -123,4 +123,4 @@ def grf_to_vector( # Cast to dataframe data = pd.DataFrame(rows).set_index(["lat", "long"]) # Set to xarray dataset - return data.to_xarray() + return data.to_xarray() # type: ignore[no-any-return] diff --git a/meshiphi/dataloaders/vector/vector_shape.py b/meshiphi/dataloaders/vector/vector_shape.py index a1d1b7be..b5dbcea9 100644 --- a/meshiphi/dataloaders/vector/vector_shape.py +++ b/meshiphi/dataloaders/vector/vector_shape.py @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) -class VectorShapeDataLoader(VectorDataLoader): # type: ignore[misc] +class VectorShapeDataLoader(VectorDataLoader): def add_default_params(self, params: dict[str, Any]) -> dict[str, Any]: """ Set default values for vector shape dataloaders @@ -93,7 +93,7 @@ def import_data(self, bounds: Boundary) -> xr.Dataset: raise ValueError(f"Unknown vector shape type: {self.dataloader_name}") # No need to trim data, as was defined by bounds - return data.set_index(["lat", "long"]).to_xarray() + return data.set_index(["lat", "long"]).to_xarray() # type: ignore[no-any-return] def gen_gradient(self, bounds: Boundary) -> pd.DataFrame: """ @@ -181,7 +181,7 @@ def gen_circle(self, bounds: Boundary) -> pd.DataFrame: if self.radius is None: raise ValueError("radius parameter is required for vector_circle") centre: tuple[float | None, float | None] = ( - self.centre if hasattr(self, "centre") else (None, None) + self.centre if hasattr(self, "centre") else (None, None) # type: ignore[assignment] ) c_y = self.lat[int(self.ny / 2)] if centre[0] is None else centre[0] c_x = self.long[int(self.nx / 2)] if centre[1] is None else centre[1] @@ -251,7 +251,7 @@ def gen_rectangle(self, bounds: Boundary) -> pd.DataFrame: if self.width is None or self.height is None: raise ValueError("width and height parameters are required for vector_rectangle") centre_rect: tuple[float | None, float | None] = ( - self.centre if hasattr(self, "centre") else (None, None) + self.centre if hasattr(self, "centre") else (None, None) # type: ignore[assignment] ) c_y = self.lat[int(self.ny / 2)] if centre_rect[0] is None else centre_rect[0] c_x = self.long[int(self.nx / 2)] if centre_rect[1] is None else centre_rect[1] diff --git a/meshiphi/mesh_generation/cellbox.py b/meshiphi/mesh_generation/cellbox.py index 1683390f..acd40fa4 100644 --- a/meshiphi/mesh_generation/cellbox.py +++ b/meshiphi/mesh_generation/cellbox.py @@ -210,11 +210,13 @@ def should_split(self, stop_index: int) -> bool: current_data_source = data_source[index] data_loader = current_data_source.get_data_loader() data_subset = current_data_source.get_data_subset() - for splitting_cond in current_data_source.get_splitting_conditions(): - hom_cond = data_loader.get_hom_condition( - self.bounds, splitting_cond, data=data_subset - ) - hom_conditions.append(hom_cond) + splitting_conditions = current_data_source.get_splitting_conditions() + if splitting_conditions is not None: + for splitting_cond in splitting_conditions: + hom_cond = data_loader.get_hom_condition( + self.bounds, splitting_cond, data=data_subset + ) + hom_conditions.append(hom_cond) if "HOM" in hom_conditions: return False if "MIN" in hom_conditions: @@ -242,16 +244,19 @@ def should_split_breadth_first(self) -> bool: will result in the CellBox being split. """ hom_conditions: list[str] = [] - if self.data_source is None: + data_source = self.data_source + if data_source is None: return False - for current_data_source in self.data_source: + for current_data_source in data_source: data_loader = current_data_source.get_data_loader() data_subset = current_data_source.get_data_subset() - for splitting_cond in current_data_source.get_splitting_conditions(): - hom_cond = data_loader.get_hom_condition( - self.bounds, splitting_cond, data=data_subset - ) - hom_conditions.append(hom_cond) + splitting_conditions = current_data_source.get_splitting_conditions() + if splitting_conditions is not None: + for splitting_cond in splitting_conditions: + hom_cond = data_loader.get_hom_condition( + self.bounds, splitting_cond, data=data_subset + ) + hom_conditions.append(hom_cond) if "HOM" in hom_conditions: return False diff --git a/meshiphi/mesh_generation/mesh_builder.py b/meshiphi/mesh_generation/mesh_builder.py index 4430ec5a..66192bb5 100644 --- a/meshiphi/mesh_generation/mesh_builder.py +++ b/meshiphi/mesh_generation/mesh_builder.py @@ -19,7 +19,7 @@ from __future__ import annotations import logging -from typing import Any, cast +from typing import Any import numpy as np from tqdm import tqdm @@ -181,7 +181,7 @@ def initialize_meta_data(self, bounds: Boundary, min_datapoints: int) -> list[Me data_source["params"]["files"] = loader.files meta_data_obj = Metadata( - loader, updated_splitting_cond, value_fill_type, loader.data + loader, updated_splitting_cond, str(value_fill_type), loader.data ) meta_data_list.append(meta_data_obj) @@ -258,21 +258,22 @@ def initialize_cellboxes( # Account for going over the antimeridian with longitude_distance (longitude_distance(bounds.get_long_min(), bounds.get_long_max()) / cell_width) - if bounds.get_long_min() < bounds.get_long_max(): - long_range = np.arange(bounds.get_long_min(), bounds.get_long_max(), cell_width) - else: - long_range = np.arange(bounds.get_long_min(), bounds.get_long_max() + 360, cell_width) + long_range_temp = ( + np.arange(bounds.get_long_min(), bounds.get_long_max(), cell_width) + if bounds.get_long_min() < bounds.get_long_max() + else np.arange(bounds.get_long_min(), bounds.get_long_max() + 360, cell_width) + ) # Cast to within -180:180 - long_range = longitude_domain(long_range) + long_range = longitude_domain(long_range_temp) for lat in lat_range: for long in long_range: - cell_lat_range = [lat, lat + cell_height] + cell_lat_range = [float(lat), float(lat + cell_height)] # If doesn't go over anti-meridian if long + cell_width <= 180: - cell_long_range = [long, long + cell_width] + cell_long_range = [float(long), float(long + cell_width)] else: - cell_long_range = [long, long + cell_width - 360] + cell_long_range = [float(long), float(long + cell_width - 360)] cell_bounds = Boundary(cell_lat_range, cell_long_range, bounds.get_time_range()) cell_id = str(len(cellboxes)) cellbox = CellBox(cell_bounds, cell_id) @@ -319,13 +320,13 @@ def add_dataloader( data_source = {"loader": name, "params": params} value_fill_type = self.check_value_fill_type(data_source) - meta_data_obj = Metadata(dataloader, updated_splitting_cond, value_fill_type) + meta_data_obj = Metadata(dataloader, updated_splitting_cond, str(value_fill_type)) for cellbox in self.mesh.cellboxes: if isinstance(cellbox, CellBox): cellbox.set_minimum_datapoints(min_dp) # Add new meta data to list of data sources per cellbox - cellbox.set_data_source([*cellbox.get_data_source(), meta_data_obj]) + cellbox.set_data_source([*cellbox.get_data_source(), meta_data_obj]) # type: ignore[misc] return self @@ -367,35 +368,35 @@ def check_global_mesh( for i in range(len(min_long_cellboxes)): self.neighbour_graph.add_neighbour( int(min_long_cellboxes[i].get_id()), - Direction.west, + str(Direction.west), int(max_long_cellboxes[i].get_id()), ) self.neighbour_graph.add_neighbour( int(max_long_cellboxes[i].get_id()), - Direction.east, + str(Direction.east), int(min_long_cellboxes[i].get_id()), ) # checks to avoid the very upper and lower cellboxes as they do not have north/south neighbours if 0 <= i < len(min_long_cellboxes) - 1: self.neighbour_graph.add_neighbour( int(min_long_cellboxes[i].get_id()), - Direction.north_west, + str(Direction.north_west), int(max_long_cellboxes[i + 1].get_id()), ) self.neighbour_graph.add_neighbour( int(max_long_cellboxes[i].get_id()), - Direction.north_east, + str(Direction.north_east), int(min_long_cellboxes[i + 1].get_id()), ) if 0 < i <= len(min_long_cellboxes) - 1: self.neighbour_graph.add_neighbour( int(min_long_cellboxes[i].get_id()), - Direction.south_west, + str(Direction.south_west), int(max_long_cellboxes[i - 1].get_id()), ) self.neighbour_graph.add_neighbour( int(max_long_cellboxes[i].get_id()), - Direction.south_east, + str(Direction.south_east), int(min_long_cellboxes[i - 1].get_id()), ) @@ -442,10 +443,10 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: south_west_indx = cellboxes.index(split_cellboxes[2]) south_east_indx = cellboxes.index(split_cellboxes[3]) - south_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, 4) - north_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, -4) - east_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, 2) - west_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, -2) + south_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, "4") + north_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, "-4") + east_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, "2") + west_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, "-2") # Create neighbour map for SW split cell. sw_neighbour_map = { @@ -455,7 +456,7 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: Direction.south: [], # update to use the NG class Direction.south_west: self.neighbour_graph.get_neighbours( - cellbox_indx, Direction.south_west + cellbox_indx, str(Direction.south_west) ), Direction.west: [], Direction.north_west: [], @@ -476,7 +477,7 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: Direction.south_west: [], Direction.west: [], Direction.north_west: self.neighbour_graph.get_neighbours( - cellbox_indx, Direction.north_west + cellbox_indx, str(Direction.north_west) ), Direction.north: [], } @@ -489,7 +490,7 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: # Create neighbour map for NE split cell ne_neighbour_map = { Direction.north_east: self.neighbour_graph.get_neighbours( - cellbox_indx, Direction.north_east + cellbox_indx, str(Direction.north_east) ), Direction.east: [], Direction.south_east: [], @@ -509,7 +510,7 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: Direction.north_east: [], Direction.east: [], Direction.south_east: self.neighbour_graph.get_neighbours( - cellbox_indx, Direction.south_east + cellbox_indx, str(Direction.south_east) ), Direction.south: [], Direction.south_west: [], @@ -543,7 +544,7 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: ) # Update corner neighbour maps self.neighbour_graph.update_corner_neighbours( - cellbox_indx, + str(cellbox_indx), north_west_indx, north_east_indx, south_west_indx, @@ -551,17 +552,29 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: ) # remove the original splitted cellbox from the neighbour_graph - self.neighbour_graph.remove_node(cellbox_indx) + self.neighbour_graph.remove_node(str(cellbox_indx)) # set the original splitted cellbox to None - cellboxes[cellbox_indx] = None - - return cast("list[CellBox]", split_cellboxes) + cellboxes[cellbox_indx] = None # type: ignore[call-overload] + + return split_cellboxes + + def _get_cellbox_by_id( + self, cellboxes: list[CellBox | None], cellbox_id: Any + ) -> CellBox | None: + """Get cellbox from list by ID, handling both int indices and Any IDs""" + if isinstance(cellbox_id, int) and 0 <= cellbox_id < len(cellboxes): + return cellboxes[cellbox_id] + # Fallback: search by ID + for cb in cellboxes: + if cb is not None and cb.id == cellbox_id: + return cb + return None ############################## methods to fill the neighbour maps of the splitted cells ######################## def fill_se_map( self, - south_east_indx: str, + south_east_indx: str | int, south_neighbour_indx: list[Any], east_neighbour_indx: list[Any], se_neighbour_map: dict[int, list[Any]], @@ -569,33 +582,30 @@ def fill_se_map( """ method that fills the South east neighbours """ - cellboxes = self.mesh.cellboxes + cellboxes: list[CellBox | None] = self.mesh.cellboxes # type: ignore[assignment] + se_cellbox = self._get_cellbox_by_id(cellboxes, south_east_indx) for indx in south_neighbour_indx: - if ( - self.neighbour_graph.get_neighbour_case(cellboxes[south_east_indx], cellboxes[indx]) - == Direction.south - ): + indx_cellbox = self._get_cellbox_by_id(cellboxes, indx) + if self.neighbour_graph.get_neighbour_case(se_cellbox, indx_cellbox) == Direction.south: se_neighbour_map[4].append(indx) if ( - self.neighbour_graph.get_neighbour_case(cellboxes[south_east_indx], cellboxes[indx]) + self.neighbour_graph.get_neighbour_case(se_cellbox, indx_cellbox) == Direction.south_west ): se_neighbour_map[-1].append(indx) for indx in east_neighbour_indx: - if ( - self.neighbour_graph.get_neighbour_case(cellboxes[south_east_indx], cellboxes[indx]) - == Direction.east - ): + indx_cellbox = self._get_cellbox_by_id(cellboxes, indx) + if self.neighbour_graph.get_neighbour_case(se_cellbox, indx_cellbox) == Direction.east: se_neighbour_map[2].append(indx) if ( - self.neighbour_graph.get_neighbour_case(cellboxes[south_east_indx], cellboxes[indx]) + self.neighbour_graph.get_neighbour_case(se_cellbox, indx_cellbox) == Direction.north_east ): se_neighbour_map[1].append(indx) def fill_ne_map( self, - north_east_indx: str, + north_east_indx: str | int, north_neighbour_indx: list[Any], east_neighbour_indx: list[Any], ne_neighbour_map: dict[int, list[Any]], @@ -603,33 +613,30 @@ def fill_ne_map( """ method that fills the North east neighbours """ - cellboxes = self.mesh.cellboxes + cellboxes: list[CellBox | None] = self.mesh.cellboxes # type: ignore[assignment] + ne_cellbox = self._get_cellbox_by_id(cellboxes, north_east_indx) for indx in north_neighbour_indx: - if ( - self.neighbour_graph.get_neighbour_case(cellboxes[north_east_indx], cellboxes[indx]) - == Direction.north - ): + indx_cellbox = self._get_cellbox_by_id(cellboxes, indx) + if self.neighbour_graph.get_neighbour_case(ne_cellbox, indx_cellbox) == Direction.north: ne_neighbour_map[-4].append(indx) if ( - self.neighbour_graph.get_neighbour_case(cellboxes[north_east_indx], cellboxes[indx]) + self.neighbour_graph.get_neighbour_case(ne_cellbox, indx_cellbox) == Direction.north_west ): ne_neighbour_map[-3].append(indx) for indx in east_neighbour_indx: - if ( - self.neighbour_graph.get_neighbour_case(cellboxes[north_east_indx], cellboxes[indx]) - == Direction.east - ): + indx_cellbox = self._get_cellbox_by_id(cellboxes, indx) + if self.neighbour_graph.get_neighbour_case(ne_cellbox, indx_cellbox) == Direction.east: ne_neighbour_map[2].append(indx) if ( - self.neighbour_graph.get_neighbour_case(cellboxes[north_east_indx], cellboxes[indx]) + self.neighbour_graph.get_neighbour_case(ne_cellbox, indx_cellbox) == Direction.south_east ): ne_neighbour_map[3].append(indx) def fill_nw_map( self, - north_west_indx: str, + north_west_indx: str | int, north_neighbour_indx: list[Any], west_neighbour_indx: list[Any], nw_neighbour_map: dict[int, list[Any]], @@ -637,33 +644,30 @@ def fill_nw_map( """ method that fills the North west neighbours """ - cellboxes = self.mesh.cellboxes + cellboxes: list[CellBox | None] = self.mesh.cellboxes # type: ignore[assignment] + nw_cellbox = self._get_cellbox_by_id(cellboxes, north_west_indx) for indx in north_neighbour_indx: - if ( - self.neighbour_graph.get_neighbour_case(cellboxes[north_west_indx], cellboxes[indx]) - == Direction.north - ): + indx_cellbox = self._get_cellbox_by_id(cellboxes, indx) + if self.neighbour_graph.get_neighbour_case(nw_cellbox, indx_cellbox) == Direction.north: nw_neighbour_map[-4].append(indx) if ( - self.neighbour_graph.get_neighbour_case(cellboxes[north_west_indx], cellboxes[indx]) + self.neighbour_graph.get_neighbour_case(nw_cellbox, indx_cellbox) == Direction.north_east ): nw_neighbour_map[1].append(indx) for indx in west_neighbour_indx: - if ( - self.neighbour_graph.get_neighbour_case(cellboxes[north_west_indx], cellboxes[indx]) - == Direction.west - ): + indx_cellbox = self._get_cellbox_by_id(cellboxes, indx) + if self.neighbour_graph.get_neighbour_case(nw_cellbox, indx_cellbox) == Direction.west: nw_neighbour_map[-2].append(indx) if ( - self.neighbour_graph.get_neighbour_case(cellboxes[north_west_indx], cellboxes[indx]) + self.neighbour_graph.get_neighbour_case(nw_cellbox, indx_cellbox) == Direction.south_west ): nw_neighbour_map[-1].append(indx) def fill_sw_neighbour_map( self, - south_west_indx: str, + south_west_indx: str | int, south_neighbour_indx: list[Any], west_neighbour_indx: list[Any], sw_neighbour_map: dict[int, list[Any]], @@ -671,26 +675,23 @@ def fill_sw_neighbour_map( """ method that fills the South west neighbours """ - cellboxes = self.mesh.cellboxes + cellboxes: list[CellBox | None] = self.mesh.cellboxes # type: ignore[assignment] + sw_cellbox = self._get_cellbox_by_id(cellboxes, south_west_indx) for indx in south_neighbour_indx: + indx_cellbox = self._get_cellbox_by_id(cellboxes, indx) if ( - self.neighbour_graph.get_neighbour_case(cellboxes[south_west_indx], cellboxes[indx]) + self.neighbour_graph.get_neighbour_case(sw_cellbox, indx_cellbox) == Direction.south_east ): sw_neighbour_map[3].append(indx) - if ( - self.neighbour_graph.get_neighbour_case(cellboxes[south_west_indx], cellboxes[indx]) - == Direction.south - ): + if self.neighbour_graph.get_neighbour_case(sw_cellbox, indx_cellbox) == Direction.south: sw_neighbour_map[4].append(indx) for indx in west_neighbour_indx: - if ( - self.neighbour_graph.get_neighbour_case(cellboxes[south_west_indx], cellboxes[indx]) - == Direction.west - ): + indx_cellbox = self._get_cellbox_by_id(cellboxes, indx) + if self.neighbour_graph.get_neighbour_case(sw_cellbox, indx_cellbox) == Direction.west: sw_neighbour_map[-2].append(indx) if ( - self.neighbour_graph.get_neighbour_case(cellboxes[south_west_indx], cellboxes[indx]) + self.neighbour_graph.get_neighbour_case(sw_cellbox, indx_cellbox) == Direction.north_west ): sw_neighbour_map[-3].append(indx) @@ -708,6 +709,8 @@ def split_to_depth(self, split_depth: int) -> None: # loop over the data_sources then cellboxes to implement depth-first splitting. should be simpler and loop over cellboxes only once we switch to breadth-first splitting # this impl assumws all the cellboxes have the same data sources. should not be the caase once we switch to breadth-first splitting. data_sources = self.mesh.cellboxes[0].get_data_source() + if data_sources is None: + return # Set up data_source progress bar ds_pbar = tqdm( diff --git a/meshiphi/mesh_generation/neighbour_graph.py b/meshiphi/mesh_generation/neighbour_graph.py index d81cb69f..3a2026af 100644 --- a/meshiphi/mesh_generation/neighbour_graph.py +++ b/meshiphi/mesh_generation/neighbour_graph.py @@ -88,7 +88,7 @@ def update_neighbour(self, index: str, direction: str, neighbours: list[Any]) -> """ self.neighbour_graph[index][direction] = neighbours - def add_neighbour(self, index: str, direction: str, neighbour_indx: Any) -> None: + def add_neighbour(self, index: Any, direction: str, neighbour_indx: Any) -> None: """ adds a neighbour in a certain direction @@ -112,13 +112,13 @@ def remove_node_and_update_neighbours(self, cellbox_index: str) -> None: self.neighbour_graph.pop(cellbox_index) - def get_neighbours(self, cellbox_indx: str, direction: str) -> list[Any]: + def get_neighbours(self, cellbox_indx: Any, direction: str) -> list[Any]: """ returns neighbour in a certain direction """ return cast("list[Any]", self.neighbour_graph[cellbox_indx][direction]) - def add_node(self, index: str, neighbour_map: dict[str, list[Any]]) -> None: + def add_node(self, index: Any, neighbour_map: dict[Any, list[Any]]) -> None: """ method that adds a node to the neighbour_graph at a given index Args: @@ -137,7 +137,7 @@ def remove_node(self, cellbox_index: str) -> None: def update_neighbours( self, - cellbox_indx: str, + cellbox_indx: Any, new_neighbours_indx: list[Any], direction: int, cellboxes: list[Any], @@ -254,44 +254,44 @@ def get_neighbour_case_bounds(self, bounds_a: Any, bounds_b: Any) -> int: lat_b = bounds_b.get_lat_min() if (long_a + bounds_a.get_width()) == long_b and (lat_a + bounds_a.get_height()) == lat_b: - return cast("int", Direction.north_east) + return Direction.north_east if ( (long_a + bounds_a.get_width() == long_b) and (lat_b < (lat_a + bounds_a.get_height())) and ((lat_b + bounds_b.get_height()) > lat_a) ): - return cast("int", Direction.east) + return Direction.east if (long_a + bounds_a.get_width()) == long_b and (lat_a == lat_b + bounds_b.get_height()): - return cast("int", Direction.south_east) + return Direction.south_east if ( ((lat_b + bounds_b.get_height()) == lat_a) and ((long_b + bounds_b.get_width()) > long_a) and (long_b < (long_a + bounds_a.get_width())) ): - return cast("int", Direction.south) + return Direction.south if long_a == (long_b + bounds_b.get_width()) and lat_a == (lat_b + bounds_b.get_height()): - return cast("int", Direction.south_west) + return Direction.south_west if ( (long_b + bounds_b.get_width() == long_a) and (lat_b < (lat_a + bounds_a.get_height())) and ((lat_b + bounds_b.get_height()) > lat_a) ): - return cast("int", Direction.west) + return Direction.west if long_a == (long_b + bounds_b.get_width()) and (lat_a + bounds_a.get_height() == lat_b): - return cast("int", Direction.north_west) + return Direction.north_west if ( (lat_b == (lat_a + bounds_a.get_height())) and ((long_b + bounds_b.get_width()) > long_a) and (long_b < (long_a + bounds_a.get_width())) ): - return cast("int", Direction.north) + return Direction.north return 0 # Cells are not neighbours. @@ -348,50 +348,50 @@ def on_global_bound(cellbox_a: Any, cellbox_b: Any) -> bool: if (long_a + cellbox_a.get_bounds().get_width()) == long_b and ( lat_a + cellbox_a.get_bounds().get_height() ) == lat_b: - return cast("int", Direction.north_east) + return Direction.north_east if ( (long_a + cellbox_a.get_bounds().get_width() == long_b) and (lat_b < (lat_a + cellbox_a.get_bounds().get_height())) and ((lat_b + cellbox_b.get_bounds().get_height()) > lat_a) ): - return cast("int", Direction.east) + return Direction.east if (long_a + cellbox_a.get_bounds().get_width()) == long_b and ( lat_a == lat_b + cellbox_b.get_bounds().get_height() ): - return cast("int", Direction.south_east) + return Direction.south_east if ( ((lat_b + cellbox_b.get_bounds().get_height()) == lat_a) and ((long_b + cellbox_b.get_bounds().get_width()) > long_a) and (long_b < (long_a + cellbox_a.get_bounds().get_width())) ): - return cast("int", Direction.south) + return Direction.south if long_a == (long_b + cellbox_b.get_bounds().get_width()) and lat_a == ( lat_b + cellbox_b.get_bounds().get_height() ): - return cast("int", Direction.south_west) + return Direction.south_west if ( (long_b + cellbox_b.get_bounds().get_width() == long_a) and (lat_b < (lat_a + cellbox_a.get_bounds().get_height())) and ((lat_b + cellbox_b.get_bounds().get_height()) > lat_a) ): - return cast("int", Direction.west) + return Direction.west if long_a == (long_b + cellbox_b.get_bounds().get_width()) and ( lat_a + cellbox_a.get_bounds().get_height() == lat_b ): - return cast("int", Direction.north_west) + return Direction.north_west if ( (lat_b == (lat_a + cellbox_a.get_bounds().get_height())) and ((long_b + cellbox_b.get_bounds().get_width()) > long_a) and (long_b < (long_a + cellbox_a.get_bounds().get_width())) ): - return cast("int", Direction.north) + return Direction.north return 0 # Cells are not neighbours. def get_global_mesh_neighbour_case(self, cellbox_a: Any, cellbox_b: Any) -> int: @@ -424,43 +424,43 @@ def get_global_mesh_neighbour_case(self, cellbox_a: Any, cellbox_b: Any) -> int: if (long_a + cellbox_a.get_bounds().get_width()) == abs(long_b) and ( lat_a + cellbox_a.get_bounds().get_height() ) == lat_b: - return cast("int", Direction.north_east) + return Direction.north_east if ( (long_a + cellbox_a.get_bounds().get_width() == abs(long_b)) and (lat_b < (lat_a + cellbox_a.get_bounds().get_height())) and ((lat_b + cellbox_b.get_bounds().get_height()) > lat_a) ): - return cast("int", Direction.east) + return Direction.east if (long_a + cellbox_a.get_bounds().get_width()) == abs(long_b) and ( lat_a == lat_b + cellbox_b.get_bounds().get_height() ): - return cast("int", Direction.south_east) + return Direction.south_east if ( ((lat_b + cellbox_b.get_bounds().get_height()) == lat_a) and ((long_b + cellbox_b.get_bounds().get_width()) > long_a) and (long_b < (long_a + cellbox_a.get_bounds().get_width())) ): - return cast("int", Direction.south) + return Direction.south if abs(long_a) == (long_b + cellbox_b.get_bounds().get_width()) and lat_a == ( lat_b + cellbox_b.get_bounds().get_height() ): - return cast("int", Direction.south_west) + return Direction.south_west if ( (long_b + cellbox_b.get_bounds().get_width() == abs(long_a)) and (lat_b < (lat_a + cellbox_a.get_bounds().get_height())) and ((lat_b + cellbox_b.get_bounds().get_height()) > lat_a) ): - return cast("int", Direction.west) + return Direction.west if abs(long_a) == (long_b + cellbox_b.get_bounds().get_width()) and ( lat_a + cellbox_a.get_bounds().get_height() == lat_b ): - return cast("int", Direction.north_west) + return Direction.north_west if ( (lat_b == (lat_a + cellbox_a.get_bounds().get_height())) and ((long_b + cellbox_b.get_bounds().get_width()) > long_a) and (long_b < (long_a + cellbox_a.get_bounds().get_width())) ): - return cast("int", Direction.north) + return Direction.north return 0 # Cells are not neighbours. def remove_neighbour(self, index: str, direction: str, neighbour_index: Any) -> None: diff --git a/meshiphi/mesh_plotting/mesh_plotter.py b/meshiphi/mesh_plotting/mesh_plotter.py index 487a5a39..96fa2095 100644 --- a/meshiphi/mesh_plotting/mesh_plotter.py +++ b/meshiphi/mesh_plotting/mesh_plotter.py @@ -1,4 +1,6 @@ -from typing import TYPE_CHECKING, cast +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, cast import cartopy.crs as ccrs import matplotlib @@ -15,7 +17,7 @@ class MeshPlotter: Object for plotting mesh json files using matplotlib and cartopy. """ - def __init__(self, mesh_json, figscale=10): + def __init__(self, mesh_json: dict[str, Any], figscale: int = 10) -> None: self.mesh_json = mesh_json self.mesh_df = pd.DataFrame(self.mesh_json["cellboxes"]) @@ -33,7 +35,7 @@ def __init__(self, mesh_json, figscale=10): self.plot = cast("GeoAxes", plt.axes(projection=ccrs.PlateCarree())) self.plot.set_extent([self.long_min, self.long_max, self.lat_min, self.lat_max]) - def plot_bool(self, value_name, colour): + def plot_bool(self, value_name: str, colour: str) -> None: """ Plots boolean values from the mesh json file. Args: @@ -51,7 +53,7 @@ def plot_bool(self, value_name, colour): edgecolor="darkslategrey", ) - def plot_cmap(self, value_name, colourmap): + def plot_cmap(self, value_name: str, colourmap: str) -> None: """ Plots colourmap values from the mesh json file. Args: @@ -78,7 +80,7 @@ def plot_cmap(self, value_name, colourmap): edgecolor="darkslategrey", ) - def save(self, filename): + def save(self, filename: str) -> None: """ Saves the plot to a file. Args: diff --git a/meshiphi/mesh_validation/full_mesh_validator.py b/meshiphi/mesh_validation/full_mesh_validator.py index b6a812c5..47a748a1 100644 --- a/meshiphi/mesh_validation/full_mesh_validator.py +++ b/meshiphi/mesh_validation/full_mesh_validator.py @@ -1,7 +1,14 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + import pandas as pd from meshiphi.mesh_generation.environment_mesh import EnvironmentMesh +if TYPE_CHECKING: + from meshiphi.mesh_generation.boundary import Boundary + class FullMeshValidator: """ @@ -13,7 +20,7 @@ class FullMeshValidator: MeshValidator object only compares a sample of the source data to the mesh data. """ - def __init__(self, mesh, source_data): + def __init__(self, mesh: dict[str, Any], source_data: pd.DataFrame) -> None: """ Args: mesh: json file containing mesh data @@ -30,7 +37,7 @@ def __init__(self, mesh, source_data): # Validate that the source data is within the require bounds - def validate_vector(self, sd_name, md_name): + def validate_vector(self, sd_name: list[str], md_name: list[str]) -> dict[str, Any]: """ Validate the mesh by comparing the vector field in the mesh to the source data Args: @@ -80,7 +87,7 @@ def validate_vector(self, sd_name, md_name): "root_mean_squared_d_vector_mag": rms_d_vector_mag, } - def validate_scalar(self, sd_name, md_name): + def validate_scalar(self, sd_name: str, md_name: str) -> dict[str, Any]: """ Validate the mesh by comparing the scalar field in the mesh to the source data Args: @@ -116,7 +123,7 @@ def validate_scalar(self, sd_name, md_name): "root_mean_squared_err": rmse, } - def slice_data(self, bounds): + def slice_data(self, bounds: Boundary) -> pd.DataFrame: """ return a slice of the source data within the bounds Args: diff --git a/meshiphi/mesh_validation/mesh_comparator.py b/meshiphi/mesh_validation/mesh_comparator.py index b3142629..3873564c 100644 --- a/meshiphi/mesh_validation/mesh_comparator.py +++ b/meshiphi/mesh_validation/mesh_comparator.py @@ -1,3 +1,7 @@ +from __future__ import annotations + +from typing import Any + import pandas as pd from meshiphi.utils import round_to_sigfig @@ -7,7 +11,9 @@ class MeshComparator: SIG_FIG_TOLERANCE = 4 @staticmethod - def _return_as(df, return_type): + def _return_as( + df: pd.DataFrame, return_type: str | None + ) -> pd.DataFrame | pd.Series[Any] | None: """ Allows for flexibility in how to return differences to user. Sets a return type for each method. @@ -32,7 +38,9 @@ def _return_as(df, return_type): return df return None - def compare_cellbox_boundaries(self, mesh1, mesh2, return_type=None): + def compare_cellbox_boundaries( + self, mesh1: dict[str, Any], mesh2: dict[str, Any], return_type: str | None = None + ) -> pd.DataFrame | pd.Series[Any] | None: """ Compare the boundaries of the cellboxes in the two meshes @@ -68,7 +76,9 @@ def compare_cellbox_boundaries(self, mesh1, mesh2, return_type=None): return self._return_as(mismatched_df, return_type) - def compare_cellbox_values(self, mesh1, mesh2, return_type=None): + def compare_cellbox_values( + self, mesh1: dict[str, Any], mesh2: dict[str, Any], return_type: str | None = None + ) -> pd.DataFrame | pd.Series[Any] | None: """ Compare the values of the cellboxes in the two meshes in cellboxes that both meshes have in common @@ -125,7 +135,9 @@ def compare_cellbox_values(self, mesh1, mesh2, return_type=None): return self._return_as(diff_df, return_type) - def compare_cellbox_attributes(self, mesh1, mesh2, return_type=None): + def compare_cellbox_attributes( + self, mesh1: dict[str, Any], mesh2: dict[str, Any], return_type: str | None = None + ) -> pd.DataFrame | pd.Series[Any] | None: """ Compare the attributes of the cellboxes in the two meshes if the meshes have cellboxes in common @@ -169,7 +181,9 @@ def compare_cellbox_attributes(self, mesh1, mesh2, return_type=None): mismatched_df = pd.DataFrame() return self._return_as(mismatched_df, return_type) - def compare_neighbour_graph_values(self, mesh1, mesh2, return_type=None): + def compare_neighbour_graph_values( + self, mesh1: dict[str, Any], mesh2: dict[str, Any], return_type: str | None = None + ) -> pd.DataFrame | pd.Series[Any] | None: """ Compare the values of the neighbour graph in the two meshes diff --git a/meshiphi/mesh_validation/mesh_validator.py b/meshiphi/mesh_validation/mesh_validator.py index 8d0a9d09..63bcfef4 100644 --- a/meshiphi/mesh_validation/mesh_validator.py +++ b/meshiphi/mesh_validation/mesh_validator.py @@ -3,7 +3,7 @@ import json import logging import math -from typing import Any +from typing import Any, cast import numpy as np import xarray as xr @@ -27,7 +27,7 @@ class MeshValidator: """ - def __init__(self, mesh_config_file): + def __init__(self, mesh_config_file: str) -> None: """ Args: @@ -43,7 +43,7 @@ def __init__(self, mesh_config_file): self.env_mesh = mesh_builder.build_environmental_mesh() self.mesh = mesh_builder.mesh - def validate_mesh(self, number_of_samples=10): + def validate_mesh(self, number_of_samples: int = 10) -> float: """ samples the mesh's lat and long space and compares the actual data within the sampled's range to the mesh agg_value then calculates the RMSE. @@ -71,7 +71,7 @@ def validate_mesh(self, number_of_samples=10): # calculate the RMSE over the samples. return math.sqrt(mean_squared_error(actual_value, mesh_value)) - def get_value_from_data(self, sample): + def get_value_from_data(self, sample: np.ndarray[Any, Any]) -> np.ndarray[Any, Any]: """ gets the actual data within the provided sample lat and long Args: @@ -85,16 +85,19 @@ def get_value_from_data(self, sample): lat_range = [sample[0], lat_end] long_range = [sample[1], long_end] time_range = self.mesh.get_bounds().get_time_range() - for source in self.mesh.cellboxes[0].get_data_source(): + data_source = self.mesh.cellboxes[0].get_data_source() + if data_source is None: + return np.array([]) + for source in data_source: data_loader = source.get_data_loader() data_name = data_loader.get_data_col_name() dp = data_loader.trim_datapoints(Boundary(lat_range, long_range, time_range)) values_array = np.append(values, dp[data_name]) values = values_array.tolist() logger.info("values from data are: {}".format(" ".join(map(str, values)))) - return values + return cast("np.ndarray[Any, Any]", np.array(values, dtype=np.float64)) - def get_range_end(self, sample): + def get_range_end(self, sample: np.ndarray[Any, Any]) -> tuple[float, float]: """ calculates the range end of the provided sample lat and long, claculation is based on the specified validation_length Args: @@ -111,7 +114,7 @@ def get_range_end(self, sample): long_end = self.mesh.get_bounds().get_long_max() return lat_end, long_end - def get_values_from_mesh(self, sample): + def get_values_from_mesh(self, sample: np.ndarray[Any, Any]) -> np.ndarray[Any, Any]: """ finds the mesh's cellboxes that contains the sample's lat and long then returns the aggregated values within. Args: @@ -127,7 +130,10 @@ def get_values_from_mesh(self, sample): long_range = [sample[1], long_end] time_range = self.mesh.get_bounds().get_time_range() - for source in self.mesh.cellboxes[0].get_data_source(): + data_source = self.mesh.cellboxes[0].get_data_source() + if data_source is None: + return cast("np.ndarray[Any, Any]", np.array([0.0])) + for source in data_source: data_loader = source.get_data_loader() dp = data_loader.trim_datapoints(Boundary(lat_range, long_range, time_range)) @@ -145,4 +151,4 @@ def get_values_from_mesh(self, sample): break # break to make sure we avoid getting multiple values (for lat and long on bounds of multiple cellboxes) logger.info("values from mesh are: {}".format(" ".join(map(str, values)))) - return values + return cast("np.ndarray[Any, Any]", np.array(values, dtype=np.float64)) diff --git a/meshiphi/mesh_validation/sampler.py b/meshiphi/mesh_validation/sampler.py index f4f70cfd..c2595d19 100644 --- a/meshiphi/mesh_validation/sampler.py +++ b/meshiphi/mesh_validation/sampler.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Any + import numpy as np from scipy.stats import qmc @@ -14,11 +16,11 @@ class Sampler: """ - def __init__(self, d, n): + def __init__(self, d: int, n: int) -> None: self.dimensions = d self.number_of_samples = n - def generate_samples(self, ranges): + def generate_samples(self, ranges: list[list[float]]) -> np.ndarray[Any, Any]: """ generates samples within the provided ranges array, the length of the ranges should equal to self.dimensions diff --git a/meshiphi/test_automation/test_automater.py b/meshiphi/test_automation/test_automater.py index ad7accc9..50d5d485 100644 --- a/meshiphi/test_automation/test_automater.py +++ b/meshiphi/test_automation/test_automater.py @@ -6,6 +6,7 @@ import shutil import subprocess as sp import tempfile +from typing import Any import geopandas as gpd import matplotlib.pyplot as plt @@ -23,13 +24,13 @@ class TestAutomater: def __init__( self, - from_branch=None, - into_branch="main", - regression=True, - unit=True, - save=False, - plot=False, - ): + from_branch: str | None = None, + into_branch: str = "main", + regression: bool = True, + unit: bool = True, + save: bool = False, + plot: bool = False, + ) -> None: """ Runs through test suite only taking into account relevant tests for the modified files @@ -98,7 +99,13 @@ def __init__( # Finally, remove temp folder shutil.rmtree(temp_dir) - def _run_tests(self, diff_files, test_dir, test_dict, save_to=None): + def _run_tests( + self, + diff_files: list[str], + test_dir: str, + test_dict: dict[str, list[str]], + save_to: str | None = None, + ) -> None: """ Runs relevant regression or unit tests within 'test_dir' @@ -147,7 +154,9 @@ def _run_tests(self, diff_files, test_dir, test_dict, save_to=None): # Change back to repo base directory os.chdir(self.repo_dir) - def run_regression_tests(self, diff_files, save_to=None, plot=False): + def run_regression_tests( + self, diff_files: list[str], save_to: str | None = None, plot: bool = False + ) -> None: """ Runs relevant regression tests for files within 'diff_files' @@ -179,7 +188,7 @@ def run_regression_tests(self, diff_files, save_to=None, plot=False): self.summarise_reg_tests(test_output_path) logger.info(self._single_separator) - def run_unit_tests(self, diff_files, save_to=None): + def run_unit_tests(self, diff_files: list[str], save_to: str | None = None) -> None: """ Runs relevant unit tests for files within 'diff_files' @@ -192,7 +201,7 @@ def run_unit_tests(self, diff_files, save_to=None): logger.info("Attempting unit tests...") self._run_tests(diff_files, unit_test_dir, UNIT_TESTS_BY_FILE, save_to=save_to) - def _setup_output_folder(self): + def _setup_output_folder(self) -> str: """ Creates a 'pytest_meshiphi' folder at the user's current working directory @@ -213,7 +222,9 @@ def _setup_output_folder(self): return output_folder @staticmethod - def parse_pytest_stdout(stdout): + def parse_pytest_stdout( + stdout: str, + ) -> tuple[list[TestInfo], list[TestInfo], list[TestInfo]]: """ Turns stdout of Pytest into TestInfo objects @@ -237,8 +248,8 @@ def parse_pytest_stdout(stdout): # Only read the lines with all necessary info if "::" in line: # Split into pertinent parts - split_line = line.replace("::", " ").replace("[", " ").replace("]", " ") - split_line = split_line.split() + split_line_str = line.replace("::", " ").replace("[", " ").replace("]", " ") + split_line: list[str] = split_line_str.split() status = split_line[0] test_file = split_line[1] test_method = os.path.basename(split_line[2]) @@ -260,7 +271,7 @@ def parse_pytest_stdout(stdout): return passes, fails, errors @staticmethod - def get_base_dir(): + def get_base_dir() -> str: """ Get base folder for repo. """ @@ -272,7 +283,9 @@ def get_base_dir(): return os.path.abspath(repo_path) @staticmethod - def get_diff_filenames(from_branch=None, into_branch=None): + def get_diff_filenames( + from_branch: str | None = None, into_branch: str | None = None + ) -> list[str]: """ Gets a list of files that have changed between 'from_branch' and 'into_branch' @@ -326,7 +339,7 @@ def get_diff_filenames(from_branch=None, into_branch=None): return diff_files - def get_relevant_tests(self, diff_file, test_dict): + def get_relevant_tests(self, diff_file: str, test_dict: dict[str, list[str]]) -> list[str]: """ Determines the relevant tests to run from filename @@ -351,7 +364,7 @@ def get_relevant_tests(self, diff_file, test_dict): return relevant_tests @staticmethod - def extract_test_meshes(test_output_file): + def extract_test_meshes(test_output_file: str) -> tuple[dict[str, Any], dict[str, Any]]: """ Reads test output file and extracts out two meshes; the old 'ground truth' mesh, and the newly generated mesh @@ -371,7 +384,9 @@ def extract_test_meshes(test_output_file): return old_json, new_json - def compare_meshes(self, old_json, new_json): + def compare_meshes( + self, old_json: dict[str, Any], new_json: dict[str, Any] + ) -> dict[str, pd.DataFrame]: """ Runs mesh comparator on old and new mesh stored within test_output_file @@ -396,7 +411,14 @@ def compare_meshes(self, old_json, new_json): "neighbour_graph": mc.compare_neighbour_graph_values(old_json, new_json), } - def save_tests(self, tmp_dir, output_folder, passes=False, fails=True, errors=True): + def save_tests( + self, + tmp_dir: str, + output_folder: str, + passes: bool = False, + fails: bool = True, + errors: bool = True, + ) -> None: """ Saves copy of newly generated test meshes to 'pytest_meshiphi' folder in current working directory. Meshes will be saved as @@ -454,7 +476,7 @@ def save_tests(self, tmp_dir, output_folder, passes=False, fails=True, errors=Tr except OSError as e: logger.debug(e) - def plot_test(self, test_output, save_to=None): + def plot_test(self, test_output: str, save_to: str | None = None) -> None: """ Creates a plot of the differences between the newly generated mesh and the ground truth mesh. The mesh displayed will be the new mesh, with @@ -465,7 +487,14 @@ def plot_test(self, test_output, save_to=None): './pytest_meshiphi/.svg' """ - def add_df_to_ax(df, ax, c="black", ids=False, label=None, a=0.2): + def add_df_to_ax( + df: gpd.GeoDataFrame, + ax: Any, + c: str = "black", + ids: bool = False, + label: str | None = None, + a: float = 0.2, + ) -> tuple[Any, Patch | None]: """ Converts dataframe output from the MeshComparator into a plotable feature @@ -577,7 +606,7 @@ def add_df_to_ax(df, ax, c="black", ids=False, label=None, a=0.2): plt.show() plt.close() - def summarise_reg_tests(self, test_output): + def summarise_reg_tests(self, test_output: str) -> None: """ Write out a summary of the difference in cellboxes in the terminal @@ -585,7 +614,7 @@ def summarise_reg_tests(self, test_output): test_output (str): Filename of saved test mesh """ - def print_summary(comparison, summary_key): + def print_summary(comparison: dict[str, pd.DataFrame], summary_key: str) -> None: """ Prints out a summary of the number of different values in the old mesh compared to the new mesh @@ -620,7 +649,7 @@ def print_summary(comparison, summary_key): print_summary(mesh_comparison, "attributes") print_summary(mesh_comparison, "neighbour_graph") - def summarise_test_stats(self): + def summarise_test_stats(self) -> None: """ Summarise statistics about the tests and print to terminal Example: 10 / 12 tests passed for test_boundary.py @@ -647,5 +676,5 @@ def __init__(self, file: str, test: str, reference: str, status: str) -> None: self.reference = reference self.status = status - def __str__(self): + def __str__(self) -> str: return f"{self.status} - {self.file} > {self.test} > {self.reference}" diff --git a/meshiphi/utils.py b/meshiphi/utils.py index 1f8d425c..a334893b 100644 --- a/meshiphi/utils.py +++ b/meshiphi/utils.py @@ -10,7 +10,7 @@ from calendar import monthrange from datetime import datetime, timedelta from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, Generator, TypeVar, overload +from typing import TYPE_CHECKING, Any, Callable, Generator, TypeVar, cast, overload import numpy as np import numpy.typing as npt @@ -173,7 +173,7 @@ def round_to_sigfig( # Cast as array if not initially, so that later processes all act as expected if orig_type in [int, float, np.float64]: - x = [x] + x = [x] # type: ignore[list-item] x = np.array(x) # Create a mask disabling any values of inf or zero being passed to log10 loggable_idxs = ([x != 0] & np.isfinite(x))[0] @@ -193,12 +193,12 @@ def round_to_sigfig( rounded = np.array([np.around(x[i], decimals=dec_pl[i]) for i in range(len(x))]) # Return as single value if input that way if orig_type in [int, float]: - return rounded.item() + return cast("int | float", rounded.item()) # Return as python list if orig_type is list: - return rounded.tolist() + return cast("list[float]", rounded.tolist()) # Otherwise, return np.array - return rounded + return cast("npt.NDArray[np.floating[Any]]", rounded) def divergence(flow: npt.NDArray[np.floating[Any]]) -> npt.NDArray[np.floating[Any]]: @@ -206,7 +206,7 @@ def divergence(flow: npt.NDArray[np.floating[Any]]) -> npt.NDArray[np.floating[A Fx, Fy = flow[:, :, 0], flow[:, :, 1] dFx_dx = np.gradient(Fx, axis=0) dFy_dy = np.gradient(Fy, axis=1) - return dFx_dx + dFy_dy + return cast("npt.NDArray[np.floating[Any]]", dFx_dx + dFy_dy) def curl(flow: npt.NDArray[np.floating[Any]]) -> npt.NDArray[np.floating[Any]]: @@ -214,7 +214,7 @@ def curl(flow: npt.NDArray[np.floating[Any]]) -> npt.NDArray[np.floating[Any]]: Fx, Fy = flow[:, :, 0], flow[:, :, 1] dFx_dy = np.gradient(Fx, axis=1) dFy_dx = np.gradient(Fy, axis=0) - return dFy_dx - dFx_dy + return cast("npt.NDArray[np.floating[Any]]", dFy_dx - dFx_dy) # GRF functions @@ -236,7 +236,7 @@ def fftind(size: int) -> npt.NDArray[np.int_]: # Create array k_ind = np.mgrid[:size, :size] - int((size + 1) / 2) # Fourier shift - return fftshift(k_ind) + return cast("npt.NDArray[np.int_]", fftshift(k_ind)) def gaussian_random_field(size: int, alpha: float) -> npt.NDArray[np.floating[Any]]: diff --git a/pyproject.toml b/pyproject.toml index 1fc765ee..7158c8b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -173,8 +173,8 @@ line-ending = "auto" python_version = "3.13" warn_return_any = true warn_unused_configs = true -disallow_untyped_defs = false -disallow_incomplete_defs = false +disallow_untyped_defs = true +disallow_incomplete_defs = true check_untyped_defs = true no_implicit_optional = true warn_redundant_casts = true From 3e14adfb4c4ba637f18a3df4ad15a719d8f0494c Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Wed, 21 Jan 2026 11:57:40 +0000 Subject: [PATCH 09/15] add make type-check to contributing.md --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d357cdbb..47484b40 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,6 +24,7 @@ The following commands are available via the Makefile: - **`make clean`** - Clean build artifacts and cache files - **`make docs`** - Build documentation - **`make docs-clean`** - Clean documentation build artifacts +- **`make type-check`** - Run a type check - **`make help`** - Show all available commands with descriptions ## Pre-commit Hooks From 35d5b2dbbbba5a77d73e23145b475b5704476df6 Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Fri, 23 Jan 2026 09:53:37 +0000 Subject: [PATCH 10/15] remove unnecessary string conversions from mesh_builder --- meshiphi/mesh_generation/environment_mesh.py | 268 ++++++++++++------- meshiphi/mesh_generation/mesh_builder.py | 32 +-- meshiphi/mesh_generation/neighbour_graph.py | 90 +++---- tests/unit_tests/test_neighbour_graph.py | 4 +- 4 files changed, 228 insertions(+), 166 deletions(-) diff --git a/meshiphi/mesh_generation/environment_mesh.py b/meshiphi/mesh_generation/environment_mesh.py index 7d160941..9ba8b3fa 100644 --- a/meshiphi/mesh_generation/environment_mesh.py +++ b/meshiphi/mesh_generation/environment_mesh.py @@ -17,6 +17,7 @@ from meshiphi import __version__ as meshiphi_version from meshiphi.mesh_generation.aggregated_cellbox import AggregatedCellBox from meshiphi.mesh_generation.boundary import Boundary +from meshiphi.mesh_generation.direction import Direction from meshiphi.mesh_generation.neighbour_graph import NeighbourGraph logger = logging.getLogger(__name__) @@ -824,22 +825,25 @@ def split_and_replace(self, cellbox_id: str) -> None: south_east_index = int(split_cellboxes[2].get_agg_data()["id"]) # Get neighbours of original cellbox - south_neighbour_index = self.neighbour_graph.get_neighbours(cellbox_id, "4") - north_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_id, "-4") - east_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_id, "2") - west_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_id, "-2") + cellbox_id_int = int(cellbox_id) + south_neighbour_index = self.neighbour_graph.get_neighbours(cellbox_id_int, Direction.south) + north_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_id_int, Direction.north) + east_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_id_int, Direction.east) + west_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_id_int, Direction.west) # ================== Create Neighbour Maps ================== # initialize the neighbour map of SW cellbox sw_neighbour_map = { - "1": [north_east_index], - "2": [south_east_index], - "3": [], - "4": [], - "-1": self.neighbour_graph.get_neighbours(cellbox_id, "-1"), - "-2": [], - "-3": [], - "-4": [north_west_index], + Direction.north_east: [north_east_index], + Direction.east: [south_east_index], + Direction.south_east: [], + Direction.south: [], + Direction.south_west: self.neighbour_graph.get_neighbours( + cellbox_id_int, Direction.south_west + ), + Direction.west: [], + Direction.north_west: [], + Direction.north: [north_west_index], } # fill remaining neighbour map sw_neighbour_map = self.fill_sw_neighbour_map( @@ -848,18 +852,20 @@ def split_and_replace(self, cellbox_id: str) -> None: south_neighbour_index, west_neighbour_indx, ) - self.neighbour_graph.add_node(str(south_west_index), sw_neighbour_map) + self.neighbour_graph.add_node(south_west_index, sw_neighbour_map) # initialize the neighbour map of NW cellbox nw_neighbour_map = { - "1": [], - "2": [north_east_index], - "3": [south_east_index], - "4": [south_west_index], - "-1": [], - "-2": [], - "-3": self.neighbour_graph.get_neighbours(cellbox_id, "-3"), - "-4": [], + Direction.north_east: [], + Direction.east: [north_east_index], + Direction.south_east: [south_east_index], + Direction.south: [south_west_index], + Direction.south_west: [], + Direction.west: [], + Direction.north_west: self.neighbour_graph.get_neighbours( + cellbox_id_int, Direction.north_west + ), + Direction.north: [], } # fill remaining neighbour map nw_neighbour_map = self.fill_nw_neighbour_map( @@ -868,18 +874,20 @@ def split_and_replace(self, cellbox_id: str) -> None: north_neighbour_indx, west_neighbour_indx, ) - self.neighbour_graph.add_node(str(north_west_index), nw_neighbour_map) + self.neighbour_graph.add_node(north_west_index, nw_neighbour_map) # initialize the neighbour map of NE cellbox ne_neighbour_map = { - "1": self.neighbour_graph.get_neighbours(cellbox_id, "1"), - "2": [], - "3": [], - "4": [south_east_index], - "-1": [south_west_index], - "-2": [north_west_index], - "-3": [], - "-4": [], + Direction.north_east: self.neighbour_graph.get_neighbours( + cellbox_id_int, Direction.north_east + ), + Direction.east: [], + Direction.south_east: [], + Direction.south: [south_east_index], + Direction.south_west: [south_west_index], + Direction.west: [north_west_index], + Direction.north_west: [], + Direction.north: [], } # fill remaining neighbour map ne_neighbour_map = self.fill_ne_neighbour_map( @@ -888,18 +896,20 @@ def split_and_replace(self, cellbox_id: str) -> None: north_neighbour_indx, east_neighbour_indx, ) - self.neighbour_graph.add_node(str(north_east_index), ne_neighbour_map) + self.neighbour_graph.add_node(north_east_index, ne_neighbour_map) # initialize the neighbour map of SE cellbox se_neighbour_map = { - "1": [], - "2": [], - "3": self.neighbour_graph.get_neighbours(cellbox_id, "3"), - "4": [], - "-1": [], - "-2": [south_west_index], - "-3": [north_west_index], - "-4": [north_east_index], + Direction.north_east: [], + Direction.east: [], + Direction.south_east: self.neighbour_graph.get_neighbours( + cellbox_id_int, Direction.south_east + ), + Direction.south: [], + Direction.south_west: [], + Direction.west: [south_west_index], + Direction.north_west: [north_west_index], + Direction.north: [north_east_index], } # fill remaining neighbour map se_neighbour_map = self.fill_se_neighbour_map( @@ -908,24 +918,32 @@ def split_and_replace(self, cellbox_id: str) -> None: south_neighbour_index, east_neighbour_indx, ) - self.neighbour_graph.add_node(str(south_east_index), se_neighbour_map) + self.neighbour_graph.add_node(south_east_index, se_neighbour_map) # ================== Update Neighbours of original cellbox ================== # update corner neighbours - ne_corner_index = self.neighbour_graph.get_neighbours(cellbox_id, "1") - se_corner_index = self.neighbour_graph.get_neighbours(cellbox_id, "3") - nw_corner_index = self.neighbour_graph.get_neighbours(cellbox_id, "-3") - sw_corner_index = self.neighbour_graph.get_neighbours(cellbox_id, "-1") + ne_corner_index = self.neighbour_graph.get_neighbours(cellbox_id_int, Direction.north_east) + se_corner_index = self.neighbour_graph.get_neighbours(cellbox_id_int, Direction.south_east) + nw_corner_index = self.neighbour_graph.get_neighbours(cellbox_id_int, Direction.north_west) + sw_corner_index = self.neighbour_graph.get_neighbours(cellbox_id_int, Direction.south_west) if len(ne_corner_index) > 0: - self.neighbour_graph.update_neighbour(str(ne_corner_index[0]), "-1", [north_east_index]) + self.neighbour_graph.update_neighbour( + ne_corner_index[0], Direction.south_west, [north_east_index] + ) if len(se_corner_index) > 0: - self.neighbour_graph.update_neighbour(str(se_corner_index[0]), "-3", [south_east_index]) + self.neighbour_graph.update_neighbour( + se_corner_index[0], Direction.north_west, [south_east_index] + ) if len(nw_corner_index) > 0: - self.neighbour_graph.update_neighbour(str(nw_corner_index[0]), "3", [north_west_index]) + self.neighbour_graph.update_neighbour( + nw_corner_index[0], Direction.south_east, [north_west_index] + ) if len(sw_corner_index) > 0: - self.neighbour_graph.update_neighbour(str(sw_corner_index[0]), "1", [south_west_index]) + self.neighbour_graph.update_neighbour( + sw_corner_index[0], Direction.north_east, [south_west_index] + ) # update sides neighbours ne_bounds = self.get_cellbox(north_east_index).get_bounds() @@ -934,7 +952,7 @@ def split_and_replace(self, cellbox_id: str) -> None: sw_bounds = self.get_cellbox(south_west_index).get_bounds() # update north cellbox neighbours - self.neighbour_graph.remove_node_from_neighbours(cellbox_id, -4) + self.neighbour_graph.remove_node_from_neighbours(cellbox_id_int, Direction.north) for indx in north_neighbour_indx: cellbox_bounds = self.get_cellbox(indx).get_bounds() @@ -943,16 +961,16 @@ def split_and_replace(self, cellbox_id: str) -> None: cellbox_bounds, ne_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), north_east_index) + self.neighbour_graph.add_neighbour(indx, crossing_case, north_east_index) crossing_case = self.neighbour_graph.get_neighbour_case_bounds( cellbox_bounds, nw_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), north_west_index) + self.neighbour_graph.add_neighbour(indx, crossing_case, north_west_index) # update south cellbox neighbours - self.neighbour_graph.remove_node_from_neighbours(cellbox_id, 4) + self.neighbour_graph.remove_node_from_neighbours(cellbox_id_int, Direction.south) for indx in south_neighbour_index: cellbox_bounds = self.get_cellbox(indx).get_bounds() @@ -961,16 +979,16 @@ def split_and_replace(self, cellbox_id: str) -> None: cellbox_bounds, se_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), south_east_index) + self.neighbour_graph.add_neighbour(indx, crossing_case, south_east_index) crossing_case = self.neighbour_graph.get_neighbour_case_bounds( cellbox_bounds, sw_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), south_west_index) + self.neighbour_graph.add_neighbour(indx, crossing_case, south_west_index) # update east cellbox neighbours - self.neighbour_graph.remove_node_from_neighbours(cellbox_id, 2) + self.neighbour_graph.remove_node_from_neighbours(cellbox_id_int, Direction.east) for indx in east_neighbour_indx: cellbox_bounds = self.get_cellbox(indx).get_bounds() @@ -979,16 +997,16 @@ def split_and_replace(self, cellbox_id: str) -> None: cellbox_bounds, ne_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), north_east_index) + self.neighbour_graph.add_neighbour(indx, crossing_case, north_east_index) crossing_case = self.neighbour_graph.get_neighbour_case_bounds( cellbox_bounds, se_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), south_east_index) + self.neighbour_graph.add_neighbour(indx, crossing_case, south_east_index) # update west cellbox neighbours - self.neighbour_graph.remove_node_from_neighbours(cellbox_id, -2) + self.neighbour_graph.remove_node_from_neighbours(cellbox_id_int, Direction.west) for indx in west_neighbour_indx: cellbox_bounds = self.get_cellbox(indx).get_bounds() @@ -997,26 +1015,26 @@ def split_and_replace(self, cellbox_id: str) -> None: cellbox_bounds, nw_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), north_west_index) + self.neighbour_graph.add_neighbour(indx, crossing_case, north_west_index) crossing_case = self.neighbour_graph.get_neighbour_case_bounds( cellbox_bounds, sw_bounds ) if crossing_case != 0: - self.neighbour_graph.add_neighbour(str(indx), str(crossing_case), south_west_index) + self.neighbour_graph.add_neighbour(indx, crossing_case, south_west_index) # remove original cellbox from mesh. cellbox = self.get_cellbox(cellbox_id) self.agg_cellboxes.remove(cellbox) - self.neighbour_graph.remove_node(cellbox_id) + self.neighbour_graph.remove_node(cellbox_id_int) def fill_se_neighbour_map( self, - se_neighbour_map: dict[str, list[Any]], + se_neighbour_map: dict[int, list[Any]], se_neighbour_id: int, south_neighbour_index: list[Any], east_neighbour_indx: list[Any], - ) -> dict[str, list[Any]]: + ) -> dict[int, list[Any]]: """ fills the south east neighbour map with the given values Args: @@ -1031,27 +1049,39 @@ def fill_se_neighbour_map( for indx in south_neighbour_index: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if self.neighbour_graph.get_neighbour_case_bounds(se_bounds, neighbour_bounds) == 4: - se_neighbour_map["4"].append(indx) - if self.neighbour_graph.get_neighbour_case_bounds(se_bounds, neighbour_bounds) == -1: - se_neighbour_map["-1"].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(se_bounds, neighbour_bounds) + == Direction.south + ): + se_neighbour_map[Direction.south].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(se_bounds, neighbour_bounds) + == Direction.south_west + ): + se_neighbour_map[Direction.south_west].append(indx) for indx in east_neighbour_indx: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if self.neighbour_graph.get_neighbour_case_bounds(se_bounds, neighbour_bounds) == 2: - se_neighbour_map["2"].append(indx) - if self.neighbour_graph.get_neighbour_case_bounds(se_bounds, neighbour_bounds) == 1: - se_neighbour_map["1"].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(se_bounds, neighbour_bounds) + == Direction.east + ): + se_neighbour_map[Direction.east].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(se_bounds, neighbour_bounds) + == Direction.north_east + ): + se_neighbour_map[Direction.north_east].append(indx) return se_neighbour_map def fill_ne_neighbour_map( self, - ne_neighbour_map: dict[str, list[Any]], + ne_neighbour_map: dict[int, list[Any]], ne_neighbour_id: int, north_neighbour_indx: list[Any], east_neighbour_indx: list[Any], - ) -> dict[str, list[Any]]: + ) -> dict[int, list[Any]]: """ fills the north east neighbour map with the given values Args: @@ -1066,27 +1096,39 @@ def fill_ne_neighbour_map( for indx in north_neighbour_indx: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if self.neighbour_graph.get_neighbour_case_bounds(ne_bounds, neighbour_bounds) == -4: - ne_neighbour_map["-4"].append(indx) - if self.neighbour_graph.get_neighbour_case_bounds(ne_bounds, neighbour_bounds) == -3: - ne_neighbour_map["-3"].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(ne_bounds, neighbour_bounds) + == Direction.north + ): + ne_neighbour_map[Direction.north].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(ne_bounds, neighbour_bounds) + == Direction.north_west + ): + ne_neighbour_map[Direction.north_west].append(indx) for indx in east_neighbour_indx: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if self.neighbour_graph.get_neighbour_case_bounds(ne_bounds, neighbour_bounds) == 2: - ne_neighbour_map["2"].append(indx) - if self.neighbour_graph.get_neighbour_case_bounds(ne_bounds, neighbour_bounds) == 3: - ne_neighbour_map["3"].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(ne_bounds, neighbour_bounds) + == Direction.east + ): + ne_neighbour_map[Direction.east].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(ne_bounds, neighbour_bounds) + == Direction.south_east + ): + ne_neighbour_map[Direction.south_east].append(indx) return ne_neighbour_map def fill_sw_neighbour_map( self, - sw_neighbour_map: dict[str, list[Any]], + sw_neighbour_map: dict[int, list[Any]], sw_neighbour_id: int, south_neighbour_index: list[Any], west_neighbour_indx: list[Any], - ) -> dict[str, list[Any]]: + ) -> dict[int, list[Any]]: """ fills the south west neighbour map with the given values Args: @@ -1101,27 +1143,39 @@ def fill_sw_neighbour_map( for indx in south_neighbour_index: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if self.neighbour_graph.get_neighbour_case_bounds(sw_bounds, neighbour_bounds) == 3: - sw_neighbour_map["3"].append(indx) - if self.neighbour_graph.get_neighbour_case_bounds(sw_bounds, neighbour_bounds) == 4: - sw_neighbour_map["4"].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(sw_bounds, neighbour_bounds) + == Direction.south_east + ): + sw_neighbour_map[Direction.south_east].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(sw_bounds, neighbour_bounds) + == Direction.south + ): + sw_neighbour_map[Direction.south].append(indx) for indx in west_neighbour_indx: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if self.neighbour_graph.get_neighbour_case_bounds(sw_bounds, neighbour_bounds) == -2: - sw_neighbour_map["-2"].append(indx) - if self.neighbour_graph.get_neighbour_case_bounds(sw_bounds, neighbour_bounds) == -3: - sw_neighbour_map["-3"].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(sw_bounds, neighbour_bounds) + == Direction.west + ): + sw_neighbour_map[Direction.west].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(sw_bounds, neighbour_bounds) + == Direction.north_west + ): + sw_neighbour_map[Direction.north_west].append(indx) return sw_neighbour_map def fill_nw_neighbour_map( self, - nw_neighbour_map: dict[str, list[Any]], + nw_neighbour_map: dict[int, list[Any]], nw_neighbour_id: int, north_neighbour_indx: list[Any], west_neighbour_indx: list[Any], - ) -> dict[str, list[Any]]: + ) -> dict[int, list[Any]]: """ fills the north west neighbour map with the given values Args: @@ -1136,17 +1190,29 @@ def fill_nw_neighbour_map( for indx in north_neighbour_indx: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if self.neighbour_graph.get_neighbour_case_bounds(nw_bounds, neighbour_bounds) == -4: - nw_neighbour_map["-4"].append(indx) - if self.neighbour_graph.get_neighbour_case_bounds(nw_bounds, neighbour_bounds) == 1: - nw_neighbour_map["1"].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(nw_bounds, neighbour_bounds) + == Direction.north + ): + nw_neighbour_map[Direction.north].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(nw_bounds, neighbour_bounds) + == Direction.north_east + ): + nw_neighbour_map[Direction.north_east].append(indx) for indx in west_neighbour_indx: neighbour_bounds = self.get_cellbox(indx).get_bounds() - if self.neighbour_graph.get_neighbour_case_bounds(nw_bounds, neighbour_bounds) == -2: - nw_neighbour_map["-2"].append(indx) - if self.neighbour_graph.get_neighbour_case_bounds(nw_bounds, neighbour_bounds) == -1: - nw_neighbour_map["-1"].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(nw_bounds, neighbour_bounds) + == Direction.west + ): + nw_neighbour_map[Direction.west].append(indx) + if ( + self.neighbour_graph.get_neighbour_case_bounds(nw_bounds, neighbour_bounds) + == Direction.south_west + ): + nw_neighbour_map[Direction.south_west].append(indx) return nw_neighbour_map diff --git a/meshiphi/mesh_generation/mesh_builder.py b/meshiphi/mesh_generation/mesh_builder.py index 66192bb5..507250b6 100644 --- a/meshiphi/mesh_generation/mesh_builder.py +++ b/meshiphi/mesh_generation/mesh_builder.py @@ -368,35 +368,35 @@ def check_global_mesh( for i in range(len(min_long_cellboxes)): self.neighbour_graph.add_neighbour( int(min_long_cellboxes[i].get_id()), - str(Direction.west), + Direction.west, int(max_long_cellboxes[i].get_id()), ) self.neighbour_graph.add_neighbour( int(max_long_cellboxes[i].get_id()), - str(Direction.east), + Direction.east, int(min_long_cellboxes[i].get_id()), ) # checks to avoid the very upper and lower cellboxes as they do not have north/south neighbours if 0 <= i < len(min_long_cellboxes) - 1: self.neighbour_graph.add_neighbour( int(min_long_cellboxes[i].get_id()), - str(Direction.north_west), + Direction.north_west, int(max_long_cellboxes[i + 1].get_id()), ) self.neighbour_graph.add_neighbour( int(max_long_cellboxes[i].get_id()), - str(Direction.north_east), + Direction.north_east, int(min_long_cellboxes[i + 1].get_id()), ) if 0 < i <= len(min_long_cellboxes) - 1: self.neighbour_graph.add_neighbour( int(min_long_cellboxes[i].get_id()), - str(Direction.south_west), + Direction.south_west, int(max_long_cellboxes[i - 1].get_id()), ) self.neighbour_graph.add_neighbour( int(max_long_cellboxes[i].get_id()), - str(Direction.south_east), + Direction.south_east, int(min_long_cellboxes[i - 1].get_id()), ) @@ -443,10 +443,10 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: south_west_indx = cellboxes.index(split_cellboxes[2]) south_east_indx = cellboxes.index(split_cellboxes[3]) - south_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, "4") - north_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, "-4") - east_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, "2") - west_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, "-2") + south_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, Direction.south) + north_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, Direction.north) + east_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, Direction.east) + west_neighbour_indx = self.neighbour_graph.get_neighbours(cellbox_indx, Direction.west) # Create neighbour map for SW split cell. sw_neighbour_map = { @@ -456,7 +456,7 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: Direction.south: [], # update to use the NG class Direction.south_west: self.neighbour_graph.get_neighbours( - cellbox_indx, str(Direction.south_west) + cellbox_indx, Direction.south_west ), Direction.west: [], Direction.north_west: [], @@ -477,7 +477,7 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: Direction.south_west: [], Direction.west: [], Direction.north_west: self.neighbour_graph.get_neighbours( - cellbox_indx, str(Direction.north_west) + cellbox_indx, Direction.north_west ), Direction.north: [], } @@ -490,7 +490,7 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: # Create neighbour map for NE split cell ne_neighbour_map = { Direction.north_east: self.neighbour_graph.get_neighbours( - cellbox_indx, str(Direction.north_east) + cellbox_indx, Direction.north_east ), Direction.east: [], Direction.south_east: [], @@ -510,7 +510,7 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: Direction.north_east: [], Direction.east: [], Direction.south_east: self.neighbour_graph.get_neighbours( - cellbox_indx, str(Direction.south_east) + cellbox_indx, Direction.south_east ), Direction.south: [], Direction.south_west: [], @@ -544,7 +544,7 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: ) # Update corner neighbour maps self.neighbour_graph.update_corner_neighbours( - str(cellbox_indx), + cellbox_indx, north_west_indx, north_east_indx, south_west_indx, @@ -552,7 +552,7 @@ def split_and_replace(self, cellbox: CellBox) -> list[CellBox]: ) # remove the original splitted cellbox from the neighbour_graph - self.neighbour_graph.remove_node(str(cellbox_indx)) + self.neighbour_graph.remove_node(cellbox_indx) # set the original splitted cellbox to None cellboxes[cellbox_indx] = None # type: ignore[call-overload] diff --git a/meshiphi/mesh_generation/neighbour_graph.py b/meshiphi/mesh_generation/neighbour_graph.py index 3a2026af..bde8bf78 100644 --- a/meshiphi/mesh_generation/neighbour_graph.py +++ b/meshiphi/mesh_generation/neighbour_graph.py @@ -36,7 +36,7 @@ class NeighbourGraph: def __init__(self, cellboxes: list[Any] | None = None, grid_width: int = 0) -> None: # initialize graph with an empty one - self.neighbour_graph: dict[str, Any] = {} + self.neighbour_graph: dict[int, Any] = {} if cellboxes is None: cellboxes = [] self.initialise_neighbour_graph(cellboxes, grid_width) @@ -49,9 +49,15 @@ def from_json(cls, ng_json: dict[str, Any]) -> NeighbourGraph: Args: ng_json (json_object): json object that contains the neighbour_graph data """ - neighbour_graph = {} + neighbour_graph: dict[int, Any] = {} for key in ng_json: - neighbour_graph[key] = ng_json[key] + # Convert string node key to integer + node_key = int(key) + # Convert string direction keys to integers in neighbour map + neighbour_map: dict[int, list[Any]] = {} + for dir_key, neighbours in ng_json[key].items(): + neighbour_map[int(dir_key)] = neighbours + neighbour_graph[node_key] = neighbour_map obj = NeighbourGraph() obj.neighbour_graph = neighbour_graph return obj @@ -65,30 +71,30 @@ def increment_ids(self, inc: int) -> None: """ ng = self.get_graph() - new_ng = {} + new_ng: dict[int, Any] = {} for node in ng: for direction in ng[node]: inc_neighbours = [x + inc for x in ng[node][direction]] ng[node][direction] = inc_neighbours - new_ng[str(int(node) + inc)] = ng[node] + new_ng[int(node) + inc] = ng[node] self.neighbour_graph = new_ng - def get_graph(self) -> dict[str, Any]: + def get_graph(self) -> dict[int, Any]: """ returns the graph dict """ return self.neighbour_graph - def update_neighbour(self, index: str, direction: str, neighbours: list[Any]) -> None: + def update_neighbour(self, index: int, direction: int, neighbours: list[Any]) -> None: """ updates the neighbour in a certain direction """ self.neighbour_graph[index][direction] = neighbours - def add_neighbour(self, index: Any, direction: str, neighbour_indx: Any) -> None: + def add_neighbour(self, index: int, direction: int, neighbour_indx: Any) -> None: """ adds a neighbour in a certain direction @@ -99,7 +105,7 @@ def add_neighbour(self, index: Any, direction: str, neighbour_indx: Any) -> None """ self.neighbour_graph[index][direction].append(neighbour_indx) - def remove_node_and_update_neighbours(self, cellbox_index: str) -> None: + def remove_node_and_update_neighbours(self, cellbox_index: int) -> None: """ method that removes a node in the neighbour_graph at a given index. remove_node_from_neighbours should be called first. Args: @@ -112,13 +118,13 @@ def remove_node_and_update_neighbours(self, cellbox_index: str) -> None: self.neighbour_graph.pop(cellbox_index) - def get_neighbours(self, cellbox_indx: Any, direction: str) -> list[Any]: + def get_neighbours(self, cellbox_indx: int, direction: int) -> list[Any]: """ returns neighbour in a certain direction """ return cast("list[Any]", self.neighbour_graph[cellbox_indx][direction]) - def add_node(self, index: Any, neighbour_map: dict[Any, list[Any]]) -> None: + def add_node(self, index: int, neighbour_map: dict[int, list[Any]]) -> None: """ method that adds a node to the neighbour_graph at a given index Args: @@ -127,7 +133,7 @@ def add_node(self, index: Any, neighbour_map: dict[Any, list[Any]]) -> None: """ self.neighbour_graph[index] = neighbour_map - def remove_node(self, cellbox_index: str) -> None: + def remove_node(self, cellbox_index: int) -> None: """ method that removes a node to the neighbour_graph at a given index Args: @@ -137,7 +143,7 @@ def remove_node(self, cellbox_index: str) -> None: def update_neighbours( self, - cellbox_indx: Any, + cellbox_indx: int, new_neighbours_indx: list[Any], direction: int, cellboxes: list[Any], @@ -170,29 +176,21 @@ def update_neighbours( if crossing_case != 0: self.neighbour_graph[indx][crossing_case].append(new_neighbours_indx[1]) - def remove_node_from_neighbours(self, cellbox_indx: str, direction: int) -> None: + def remove_node_from_neighbours(self, cellbox_indx: int, direction: int) -> None: """ method that goes through neighbours in a given direction and remove cellbox_index from their neighbour_maps Args: cellbox_indx (int): the index of the cellbox that we will go through its neighbours and remove this index from their neighbour_map direction (int): an int that represents the direction of the neighbours that will get updated (e.g. north, south ,..) """ - - # Try with int keys first as in mesh construction then try with str keys for json mesh - try: - neighbour_indx_list = self.neighbour_graph[cellbox_indx][direction] - except KeyError: - neighbour_indx_list = self.neighbour_graph[cellbox_indx][str(direction)] + neighbour_indx_list = self.neighbour_graph[cellbox_indx][direction] for indx in neighbour_indx_list: - try: - self.neighbour_graph[indx][-1 * direction].remove(cellbox_indx) - except KeyError: - self.neighbour_graph[str(indx)][str(-1 * direction)].remove(int(cellbox_indx)) + self.neighbour_graph[indx][-1 * direction].remove(cellbox_indx) def update_corner_neighbours( self, - cellbox_indx: str, + cellbox_indx: int, north_west_indx: Any, north_east_indx: Any, south_west_indx: Any, @@ -201,29 +199,27 @@ def update_corner_neighbours( """ method that updates the corner neighbours of cellbox_indx with the given indeces """ - north_east_corner_indx = self.neighbour_graph[cellbox_indx][Direction.north_east] + node_map = self.neighbour_graph[cellbox_indx] + + north_east_corner_indx = node_map[Direction.north_east] if len(north_east_corner_indx) > 0: - self.neighbour_graph[north_east_corner_indx[0]][Direction.south_west] = [ - north_east_indx - ] + corner_idx = north_east_corner_indx[0] + self.neighbour_graph[corner_idx][Direction.south_west] = [north_east_indx] - north_west_corner_indx = self.neighbour_graph[cellbox_indx][Direction.north_west] + north_west_corner_indx = node_map[Direction.north_west] if len(north_west_corner_indx) > 0: - self.neighbour_graph[north_west_corner_indx[0]][Direction.south_east] = [ - north_west_indx - ] + corner_idx = north_west_corner_indx[0] + self.neighbour_graph[corner_idx][Direction.south_east] = [north_west_indx] - south_east_corner_indx = self.neighbour_graph[cellbox_indx][Direction.south_east] + south_east_corner_indx = node_map[Direction.south_east] if len(south_east_corner_indx) > 0: - self.neighbour_graph[south_east_corner_indx[0]][Direction.north_west] = [ - south_east_indx - ] + corner_idx = south_east_corner_indx[0] + self.neighbour_graph[corner_idx][Direction.north_west] = [south_east_indx] - south_west_corner_indx = self.neighbour_graph[cellbox_indx][Direction.south_west] + south_west_corner_indx = node_map[Direction.south_west] if len(south_west_corner_indx) > 0: - self.neighbour_graph[south_west_corner_indx[0]][Direction.north_east] = [ - south_west_indx - ] + corner_idx = south_west_corner_indx[0] + self.neighbour_graph[corner_idx][Direction.north_east] = [south_west_indx] def get_neighbour_case_bounds(self, bounds_a: Any, bounds_b: Any) -> int: """ @@ -463,7 +459,7 @@ def get_global_mesh_neighbour_case(self, cellbox_a: Any, cellbox_b: Any) -> int: return Direction.north return 0 # Cells are not neighbours. - def remove_neighbour(self, index: str, direction: str, neighbour_index: Any) -> None: + def remove_neighbour(self, index: int, direction: int, neighbour_index: Any) -> None: """ remove certain neighbour in a specific direction """ @@ -477,15 +473,15 @@ def initialise_neighbour_graph(self, cellboxes: list[Any], grid_width: int) -> N cellbox_indx = cellboxes.index(cellbox) neighbour_map = self.initialise_map(cellbox_indx, grid_width, len(cellboxes)) - self.add_node(str(cellbox_indx), neighbour_map) + self.add_node(cellbox_indx, neighbour_map) def initialise_map( self, cellbox_indx: int, grid_width: int, cellboxes_length: int - ) -> dict[str, list[Any]]: + ) -> dict[int, list[Any]]: """ initialse the neighbour map of a cellbox with the given cellbox_index """ - neighbour_map: dict[Any, list[Any]] = { + neighbour_map: dict[int, list[Any]] = { Direction.north_east: [], Direction.east: [], Direction.south_east: [], @@ -531,5 +527,5 @@ def set_global_mesh(self, is_global: bool) -> None: def is_global_mesh(self) -> bool: return self._is_global_mesh - def get_neighbour_map(self, _id: str) -> dict[str, list[Any]]: - return cast("dict[str, list[Any]]", self.neighbour_graph[_id]) + def get_neighbour_map(self, _id: int) -> dict[int, list[Any]]: + return cast("dict[int, list[Any]]", self.neighbour_graph[_id]) diff --git a/tests/unit_tests/test_neighbour_graph.py b/tests/unit_tests/test_neighbour_graph.py index 44e0c705..7fcec42d 100644 --- a/tests/unit_tests/test_neighbour_graph.py +++ b/tests/unit_tests/test_neighbour_graph.py @@ -100,9 +100,9 @@ def test_increment_ids(ng_dict_3x3): ng.increment_ids(10) # Add 'increment' to nodes and neighbours stored within the neighbourgraph dict - # Creates a dict of form {str(node + increment): {direction: [neighbours + increment]}} + # Creates a dict of form {int(node + increment): {direction: [neighbours + increment]}} manually_incremented_dict = { - str(int(node) + increment): { + int(node) + increment: { direction: [neighbour + increment for neighbour in neighbours] for direction, neighbours in dir_map.items() } From f975fb1ba109d67dadff9b73a12a19b0881892c2 Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Wed, 28 Jan 2026 10:48:59 +0000 Subject: [PATCH 11/15] update project proposal template to include documents section and main proposal lead --- meshiphi/dataloaders/scalar/scalar_grf.py | 5 ++- meshiphi/dataloaders/scalar/shape.py | 19 ++++------ meshiphi/dataloaders/scalar/visual_iced.py | 4 +- meshiphi/dataloaders/vector/vector_grf.py | 7 ++-- meshiphi/dataloaders/vector/vector_shape.py | 15 +++----- meshiphi/mesh_generation/boundary.py | 23 ++++++------ meshiphi/mesh_generation/environment_mesh.py | 39 +++++++++++++++----- meshiphi/utils.py | 38 +++++++++++-------- pyproject.toml | 4 +- 9 files changed, 87 insertions(+), 67 deletions(-) diff --git a/meshiphi/dataloaders/scalar/scalar_grf.py b/meshiphi/dataloaders/scalar/scalar_grf.py index 38bc2b55..60e52a8a 100644 --- a/meshiphi/dataloaders/scalar/scalar_grf.py +++ b/meshiphi/dataloaders/scalar/scalar_grf.py @@ -4,12 +4,13 @@ import numpy as np import pandas as pd -import xarray as xr # noqa: TC002 from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader from meshiphi.utils import gaussian_random_field if TYPE_CHECKING: + import xarray as xr + from meshiphi.mesh_generation.boundary import Boundary @@ -147,4 +148,4 @@ def grf_to_scalar( # Cast to dataframe data = pd.DataFrame(rows).set_index(["lat", "long"]) # Set to xarray dataset - return data.to_xarray() # type: ignore[no-any-return] + return cast("xr.Dataset", data.to_xarray()) diff --git a/meshiphi/dataloaders/scalar/shape.py b/meshiphi/dataloaders/scalar/shape.py index 06347bf8..56e85f18 100644 --- a/meshiphi/dataloaders/scalar/shape.py +++ b/meshiphi/dataloaders/scalar/shape.py @@ -1,15 +1,16 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import numpy as np import pandas as pd -import xarray as xr # noqa: TC002 from meshiphi.dataloaders.scalar.abstract_scalar import ScalarDataLoader if TYPE_CHECKING: + import xarray as xr + from meshiphi.mesh_generation.boundary import Boundary logger = logging.getLogger(__name__) @@ -99,7 +100,7 @@ def import_data(self, bounds: Boundary) -> xr.Dataset: raise ValueError(f"Unknown abstract shape type: {self.dataloader_name}") # No need to trim data, as was defined by bounds - return data.set_index(["lat", "long"]).to_xarray() # type: ignore[no-any-return] + return cast("xr.Dataset", data.set_index(["lat", "long"]).to_xarray()) def gen_circle(self, bounds: Boundary) -> pd.DataFrame: """ @@ -118,9 +119,7 @@ def gen_circle(self, bounds: Boundary) -> pd.DataFrame: # Set centre as centre of data_grid if none specified if self.radius is None: raise ValueError("radius parameter is required for circle") - centre: tuple[float | None, float | None] = ( - self.centre if hasattr(self, "centre") else (None, None) # type: ignore[assignment] - ) + centre = self.centre if self.centre is not None else (None, None) c_y = self.lat[int(self.ny / 2)] if centre[0] is None else centre[0] c_x = self.long[int(self.nx / 2)] if centre[1] is None else centre[1] @@ -154,9 +153,7 @@ def gen_circle(self, bounds: Boundary) -> pd.DataFrame: # Change boolean values to int dummy_df["dummy_data"] = dummy_df["dummy_data"].astype(int) # Multiply by scaling factor if present - multiplier: float = ( - self.multiplier if hasattr(self, "multiplier") and self.multiplier is not None else 1.0 - ) + multiplier = self.multiplier if self.multiplier is not None else 1.0 dummy_df["dummy_data"] = dummy_df["dummy_data"] * multiplier return dummy_df @@ -276,9 +273,7 @@ def gen_rectangle(self, bounds: Boundary) -> pd.DataFrame: # Set centre as centre of data_grid if none specified if self.width is None or self.height is None: raise ValueError("width and height parameters are required for rectangle") - centre_rect: tuple[float | None, float | None] = ( - self.centre if hasattr(self, "centre") else (None, None) # type: ignore[assignment] - ) + centre_rect = self.centre if self.centre is not None else (None, None) c_y = self.lat[int(self.ny / 2)] if centre_rect[0] is None else centre_rect[0] c_x = self.long[int(self.nx / 2)] if centre_rect[1] is None else centre_rect[1] diff --git a/meshiphi/dataloaders/scalar/visual_iced.py b/meshiphi/dataloaders/scalar/visual_iced.py index 044364eb..98915d15 100644 --- a/meshiphi/dataloaders/scalar/visual_iced.py +++ b/meshiphi/dataloaders/scalar/visual_iced.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast import xarray as xr @@ -87,4 +87,4 @@ def import_from_tiff(self, xarray_dataset: xr.Dataset) -> xr.Dataset: # convert back to xarray vi_dataframe = vi_dataframe.set_index(["x", "y"]) - return vi_dataframe.to_xarray() # type: ignore[no-any-return] + return cast("xr.Dataset", vi_dataframe.to_xarray()) diff --git a/meshiphi/dataloaders/vector/vector_grf.py b/meshiphi/dataloaders/vector/vector_grf.py index 86d8518e..70dcaf79 100644 --- a/meshiphi/dataloaders/vector/vector_grf.py +++ b/meshiphi/dataloaders/vector/vector_grf.py @@ -1,15 +1,16 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import numpy as np import pandas as pd -import xarray as xr # noqa: TC002 from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader from meshiphi.utils import gaussian_random_field if TYPE_CHECKING: + import xarray as xr + from meshiphi.mesh_generation.boundary import Boundary @@ -123,4 +124,4 @@ def grf_to_vector( # Cast to dataframe data = pd.DataFrame(rows).set_index(["lat", "long"]) # Set to xarray dataset - return data.to_xarray() # type: ignore[no-any-return] + return cast("xr.Dataset", data.to_xarray()) diff --git a/meshiphi/dataloaders/vector/vector_shape.py b/meshiphi/dataloaders/vector/vector_shape.py index b5dbcea9..1e5682ff 100644 --- a/meshiphi/dataloaders/vector/vector_shape.py +++ b/meshiphi/dataloaders/vector/vector_shape.py @@ -1,15 +1,16 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import numpy as np import pandas as pd -import xarray as xr # noqa: TC002 from meshiphi.dataloaders.vector.abstract_vector import VectorDataLoader if TYPE_CHECKING: + import xarray as xr + from meshiphi.mesh_generation.boundary import Boundary logger = logging.getLogger(__name__) @@ -93,7 +94,7 @@ def import_data(self, bounds: Boundary) -> xr.Dataset: raise ValueError(f"Unknown vector shape type: {self.dataloader_name}") # No need to trim data, as was defined by bounds - return data.set_index(["lat", "long"]).to_xarray() # type: ignore[no-any-return] + return cast("xr.Dataset", data.set_index(["lat", "long"]).to_xarray()) def gen_gradient(self, bounds: Boundary) -> pd.DataFrame: """ @@ -180,9 +181,7 @@ def gen_circle(self, bounds: Boundary) -> pd.DataFrame: # Set centre as centre of data_grid if none specified if self.radius is None: raise ValueError("radius parameter is required for vector_circle") - centre: tuple[float | None, float | None] = ( - self.centre if hasattr(self, "centre") else (None, None) # type: ignore[assignment] - ) + centre = self.centre if self.centre is not None else (None, None) c_y = self.lat[int(self.ny / 2)] if centre[0] is None else centre[0] c_x = self.long[int(self.nx / 2)] if centre[1] is None else centre[1] @@ -250,9 +249,7 @@ def gen_rectangle(self, bounds: Boundary) -> pd.DataFrame: # Set centre as centre of data_grid if none specified if self.width is None or self.height is None: raise ValueError("width and height parameters are required for vector_rectangle") - centre_rect: tuple[float | None, float | None] = ( - self.centre if hasattr(self, "centre") else (None, None) # type: ignore[assignment] - ) + centre_rect = self.centre if self.centre is not None else (None, None) c_y = self.lat[int(self.ny / 2)] if centre_rect[0] is None else centre_rect[0] c_x = self.long[int(self.nx / 2)] if centre_rect[1] is None else centre_rect[1] diff --git a/meshiphi/mesh_generation/boundary.py b/meshiphi/mesh_generation/boundary.py index 085e4df9..848328b7 100644 --- a/meshiphi/mesh_generation/boundary.py +++ b/meshiphi/mesh_generation/boundary.py @@ -4,10 +4,10 @@ from math import asin, cos, radians, sin, sqrt from typing import TYPE_CHECKING, Any -from shapely import MultiPolygon, wkt +from shapely import wkt if TYPE_CHECKING: - from shapely.geometry import Polygon + from shapely.geometry import MultiPolygon, Polygon class Boundary: @@ -423,22 +423,23 @@ def to_polygon(self) -> Polygon | MultiPolygon: + f"{self.get_long_min()} {self.get_lat_min()}))" ) else: - # Create a multipolygon of boundary - polygon_1 = wkt.loads( - f"POLYGON(({self.get_long_min()} {self.get_lat_min()}," + # Create a multipolygon of boundary directly from WKT string + # This avoids numpy/shapely compatibility issues when constructing from separate polygons + polygon = wkt.loads( + "MULTIPOLYGON(((" + + f"{self.get_long_min()} {self.get_lat_min()}," + f"{self.get_long_min()} {self.get_lat_max()}," + f"180 {self.get_lat_max()}," + f"180 {self.get_lat_min()}," - + f"{self.get_long_min()} {self.get_lat_min()}))" - ) - polygon_2 = wkt.loads( - f"POLYGON((-180 {self.get_lat_min()}," + + f"{self.get_long_min()} {self.get_lat_min()}" + + ")),((" + + f"-180 {self.get_lat_min()}," + f"-180 {self.get_lat_max()}," + f"{self.get_long_max()} {self.get_lat_max()}," + f"{self.get_long_max()} {self.get_lat_min()}," - + f"-180 {self.get_lat_min()}))" + + f"-180 {self.get_lat_min()}" + + ")))" ) - polygon = MultiPolygon([polygon_1, polygon_2]) return polygon def to_poly_string(self) -> str: diff --git a/meshiphi/mesh_generation/environment_mesh.py b/meshiphi/mesh_generation/environment_mesh.py index 9ba8b3fa..9cdcc9a1 100644 --- a/meshiphi/mesh_generation/environment_mesh.py +++ b/meshiphi/mesh_generation/environment_mesh.py @@ -365,12 +365,19 @@ def tie_northern_cellbox_ng( if str(neighbour_case) == "1": north_east_neighbours.append(int(cellbox_l.get_id())) + # Ensure the cellbox exists in neighbour graph before appending + cellbox_id = cellbox_s.get_id() + if cellbox_id not in self.neighbour_graph.get_graph(): + self.neighbour_graph.add_node( + cellbox_id, {-4: [], -3: [], -2: [], -1: [], 1: [], 2: [], 3: [], 4: []} + ) + for neighbour in north_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-4"].append(neighbour) # + self.neighbour_graph.get_graph()[cellbox_id]["-4"].append(neighbour) for neighbour in north_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-3"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_id]["-3"].append(neighbour) for neighbour in north_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["1"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_id]["1"].append(neighbour) def tie_southern_cellbox_ng( self, @@ -407,12 +414,19 @@ def tie_southern_cellbox_ng( if str(neighbour_case) == "1": north_east_neighbours.append(int(cellbox_s.get_id())) + # Ensure the cellbox exists in neighbour graph before appending + cellbox_id = cellbox_l.get_id() + if cellbox_id not in self.neighbour_graph.get_graph(): + self.neighbour_graph.add_node( + cellbox_id, {-4: [], -3: [], -2: [], -1: [], 1: [], 2: [], 3: [], 4: []} + ) + for neighbour in north_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-4"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_id][-4].append(neighbour) for neighbour in north_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-3"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_id][-3].append(neighbour) for neighbour in north_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["1"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_id][1].append(neighbour) # Tie southern interior cellboxes to southern exterior cellboxes @@ -436,12 +450,19 @@ def tie_southern_cellbox_ng( if str(neighbour_case) == "3": south_east_neighbours.append(int(cellbox_l.get_id())) + # Ensure the cellbox exists in neighbour graph before appending + cellbox_id = cellbox_s.get_id() + if cellbox_id not in self.neighbour_graph.get_graph(): + self.neighbour_graph.add_node( + cellbox_id, {-4: [], -3: [], -2: [], -1: [], 1: [], 2: [], 3: [], 4: []} + ) + for neighbour in south_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["4"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_id][4].append(neighbour) for neighbour in south_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-1"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_id][-1].append(neighbour) for neighbour in south_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["3"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_id][3].append(neighbour) def tie_eastern_cellbox_ng( self, diff --git a/meshiphi/utils.py b/meshiphi/utils.py index a334893b..a1c79880 100644 --- a/meshiphi/utils.py +++ b/meshiphi/utils.py @@ -10,7 +10,7 @@ from calendar import monthrange from datetime import datetime, timedelta from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, Generator, TypeVar, cast, overload +from typing import TYPE_CHECKING, Any, Callable, Generator, ParamSpec, TypeVar, cast, overload import numpy as np import numpy.typing as npt @@ -26,6 +26,8 @@ logger = logging.getLogger(__name__) F = TypeVar("F", bound=Callable[..., Any]) +P = ParamSpec("P") +R = TypeVar("R") @overload @@ -172,11 +174,13 @@ def round_to_sigfig( raise ValueError(f"Cannot round {type(x)} to sig figs!") # Cast as array if not initially, so that later processes all act as expected + x_array: npt.NDArray[np.floating[Any]] if orig_type in [int, float, np.float64]: - x = [x] # type: ignore[list-item] - x = np.array(x) + x_array = np.array([float(x)], dtype=np.float64) # Explicit conversion to array + else: + x_array = np.array(x, dtype=np.float64) # Create a mask disabling any values of inf or zero being passed to log10 - loggable_idxs = ([x != 0] & np.isfinite(x))[0] + loggable_idxs = ([x_array != 0] & np.isfinite(x_array))[0] # Determine number of decimal places to round each number to # np.abs because can't find log of negative number # np.log10 to get position of most significant digit @@ -186,11 +190,13 @@ def round_to_sigfig( # np.array.astype(int) to enable np.around to work later dec_pl = ( sigfig - - np.floor(np.log10(np.abs(x), where=loggable_idxs, out=np.zeros_like(x))).astype(int) + - np.floor( + np.log10(np.abs(x_array), where=loggable_idxs, out=np.zeros_like(x_array)) + ).astype(int) - 1 ) # Round to sig figs - rounded = np.array([np.around(x[i], decimals=dec_pl[i]) for i in range(len(x))]) + rounded = np.array([np.around(x_array[i], decimals=dec_pl[i]) for i in range(len(x_array))]) # Return as single value if input that way if orig_type in [int, float]: return cast("int | float", rounded.item()) @@ -276,9 +282,9 @@ def gaussian_random_field(size: int, alpha: float) -> npt.NDArray[np.floating[An return grf / (np.max(grf) - np.min(grf)) -def memory_trace(func: F) -> F: +def memory_trace(func: Callable[P, R]) -> Callable[P, R]: @wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: tracemalloc.start(20) res = func(*args, **kwargs) snapshot = tracemalloc.take_snapshot() @@ -289,25 +295,25 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: logging.info("\n".join(stat.traceback.format())) return res - return wrapper # type: ignore[return-value] + return wrapper -def timed_call(func: F) -> F: +def timed_call(func: Callable[P, R]) -> Callable[P, R]: @wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: start = time.perf_counter() res = func(*args, **kwargs) end = time.perf_counter() logger.info(f"Timed call to {func.__name__} took {end - start:02f} seconds") return res - return wrapper # type: ignore[return-value] + return wrapper # CLI utilities def setup_logging( - func: F, log_format: str = "[%(asctime)-17s :%(levelname)-8s] - %(message)s" -) -> F: + func: Callable[P, R], log_format: str = "[%(asctime)-17s :%(levelname)-8s] - %(message)s" +) -> Callable[P, R]: """Wraps a CLI endpoint and sets up logging for it This is probably not the smoothest implementation, but it's an educational @@ -322,7 +328,7 @@ def setup_logging( """ @wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: parsed_args = func(*args, **kwargs) level = logging.INFO @@ -343,4 +349,4 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: logging.getLogger("urllib3").setLevel(logging.WARNING) return parsed_args - return wrapper # type: ignore[return-value] + return wrapper diff --git a/pyproject.toml b/pyproject.toml index 7158c8b0..99de1582 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ ] readme = "README.md" license = {file = "LICENSE"} -requires-python = ">= 3.8" +requires-python = ">= 3.10" classifiers = [ "Intended Audience :: Science/Research", "Development Status :: 4 - Beta", @@ -18,8 +18,6 @@ classifiers = [ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", From 309778aad6eb02aac075c85d9caf720650d611e0 Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Fri, 30 Jan 2026 11:41:16 +0000 Subject: [PATCH 12/15] don't run tests on draft PR --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 89f2d42b..169eee63 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,9 +5,11 @@ on: branches: [main] pull_request: branches: [main] + types: [opened, synchronize, reopened, ready_for_review] jobs: test: + if: github.event.pull_request.draft == false runs-on: ubuntu-latest strategy: matrix: From c6bdff31494ce4e0aa9eb6d75acd069f8620ecb2 Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Fri, 30 Jan 2026 11:44:27 +0000 Subject: [PATCH 13/15] add py.typed indicating this is a typed package --- meshiphi/py.typed | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 meshiphi/py.typed diff --git a/meshiphi/py.typed b/meshiphi/py.typed new file mode 100644 index 00000000..e69de29b From 23da04c4487ebe5f72fd85a4e5e0dd20e566abe4 Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Tue, 3 Feb 2026 11:06:37 +0000 Subject: [PATCH 14/15] cellbox get id returns int, classes that depend on it accept int --- .../mesh_generation/aggregated_cellbox.py | 14 +- meshiphi/mesh_generation/cellbox.py | 4 +- meshiphi/mesh_generation/environment_mesh.py | 174 +++++++++--------- meshiphi/mesh_generation/mesh_builder.py | 24 +-- meshiphi/utils.py | 2 +- pyproject.toml | 7 + tests/conftest.py | 8 +- tests/unit_tests/test_aggregated_cellbox.py | 6 +- 8 files changed, 123 insertions(+), 116 deletions(-) diff --git a/meshiphi/mesh_generation/aggregated_cellbox.py b/meshiphi/mesh_generation/aggregated_cellbox.py index 3874e2dc..b77ca820 100644 --- a/meshiphi/mesh_generation/aggregated_cellbox.py +++ b/meshiphi/mesh_generation/aggregated_cellbox.py @@ -28,7 +28,7 @@ def from_json(cls, cellbox_json: dict[str, Any]) -> AggregatedCellBox: Args: cellbox_json(Json): json object that encapsulates boundary, agg_data and id of the CellBox """ - cellbox_id = cellbox_json["id"] + cellbox_id = int(cellbox_json["id"]) def load_bounds(cellbox_json: dict[str, Any]) -> Boundary: shape = shapely.wkt.loads(cellbox_json["geometry"]) @@ -71,13 +71,13 @@ def load_agg_data(cellbox_json: dict[str, Any]) -> dict[str, Any]: agg_data = load_agg_data(cellbox_json) return AggregatedCellBox(boundary, agg_data, cellbox_id) - def __init__(self, boundary: Boundary, agg_data: dict[str, Any], id: str) -> None: + def __init__(self, boundary: Boundary, agg_data: dict[str, Any], id: int) -> None: """ Args: boundary(Boundary): encapsulates latitude, longtitude and time range of the CellBox agg_data (dict): a dictionary that contains data_names and agg values - id (string): a string represents cellbox id + id (int): an integer representing the cellbox id """ # Box information relative to bottom left self.boundary = boundary @@ -97,7 +97,7 @@ def set_agg_data(self, agg_data: dict[str, Any]) -> None: """ self.agg_data = agg_data - def set_id(self, id: str) -> None: + def set_id(self, id: int) -> None: """ sets the cellbox id """ @@ -115,11 +115,11 @@ def get_agg_data(self) -> dict[str, Any]: """ return self.agg_data - def get_id(self) -> str: + def get_id(self) -> int: """ - returns the id + returns the id as an integer """ - return self.id + return int(self.id) def to_json(self) -> dict[str, Any]: """ diff --git a/meshiphi/mesh_generation/cellbox.py b/meshiphi/mesh_generation/cellbox.py index acd40fa4..ac4802cd 100644 --- a/meshiphi/mesh_generation/cellbox.py +++ b/meshiphi/mesh_generation/cellbox.py @@ -110,11 +110,11 @@ def set_bounds(self, bounds: Boundary) -> None: """ self.bounds = bounds - def get_id(self) -> Any: + def get_id(self) -> int: """ method returns cellbox cell id """ - return self.id + return int(self.id) def get_minimum_datapoints(self) -> int: """ diff --git a/meshiphi/mesh_generation/environment_mesh.py b/meshiphi/mesh_generation/environment_mesh.py index 9cdcc9a1..758c0946 100644 --- a/meshiphi/mesh_generation/environment_mesh.py +++ b/meshiphi/mesh_generation/environment_mesh.py @@ -210,7 +210,7 @@ def get_cellbox(self, cellbox_id: str | int) -> AggregatedCellBox: AggregatedCellBox: the cellbox with the given id """ for cellbox in self.agg_cellboxes: - if str(cellbox.get_id()) == str(cellbox_id): + if cellbox.get_id() == int(cellbox_id): return cellbox raise ValueError(f"Cellbox with id {cellbox_id} not found") @@ -329,19 +329,19 @@ def tie_northern_cellbox_ng( cb_l_bounds, cb_s_bounds ) - if str(neighbour_case) == "4": - south_neighbours.append(int(cellbox_s.get_id())) - if str(neighbour_case) == "-1": - south_west_neighbours.append(int(cellbox_s.get_id())) - if str(neighbour_case) == "3": - south_east_neighbours.append(int(cellbox_s.get_id())) + if neighbour_case == 4: + south_neighbours.append(cellbox_s.get_id()) + if neighbour_case == -1: + south_west_neighbours.append(cellbox_s.get_id()) + if neighbour_case == 3: + south_east_neighbours.append(cellbox_s.get_id()) for neighbour in south_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["4"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_l.get_id()][4].append(neighbour) for neighbour in south_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-1"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_l.get_id()][-1].append(neighbour) for neighbour in south_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["3"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_l.get_id()][3].append(neighbour) # Tie northen interior cellboxes to northen exterior cellboxes @@ -358,12 +358,12 @@ def tie_northern_cellbox_ng( cb_s_bounds, cb_l_bounds ) - if str(neighbour_case) == "-4": - north_neighbours.append(int(cellbox_l.get_id())) - if str(neighbour_case) == "-3": - north_west_neighbours.append(int(cellbox_l.get_id())) - if str(neighbour_case) == "1": - north_east_neighbours.append(int(cellbox_l.get_id())) + if neighbour_case == -4: + north_neighbours.append(cellbox_l.get_id()) + if neighbour_case == -3: + north_west_neighbours.append(cellbox_l.get_id()) + if neighbour_case == 1: + north_east_neighbours.append(cellbox_l.get_id()) # Ensure the cellbox exists in neighbour graph before appending cellbox_id = cellbox_s.get_id() @@ -373,11 +373,11 @@ def tie_northern_cellbox_ng( ) for neighbour in north_neighbours: - self.neighbour_graph.get_graph()[cellbox_id]["-4"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_id][-4].append(neighbour) for neighbour in north_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_id]["-3"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_id][-3].append(neighbour) for neighbour in north_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_id]["1"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_id][1].append(neighbour) def tie_southern_cellbox_ng( self, @@ -407,12 +407,12 @@ def tie_southern_cellbox_ng( cb_l_bounds, cb_s_bounds ) - if str(neighbour_case) == "-4": - north_neighbours.append(int(cellbox_s.get_id())) - if str(neighbour_case) == "-3": - north_west_neighbours.append(int(cellbox_s.get_id())) - if str(neighbour_case) == "1": - north_east_neighbours.append(int(cellbox_s.get_id())) + if neighbour_case == -4: + north_neighbours.append(cellbox_s.get_id()) + if neighbour_case == -3: + north_west_neighbours.append(cellbox_s.get_id()) + if neighbour_case == 1: + north_east_neighbours.append(cellbox_s.get_id()) # Ensure the cellbox exists in neighbour graph before appending cellbox_id = cellbox_l.get_id() @@ -443,12 +443,12 @@ def tie_southern_cellbox_ng( cb_s_bounds, cb_l_bounds ) - if str(neighbour_case) == "4": - south_neighbours.append(int(cellbox_l.get_id())) - if str(neighbour_case) == "-1": - south_west_neighbours.append(int(cellbox_l.get_id())) - if str(neighbour_case) == "3": - south_east_neighbours.append(int(cellbox_l.get_id())) + if neighbour_case == 4: + south_neighbours.append(cellbox_l.get_id()) + if neighbour_case == -1: + south_west_neighbours.append(cellbox_l.get_id()) + if neighbour_case == 3: + south_east_neighbours.append(cellbox_l.get_id()) # Ensure the cellbox exists in neighbour graph before appending cellbox_id = cellbox_s.get_id() @@ -492,19 +492,19 @@ def tie_eastern_cellbox_ng( cb_l_bounds, cb_s_bounds ) - if str(neighbour_case) == "-2": - west_neighbours.append(int(cellbox_s.get_id())) - if str(neighbour_case) == "-1": - south_west_neighbours.append(int(cellbox_s.get_id())) - if str(neighbour_case) == "-3": - north_west_neighbours.append(int(cellbox_s.get_id())) + if neighbour_case == -2: + west_neighbours.append(cellbox_s.get_id()) + if neighbour_case == -1: + south_west_neighbours.append(cellbox_s.get_id()) + if neighbour_case == -3: + north_west_neighbours.append(cellbox_s.get_id()) for neighbour in west_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-2"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_l.get_id()][-2].append(neighbour) for neighbour in south_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-1"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_l.get_id()][-1].append(neighbour) for neighbour in north_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["-3"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_l.get_id()][-3].append(neighbour) # Tie eastern interior cellboxes to eastern exterior cellboxes @@ -521,19 +521,19 @@ def tie_eastern_cellbox_ng( cb_s_bounds, cb_l_bounds ) - if str(neighbour_case) == "2": - east_neighbours.append(int(cellbox_l.get_id())) - if str(neighbour_case) == "3": - south_east_neighbours.append(int(cellbox_l.get_id())) - if str(neighbour_case) == "1": - north_east_neighbours.append(int(cellbox_l.get_id())) + if neighbour_case == 2: + east_neighbours.append(cellbox_l.get_id()) + if neighbour_case == 3: + south_east_neighbours.append(cellbox_l.get_id()) + if neighbour_case == 1: + north_east_neighbours.append(cellbox_l.get_id()) for neighbour in east_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["2"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_s.get_id()][2].append(neighbour) for neighbour in south_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["3"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_s.get_id()][3].append(neighbour) for neighbour in north_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["1"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_s.get_id()][1].append(neighbour) def tie_western_cellbox_ng( self, @@ -563,19 +563,19 @@ def tie_western_cellbox_ng( cb_l_bounds, cb_s_bounds ) - if str(neighbour_case) == "2": - east_neighbours.append(int(cellbox_s.get_id())) - if str(neighbour_case) == "3": - south_east_neighbours.append(int(cellbox_s.get_id())) - if str(neighbour_case) == "1": - north_east_neighbours.append(int(cellbox_s.get_id())) + if neighbour_case == 2: + east_neighbours.append(cellbox_s.get_id()) + if neighbour_case == 3: + south_east_neighbours.append(cellbox_s.get_id()) + if neighbour_case == 1: + north_east_neighbours.append(cellbox_s.get_id()) for neighbour in east_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["2"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_l.get_id()][2].append(neighbour) for neighbour in south_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["3"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_l.get_id()][3].append(neighbour) for neighbour in north_east_neighbours: - self.neighbour_graph.get_graph()[cellbox_l.get_id()]["1"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_l.get_id()][1].append(neighbour) # Tie western interior cellboxes to western exterior cellboxes @@ -592,19 +592,19 @@ def tie_western_cellbox_ng( cb_s_bounds, cb_l_bounds ) - if str(neighbour_case) == "-2": - west_neighbours.append(int(cellbox_l.get_id())) - if str(neighbour_case) == "-1": - south_west_neighbours.append(int(cellbox_l.get_id())) - if str(neighbour_case) == "-3": - north_west_neighbours.append(int(cellbox_l.get_id())) + if neighbour_case == -2: + west_neighbours.append(cellbox_l.get_id()) + if neighbour_case == -1: + south_west_neighbours.append(cellbox_l.get_id()) + if neighbour_case == -3: + north_west_neighbours.append(cellbox_l.get_id()) for neighbour in west_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-2"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_s.get_id()][-2].append(neighbour) for neighbour in south_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-1"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_s.get_id()][-1].append(neighbour) for neighbour in north_west_neighbours: - self.neighbour_graph.get_graph()[cellbox_s.get_id()]["-3"].append(neighbour) + self.neighbour_graph.get_graph()[cellbox_s.get_id()][-3].append(neighbour) def get_cellboxes_within_bounds(self, bounds: Boundary) -> list[AggregatedCellBox]: """ @@ -763,8 +763,8 @@ def get_max_cellbox_id(self) -> int: """ max_id = 0 for cellbox in self.agg_cellboxes: - if int(cellbox.get_id()) > max_id: - max_id = int(cellbox.get_id()) + if cellbox.get_id() > max_id: + max_id = cellbox.get_id() return max_id def remove_cellbox(self, cellbox: AggregatedCellBox) -> None: @@ -786,18 +786,18 @@ def add_cellbox(self, cellbox: AggregatedCellBox) -> None: def increment_ids(self, increment: int) -> None: for cellbox in self.agg_cellboxes: - cellbox.set_id(str(int(cellbox.get_id()) + increment)) - cellbox.agg_data["id"] = str(int(cellbox.get_id()) + increment) + cellbox.set_id(cellbox.get_id() + increment) + cellbox.agg_data["id"] = cellbox.get_id() self.neighbour_graph.increment_ids(increment) # Splitting - def sim_split_cellbox(self, cellbox_id: str) -> list[AggregatedCellBox]: + def sim_split_cellbox(self, cellbox_id: str | int) -> list[AggregatedCellBox]: """ splits the cellbox with the given id Args: - cellbox_id (string): the id of the cellbox to be split + cellbox_id (str | int): the id of the cellbox to be split Returns: split_cells (list): a list of the new cellboxes created by the split """ @@ -808,30 +808,30 @@ def sim_split_cellbox(self, cellbox_id: str) -> list[AggregatedCellBox]: # Copy agg_data, update id's agg_data = cellbox.get_agg_data() agg_data1 = agg_data.copy() - agg_data1["id"] = str(max_id + 1) + agg_data1["id"] = max_id + 1 agg_data2 = agg_data.copy() - agg_data2["id"] = str(max_id + 2) + agg_data2["id"] = max_id + 2 agg_data3 = agg_data.copy() - agg_data3["id"] = str(max_id + 3) + agg_data3["id"] = max_id + 3 agg_data4 = agg_data.copy() - agg_data4["id"] = str(max_id + 4) + agg_data4["id"] = max_id + 4 bounds = cellbox.get_bounds() split_bounds = bounds.split() - cellbox1 = AggregatedCellBox(split_bounds[0], agg_data1, str(max_id + 1)) - cellbox2 = AggregatedCellBox(split_bounds[1], agg_data2, str(max_id + 2)) - cellbox3 = AggregatedCellBox(split_bounds[2], agg_data3, str(max_id + 3)) - cellbox4 = AggregatedCellBox(split_bounds[3], agg_data4, str(max_id + 4)) + cellbox1 = AggregatedCellBox(split_bounds[0], agg_data1, max_id + 1) + cellbox2 = AggregatedCellBox(split_bounds[1], agg_data2, max_id + 2) + cellbox3 = AggregatedCellBox(split_bounds[2], agg_data3, max_id + 3) + cellbox4 = AggregatedCellBox(split_bounds[3], agg_data4, max_id + 4) return [cellbox1, cellbox2, cellbox3, cellbox4] - def split_and_replace(self, cellbox_id: str) -> None: + def split_and_replace(self, cellbox_id: str | int) -> None: """ splits the cellbox with the given id and replaces it with the new cellboxes Args: - cellbox_id (string): the id of the cellbox to be split + cellbox_id (str | int): the id of the cellbox to be split """ # Create new cellboxes and append to agg_cellboxes @@ -840,10 +840,10 @@ def split_and_replace(self, cellbox_id: str) -> None: self.agg_cellboxes.append(cellbox) # get ID's of new cellboxes - north_west_index = int(split_cellboxes[1].get_agg_data()["id"]) - north_east_index = int(split_cellboxes[3].get_agg_data()["id"]) - south_west_index = int(split_cellboxes[0].get_agg_data()["id"]) - south_east_index = int(split_cellboxes[2].get_agg_data()["id"]) + north_west_index = split_cellboxes[1].get_agg_data()["id"] + north_east_index = split_cellboxes[3].get_agg_data()["id"] + south_west_index = split_cellboxes[0].get_agg_data()["id"] + south_east_index = split_cellboxes[2].get_agg_data()["id"] # Get neighbours of original cellbox cellbox_id_int = int(cellbox_id) diff --git a/meshiphi/mesh_generation/mesh_builder.py b/meshiphi/mesh_generation/mesh_builder.py index 507250b6..c646dcdf 100644 --- a/meshiphi/mesh_generation/mesh_builder.py +++ b/meshiphi/mesh_generation/mesh_builder.py @@ -367,37 +367,37 @@ def check_global_mesh( # update NG to connect cellboxes for i in range(len(min_long_cellboxes)): self.neighbour_graph.add_neighbour( - int(min_long_cellboxes[i].get_id()), + min_long_cellboxes[i].get_id(), Direction.west, - int(max_long_cellboxes[i].get_id()), + max_long_cellboxes[i].get_id(), ) self.neighbour_graph.add_neighbour( - int(max_long_cellboxes[i].get_id()), + max_long_cellboxes[i].get_id(), Direction.east, - int(min_long_cellboxes[i].get_id()), + min_long_cellboxes[i].get_id(), ) # checks to avoid the very upper and lower cellboxes as they do not have north/south neighbours if 0 <= i < len(min_long_cellboxes) - 1: self.neighbour_graph.add_neighbour( - int(min_long_cellboxes[i].get_id()), + min_long_cellboxes[i].get_id(), Direction.north_west, - int(max_long_cellboxes[i + 1].get_id()), + max_long_cellboxes[i + 1].get_id(), ) self.neighbour_graph.add_neighbour( - int(max_long_cellboxes[i].get_id()), + max_long_cellboxes[i].get_id(), Direction.north_east, - int(min_long_cellboxes[i + 1].get_id()), + min_long_cellboxes[i + 1].get_id(), ) if 0 < i <= len(min_long_cellboxes) - 1: self.neighbour_graph.add_neighbour( - int(min_long_cellboxes[i].get_id()), + min_long_cellboxes[i].get_id(), Direction.south_west, - int(max_long_cellboxes[i - 1].get_id()), + max_long_cellboxes[i - 1].get_id(), ) self.neighbour_graph.add_neighbour( - int(max_long_cellboxes[i].get_id()), + max_long_cellboxes[i].get_id(), Direction.south_east, - int(min_long_cellboxes[i - 1].get_id()), + min_long_cellboxes[i - 1].get_id(), ) return is_global_mesh diff --git a/meshiphi/utils.py b/meshiphi/utils.py index a1c79880..307fb085 100644 --- a/meshiphi/utils.py +++ b/meshiphi/utils.py @@ -175,7 +175,7 @@ def round_to_sigfig( # Cast as array if not initially, so that later processes all act as expected x_array: npt.NDArray[np.floating[Any]] - if orig_type in [int, float, np.float64]: + if isinstance(x, (int, float, np.floating)): x_array = np.array([float(x)], dtype=np.float64) # Explicit conversion to array else: x_array = np.array(x, dtype=np.float64) diff --git a/pyproject.toml b/pyproject.toml index afbf5a38..9c678b04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -171,6 +171,7 @@ line-ending = "auto" [tool.mypy] python_version = "3.13" +strict = true warn_return_any = true warn_unused_configs = true disallow_untyped_defs = true @@ -184,6 +185,12 @@ strict_equality = true show_error_codes = true follow_imports = "normal" +# Exclude tests from strict checking +[[tool.mypy.overrides]] +module = ["tests.*"] +strict = false +disallow_untyped_defs = false + # Ignore missing imports and untyped libraries for third-party libraries without type stubs [[tool.mypy.overrides]] module = [ diff --git a/tests/conftest.py b/tests/conftest.py index e7c4eee6..e3aac08a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -290,28 +290,28 @@ def dummy_cellbox(arbitrary_boundary): def dummy_agg_cellbox(): """Basic aggregated cellbox for testing.""" arbitrary_agg_data = {"dummy_data": 1} - return AggregatedCellBox(Boundary([45, 60], [45, 60]), arbitrary_agg_data, "0") + return AggregatedCellBox(Boundary([45, 60], [45, 60]), arbitrary_agg_data, 0) @pytest.fixture def arbitrary_agg_cellbox(): """Arbitrary aggregated cellbox for testing.""" arbitrary_agg_data = {"dummy_data": 1} - return AggregatedCellBox(Boundary([45, 60], [45, 60]), arbitrary_agg_data, "1") + return AggregatedCellBox(Boundary([45, 60], [45, 60]), arbitrary_agg_data, 1) @pytest.fixture def equatorial_agg_cellbox(): """Aggregated cellbox crossing equator.""" arbitrary_agg_data = {"dummy_data": 1} - return AggregatedCellBox(Boundary([-10, 10], [45, 60]), arbitrary_agg_data, "2") + return AggregatedCellBox(Boundary([-10, 10], [45, 60]), arbitrary_agg_data, 2) @pytest.fixture def meridian_agg_cellbox(): """Aggregated cellbox crossing prime meridian.""" arbitrary_agg_data = {"dummy_data": 1} - return AggregatedCellBox(Boundary([45, 60], [-10, 10]), arbitrary_agg_data, "3") + return AggregatedCellBox(Boundary([45, 60], [-10, 10]), arbitrary_agg_data, 3) @pytest.fixture diff --git a/tests/unit_tests/test_aggregated_cellbox.py b/tests/unit_tests/test_aggregated_cellbox.py index f73ebcd2..2e92fb38 100644 --- a/tests/unit_tests/test_aggregated_cellbox.py +++ b/tests/unit_tests/test_aggregated_cellbox.py @@ -20,7 +20,7 @@ def test_from_json(): "SIC": 0.0, "thickness": 0.82, "density": 900.0, - "id": "1", + "id": 1, } agg_cb_from_json = AggregatedCellBox.from_json(agg_cb_json) @@ -39,7 +39,7 @@ def test_from_json(): ("attr", "test_value", "getter", "setter"), [ ("boundary", Boundary([10, 20], [30, 40]), "get_bounds", "set_bounds"), - ("id", "123", "get_id", "set_id"), + ("id", 123, "get_id", "set_id"), ("agg_data", {"dummy_data": "456"}, "get_agg_data", "set_agg_data"), ], ) @@ -66,7 +66,7 @@ def test_to_json(): "SIC": 0.0, "thickness": 0.82, "density": 900.0, - "id": "1", + "id": 1, } agg_cb_boundary = Boundary.from_poly_string(agg_cb_json["geometry"]) From fee9fa9f4a9ba19c84b5e9ac31b2007e6cc46489 Mon Sep 17 00:00:00 2001 From: thomaszwagerman Date: Tue, 3 Feb 2026 11:16:26 +0000 Subject: [PATCH 15/15] add ignore to optional osgeo import --- meshiphi/mesh_generation/environment_mesh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshiphi/mesh_generation/environment_mesh.py b/meshiphi/mesh_generation/environment_mesh.py index 758c0946..7c09b0ce 100644 --- a/meshiphi/mesh_generation/environment_mesh.py +++ b/meshiphi/mesh_generation/environment_mesh.py @@ -1372,7 +1372,7 @@ def to_tif(self, params_file: str | None, path: str) -> None: """ # Only import if we need GDAL, to avoid having it as a requirement try: - from osgeo import gdal, osr + from osgeo import gdal, osr # type: ignore except ImportError as err: raise ImportError( "GDAL is required for tif export. Install with: conda install -c conda-forge gdal"