From 030185a96d9f6d99fc938ea31f23138695529be4 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Mon, 9 Sep 2024 16:30:19 +0200 Subject: [PATCH 01/28] Moved py wrappers from main.py to db_operations.py --- python/roadgraphtool/db_operations.py | 93 +++++++++++++++++++ python/scripts/main.py | 124 ++++---------------------- 2 files changed, 108 insertions(+), 109 deletions(-) create mode 100644 python/roadgraphtool/db_operations.py diff --git a/python/roadgraphtool/db_operations.py b/python/roadgraphtool/db_operations.py new file mode 100644 index 0000000..ac1215d --- /dev/null +++ b/python/roadgraphtool/db_operations.py @@ -0,0 +1,93 @@ +from roadgraphtool.db import db + +from .insert_area import insert_area + + +def get_area_for_demand( + srid_plain: int, + dataset_ids: list, + zone_types: list, + buffer_meters: int, + min_requests_in_zone: int, + datetime_min: str, + datetime_max: str, + center_point: tuple, + max_distance_from_center_point_meters: int, +) -> list: + sql_query = """ + select * from get_area_for_demand( + srid_plain := :srid_plain, + dataset_ids := (:dataset_ids)::smallint[], + zone_types := (:zone_types)::smallint[], + buffer_meters := (:buffer_meters)::smallint, + min_requests_in_zone := (:min_requests_in_zone)::smallint, + datetime_min := :datetime_min, + datetime_max := :datetime_max, + center_point := st_makepoint(:center_x, :center_y), + max_distance_from_center_point_meters := (:max_distance_from_center_point_meters)::smallint + );""" + params = { + "srid_plain": srid_plain, + "dataset_ids": dataset_ids, + "zone_types": zone_types, + "buffer_meters": buffer_meters, + "min_requests_in_zone": min_requests_in_zone, + "datetime_min": datetime_min, + "datetime_max": datetime_max, + "center_x": center_point[0], + "center_y": center_point[1], + "max_distance_from_center_point_meters": max_distance_from_center_point_meters, + } + return db.execute_sql_and_fetch_all_rows(sql_query, params) + + +def contract_graph_in_area( + target_area_id: int, target_area_srid: int, fill_speed: bool = True +): + sql_query = f'call public.contract_graph_in_area({target_area_id}::smallint, {target_area_srid}::int{", FALSE" if not fill_speed else ""})' + db.execute_sql(sql_query) + + +def select_network_nodes_in_area(target_area_id: int) -> list: + sql_query = ( + f"select * from select_network_nodes_in_area({target_area_id}::smallint)" + ) + return db.execute_sql_and_fetch_all_rows(sql_query) + + +def assign_average_speed_to_all_segments_in_area( + target_area_id: int, target_area_srid: int +): + sql_query = ( + f"call public.assign_average_speed_to_all_segments_in_area({target_area_id}::smallint, " + f"{target_area_srid}::int)" + ) + db.execute_sql(sql_query) + + +def compute_strong_components(target_area_id: int): + sql_query = f"call public.compute_strong_components({target_area_id}::smallint)" + db.execute_sql(sql_query) + + +def compute_speeds_for_segments( + target_area_id: int, + speed_records_dataset: int, + hour: int, + day_of_week: int, +): + sql_query = ( + f"call public.compute_speeds_for_segments({target_area_id}::smallint, " + f"{speed_records_dataset}::smallint, {hour}::smallint, {day_of_week}::smallint)" + ) + db.execute_sql(sql_query) + + +def compute_speeds_from_neighborhood_segments( + target_area_id: int, target_area_srid: int +): + sql_query = ( + f"call public.compute_speeds_from_neighborhood_segments({target_area_id}::smallint, " + f"{target_area_srid}::int)" + ) + db.execute_sql(sql_query) diff --git a/python/scripts/main.py b/python/scripts/main.py index a5f239c..0b872b1 100644 --- a/python/scripts/main.py +++ b/python/scripts/main.py @@ -1,111 +1,17 @@ import argparse -import argparse -import json import logging + import psycopg2.errors -from roadgraphtool.credentials_config import CREDENTIALS -from roadgraphtool.db import db -from scripts.process_osm import import_osm_to_db +from roadgraphtool.db_operations import ( + assign_average_speed_to_all_segments_in_area, compute_speeds_for_segments, + compute_speeds_from_neighborhood_segments, compute_strong_components, + contract_graph_in_area, get_area_for_demand, insert_area, + select_network_nodes_in_area) from roadgraphtool.export import get_map_nodes_from_db -from roadgraphtool.db import db from scripts.process_osm import import_osm_to_db -def get_area_for_demand( - srid_plain: int, - dataset_ids: list, - zone_types: list, - buffer_meters: int, - min_requests_in_zone: int, - datetime_min: str, - datetime_max: str, - center_point: tuple, - max_distance_from_center_point_meters: int, -) -> list: - sql_query = """ - select * from get_area_for_demand( - srid_plain := :srid_plain, - dataset_ids := (:dataset_ids)::smallint[], - zone_types := (:zone_types)::smallint[], - buffer_meters := (:buffer_meters)::smallint, - min_requests_in_zone := (:min_requests_in_zone)::smallint, - datetime_min := :datetime_min, - datetime_max := :datetime_max, - center_point := st_makepoint(:center_x, :center_y), - max_distance_from_center_point_meters := (:max_distance_from_center_point_meters)::smallint - );""" - params = { - "srid_plain": srid_plain, - "dataset_ids": dataset_ids, - "zone_types": zone_types, - "buffer_meters": buffer_meters, - "min_requests_in_zone": min_requests_in_zone, - "datetime_min": datetime_min, - "datetime_max": datetime_max, - "center_x": center_point[0], - "center_y": center_point[1], - "max_distance_from_center_point_meters": max_distance_from_center_point_meters, - } - return db.execute_sql_and_fetch_all_rows(sql_query, params) - - -def insert_area(name: str, coordinates: list): - geom_json = {"type": "MultiPolygon", "coordinates": coordinates} - params = {"name": name, "json_data": json.dumps(geom_json)} - sql_query = """insert into areas (name, geom) values (:name, st_geomfromgeojson(:json_data))""" - db.execute_sql(sql_query, params) - - -def contract_graph_in_area( - target_area_id: int, target_area_srid: int, fill_speed: bool = True -): - sql_query = f'call public.contract_graph_in_area({target_area_id}::smallint, {target_area_srid}::int{", FALSE" if not fill_speed else ""})' - db.execute_sql(sql_query) - - -def select_network_nodes_in_area(target_area_id: int) -> list: - sql_query = ( - f"select * from select_network_nodes_in_area({target_area_id}::smallint)" - ) - return db.execute_sql_and_fetch_all_rows(sql_query) - - -def assign_average_speed_to_all_segments_in_area( - target_area_id: int, target_area_srid: int -): - sql_query = ( - f"call public.assign_average_speed_to_all_segments_in_area({target_area_id}::smallint, " - f"{target_area_srid}::int)" - ) - db.execute_sql(sql_query) - - -def compute_strong_components(target_area_id: int): - sql_query = f"call public.compute_strong_components({target_area_id}::smallint)" - db.execute_sql(sql_query) - - -def compute_speeds_for_segments( - target_area_id: int, speed_records_dataset: int, hour: int, day_of_week: int -): - sql_query = ( - f"call public.compute_speeds_for_segments({target_area_id}::smallint, " - f"{speed_records_dataset}::smallint, {hour}::smallint, {day_of_week}::smallint)" - ) - db.execute_sql(sql_query) - - -def compute_speeds_from_neighborhood_segments( - target_area_id: int, target_area_srid: int -): - sql_query = ( - f"call public.compute_speeds_from_neighborhood_segments({target_area_id}::smallint, " - f"{target_area_srid}::int)" - ) - db.execute_sql(sql_query) - - def configure_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="File containing the main flow of your application" @@ -131,11 +37,11 @@ def configure_arg_parser() -> argparse.ArgumentParser: required=False, ) parser.add_argument( - '-i', - '--import', - dest='importing', - action="store_true", - help='Import OSM data to database specified in config.ini' + "-i", + "--import", + dest="importing", + action="store_true", + help="Import OSM data to database specified in config.ini", ) return parser @@ -147,7 +53,7 @@ def main(arg_list: list[str] | None = None): if args.importing: import_osm_to_db() - + area_id = args.area_id area_srid = args.area_srid fill_speed = args.fill_speed @@ -164,7 +70,7 @@ def main(arg_list: list[str] | None = None): compute_strong_components(area_id) logging.info("storing the results in the component_data table") - insert_area("test1", []) + # insert_area("test1", []) area = get_area_for_demand( 4326, @@ -195,5 +101,5 @@ def main(arg_list: list[str] | None = None): compute_speeds_from_neighborhood_segments(area_id, area_srid) -if __name__ == '__main__': - main() \ No newline at end of file +if __name__ == "__main__": + main() From 8b293a51c1c16f70e3e9ce0f15990fdd82690286 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Mon, 9 Sep 2024 16:45:15 +0200 Subject: [PATCH 02/28] Resolved rebase conflict of 030185a9 --- python/scripts/filter_osm.py | 136 +++++++++++++++++++++------- python/scripts/main.py | 23 ++++- python/scripts/process_osm.py | 162 +++++++++++++++++++++++++--------- 3 files changed, 245 insertions(+), 76 deletions(-) diff --git a/python/scripts/filter_osm.py b/python/scripts/filter_osm.py index a1f8272..5919215 100644 --- a/python/scripts/filter_osm.py +++ b/python/scripts/filter_osm.py @@ -1,34 +1,43 @@ import argparse -import re import os +import re import subprocess import tempfile + import requests + class InvalidInputError(Exception): pass + class MissingInputError(Exception): pass + def is_valid_extension(file: str): """Function to check if the file has a valid extension.""" valid_extensions = ["osm", "osm.pbf", "osm.bz2"] return any(file.endswith(f".{ext}") for ext in valid_extensions) + def check_strategy(strategy: str | None): """Function to check strategy type""" valid_strategies = ["simple", "complete_ways", "smart"] if strategy and strategy not in valid_strategies: - raise InvalidInputError(f"Invalid strategy type. Call {os.path.basename(__file__)} -h/--help to display help.") + raise InvalidInputError( + f"Invalid strategy type. Call {os.path.basename(__file__)} -h/--help to display help." + ) + -def load_multipolygon_by_id(relation_id: str): +def load_multipolygon_by_id(relation_id: str | int): """Function to load multigon content by relation ID.""" url = f"https://www.openstreetmap.org/api/0.6/relation/{relation_id}/full" response = requests.get(url) response.raise_for_status() return response.content + def extract_id(input_file: str, relation_id: str, strategy: str = None): """Function to filter out data based on relation ID.""" content = load_multipolygon_by_id(relation_id) @@ -36,18 +45,35 @@ def extract_id(input_file: str, relation_id: str, strategy: str = None): with tempfile.NamedTemporaryFile(delete=False, suffix=".osm") as tmp_file: tmp_file.write(content) tmp_file_path = tmp_file.name - command = ["osmium", "extract", "-p", tmp_file_path, input_file, "-o", "resources/id_extract.osm"] + command = [ + "osmium", + "extract", + "-p", + tmp_file_path, + input_file, + "-o", + "resources/id_extract.osm", + ] if strategy: command.extend(["-s", strategy]) subprocess.run(command) + def extract_bbox(input_file: str, coords: str, strategy: str = None): """Function to extract based on bounding box with osmium""" # should match four floats: - float_regex = r'[0-9]+(.[0-9]+)?' - coords_regex = f'{float_regex},{float_regex},{float_regex},{float_regex}' + float_regex = r"[0-9]+(.[0-9]+)?" + coords_regex = f"{float_regex},{float_regex},{float_regex},{float_regex}" if re.match(coords_regex, coords): - command = ["osmium", "extract", "-b", coords, input_file, "-o", "extracted-bbox.osm.pbf"] + command = [ + "osmium", + "extract", + "-b", + coords, + input_file, + "-o", + "extracted-bbox.osm.pbf", + ] elif os.path.isfile(coords) and coords.endswith((".json", ".geojson")): command = ["osmium", "extract", "-c", coords, input_file] else: @@ -55,64 +81,105 @@ def extract_bbox(input_file: str, coords: str, strategy: str = None): if strategy: command.extend(["-s", strategy]) - + subprocess.run(command) + def run_osmium_filter(input_file: str, expression_file: str, omit_referenced: bool): """Function to filter objects based on tags in expression file. - Nodes referenced in ways and members referenced in relations will not + Nodes referenced in ways and members referenced in relations will not be added to output if omit_referenced set to True. """ - cmd = ["osmium", "tags-filter", input_file, "-e", expression_file, "-o", "filtered.osm.pbf"] + cmd = [ + "osmium", + "tags-filter", + input_file, + "-e", + expression_file, + "-o", + "filtered.osm.pbf", + ] if omit_referenced: cmd.extend(["-R"]) subprocess.run(cmd) -def parse_args(arg_list: list[str] | None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Filter OSM files with various operations.", formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument("tag", choices=["id", "b", "f"], metavar="tag", - help=""" +def parse_args(arg_list: list[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Filter OSM files with various operations.", + formatter_class=argparse.RawTextHelpFormatter, + ) + + parser.add_argument( + "tag", + choices=["id", "b", "f"], + metavar="tag", + help=""" id : Filter geographic objects based on relation ID b : Filter geographic objects based on bounding box (with osmium) f : Filter objects based on tags in expression_file -""") - parser.add_argument('input_file', nargs='?', help='Path to input OSM file') - parser.add_argument("-e", dest="expression_file", nargs="?", help="Path to expression file for filtering tags (required for 'f' tag)") - parser.add_argument("-c", dest="coords", nargs="?",help="Bounding box coordinates or path to config file (required for 'b' tag)") - parser.add_argument("-rid", dest="relation_id", nargs="?", help="relation ID (required for 'b' tag)") - parser.add_argument("-s", dest="strategy", help="Strategy type (optional for 'id', 'b' tags)") - parser.add_argument("-R", dest="omit_referenced", action="store_true", help="Omit referenced objects (optional for 'f' tag)") +""", + ) + parser.add_argument("input_file", nargs="?", help="Path to input OSM file") + parser.add_argument( + "-e", + dest="expression_file", + nargs="?", + help="Path to expression file for filtering tags (required for 'f' tag)", + ) + parser.add_argument( + "-c", + dest="coords", + nargs="?", + help="Bounding box coordinates or path to config file (required for 'b' tag)", + ) + parser.add_argument( + "-rid", dest="relation_id", nargs="?", help="relation ID (required for 'b' tag)" + ) + parser.add_argument( + "-s", dest="strategy", help="Strategy type (optional for 'id', 'b' tags)" + ) + parser.add_argument( + "-R", + dest="omit_referenced", + action="store_true", + help="Omit referenced objects (optional for 'f' tag)", + ) args = parser.parse_args(arg_list) return args + def main(arg_list: list[str] | None = None): args = parse_args(arg_list) if not os.path.exists(args.input_file): raise FileNotFoundError(f"File '{args.input_file}' does not exist.") elif not is_valid_extension(args.input_file): - raise InvalidInputError("File must have one of the following extensions: osm, osm.pbf, osm.bz2") - + raise InvalidInputError( + "File must have one of the following extensions: osm, osm.pbf, osm.bz2" + ) + match args.tag: case "id": # Filter geographic objects based on relation ID if not args.relation_id: raise MissingInputError("Existing relation ID must be specified.") - + check_strategy(args.strategy) - + extract_id(args.input_file, args.relation_id, args.strategy) case "b": # Filter geographic objects based on bounding box (with osmium) if not args.coords: - raise MissingInputError("Coordinates or config file need to be specified with the 'b' tag.") - + raise MissingInputError( + "Coordinates or config file need to be specified with the 'b' tag." + ) + check_strategy(args.strategy) - + extract_bbox(args.input_file, args.coords, args.strategy) case "f": @@ -120,9 +187,14 @@ def main(arg_list: list[str] | None = None): if not args.expression_file: raise MissingInputError("Expression file needs to be specified.") elif not os.path.exists(args.expression_file): - raise FileNotFoundError(f"File '{args.expression_file}' does not exist.") + raise FileNotFoundError( + f"File '{args.expression_file}' does not exist." + ) + + run_osmium_filter( + args.input_file, args.expression_file, args.omit_referenced + ) - run_osmium_filter(args.input_file, args.expression_file, args.omit_referenced) -if __name__ == '__main__': - main() \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/python/scripts/main.py b/python/scripts/main.py index 0b872b1..106eed7 100644 --- a/python/scripts/main.py +++ b/python/scripts/main.py @@ -9,6 +9,7 @@ contract_graph_in_area, get_area_for_demand, insert_area, select_network_nodes_in_area) from roadgraphtool.export import get_map_nodes_from_db +# from roadgraphtool.credentials_config import CREDENTIALS from scripts.process_osm import import_osm_to_db @@ -30,8 +31,7 @@ def configure_arg_parser() -> argparse.ArgumentParser: parser.add_argument( "-f", "--fill-speed", - type=bool, - choices=[True, False], + action="store_true", help="An option indicating if specific functions should process speed data. Default is set to False.", default=False, required=False, @@ -43,6 +43,14 @@ def configure_arg_parser() -> argparse.ArgumentParser: action="store_true", help="Import OSM data to database specified in config.ini", ) + parser.add_argument( + "-S", + "--Style", + help='Filename of the osm2pgsql style import in the directory "resources/lua_styles".' + + '\nDefault is set to "pipeline.lua"', + default="pipeline.lua", + required=False, + ) return parser @@ -52,7 +60,12 @@ def main(arg_list: list[str] | None = None): args = parser.parse_args(arg_list) if args.importing: - import_osm_to_db() + logging.info("Importing OSM data to database...") + retcode = import_osm_to_db(filename=args.importing, style_filename=args.Style) + if retcode != 0: + logging.error(f"Error during OSM data import. Return code: {retcode}") + return retcode + logging.info("OSM data imported successfully.") area_id = args.area_id area_srid = args.area_srid @@ -100,6 +113,8 @@ def main(arg_list: list[str] | None = None): logging.info("Execution of compute_speeds_from_neighborhood_segments") compute_speeds_from_neighborhood_segments(area_id, area_srid) + return 0 + if __name__ == "__main__": - main() + exit(main()) diff --git a/python/scripts/process_osm.py b/python/scripts/process_osm.py index a2b22e6..f336c34 100644 --- a/python/scripts/process_osm.py +++ b/python/scripts/process_osm.py @@ -1,21 +1,31 @@ import argparse import os import subprocess +from pathlib import Path -from roadgraphtool.credentials_config import CREDENTIALS as config, CredentialsConfig -from scripts.filter_osm import InvalidInputError, MissingInputError, load_multipolygon_by_id, is_valid_extension +from roadgraphtool.credentials_config import CREDENTIALS as config +from roadgraphtool.credentials_config import CredentialsConfig +from scripts.filter_osm import (InvalidInputError, MissingInputError, + is_valid_extension, load_multipolygon_by_id) from scripts.find_bbox import find_min_max +RESOURCES_DIR = Path(__file__).parent.parent.parent / "resources" +STYLES_DIR = RESOURCES_DIR / "lua_styles" + + def extract_bbox(relation_id: int): """Function to determine bounding box coordinations.""" content = load_multipolygon_by_id(relation_id) min_lon, min_lat, max_lon, max_lat = find_min_max(content) return min_lon, min_lat, max_lon, max_lat -def run_osmium_cmd(tag: str, input_file: str, output_file: str = None): + +def run_osmium_cmd(tag: str, input_file: str, output_file: str): """Function to run osmium command based on tag.""" if output_file and not is_valid_extension(output_file): - raise InvalidInputError("File must have one of the following extensions: osm, osm.pbf, osm.bz2") + raise InvalidInputError( + "File must have one of the following extensions: osm, osm.pbf, osm.bz2" + ) match tag: case "d": subprocess.run(["osmium", "show", input_file]) @@ -23,44 +33,99 @@ def run_osmium_cmd(tag: str, input_file: str, output_file: str = None): subprocess.run(["osmium", "fileinfo", input_file]) case "ie": subprocess.run(["osmium", "fileinfo", "-e", input_file]) - case 'r': + case "r": subprocess.run(["osmium", "renumber", input_file, "-o", output_file]) - case 's': + case "s": subprocess.run(["osmium", "sort", input_file, "-o", output_file]) - case 'sr': - tmp_file = 'tmp.osm' + case "sr": + tmp_file = "tmp.osm" subprocess.run(["osmium", "sort", input_file, "-o", tmp_file]) subprocess.run(["osmium", "renumber", tmp_file, "-o", output_file]) os.remove(tmp_file) -def run_osm2pgsql_cmd(config: CredentialsConfig, input_file: str, style_file_path: str, coords: str| list[int] = None): - """Function to run osm2pgsql command.""" - command = ["osm2pgsql", "-d", config.db_name, "-U", config.username, "-W", "-H", config.db_host, - "-P", str(config.db_server_port), "--output=flex", "-S", style_file_path, input_file, "-x"] + +def run_osm2pgsql_cmd( + config: CredentialsConfig, + input_file: str, + style_file_path: str, + coords: str | list[int] | None = None, +) -> int: + """Function to run osm2pgsl command. + Returns return code of the subprocess.""" + command = [ + "osm2pgsql", + "-d", + config.db_name, + "-U", + config.username, + "-W", + "-H", + config.db_host, + "-P", + str(config.db_server_port), + "--output=flex", + "-S", + style_file_path, + input_file, + "-x", + ] if coords: command.extend(["-b", coords]) - subprocess.run(command) + return subprocess.run(command).returncode + -def import_osm_to_db(): - """Function to import OSM file do database specified in config.ini file. +def import_osm_to_db(filename: str | None = None, style_filename: str = "pipeline.lua"): + """Function to import OSM file specified in config.ini file to database. The function expects the OSM file to be saved as resources/to_import.*. - The default.lua style file is used. + The pipeline.lua style file is used as default style. + + The function will try to import the file with the following extensions: + .osm, .osm.pbf, .osm.bz2. + If the file is not found, a FileNotFoundError is raised. + If the file has an invalid extension, an InvalidInputError is raised. + + The function returns the return code of the subprocess. """ - input_files = ["resources/to_import.osm", "resources/to_import.osm.pbf", "resources/to_import.osm.bz2"] + input_file = None - for file in input_files: - if os.path.exists(file) and is_valid_extension(file): - input_file = file + + if not filename: + filename = "to_import" + + input_files = [ + str(RESOURCES_DIR / f"{filename}.osm"), + str(RESOURCES_DIR / f"{filename}.osm.pbf"), + str(RESOURCES_DIR / f"{filename}.osm.bz2"), + ] + + for file in input_files: + if os.path.exists(file) and is_valid_extension(file): + input_file = file + elif os.path.exists(RESOURCES_DIR / filename): + input_file = str(RESOURCES_DIR / filename) + if not input_file: raise FileNotFoundError("There is no valid file to import.") - style_file_path = "resources/lua_styles/default.lua" - run_osm2pgsql_cmd(config, input_file, style_file_path) + + style_file_path = str(STYLES_DIR / style_filename) + + return run_osm2pgsql_cmd(config, input_file, style_file_path) + + +# Main flow of the current file, including functions used only within this file + def parse_args(arg_list: list[str] | None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Process OSM files and interact with PostgreSQL database.", formatter_class=argparse.RawTextHelpFormatter) + parser = argparse.ArgumentParser( + description="Process OSM files and interact with PostgreSQL database.", + formatter_class=argparse.RawTextHelpFormatter, + ) - parser.add_argument("tag", choices=["d", "i", "ie", "s", "r", "sr", "b", "u"], metavar="tag", - help=""" + parser.add_argument( + "tag", + choices=["d", "i", "ie", "s", "r", "sr", "b", "u"], + metavar="tag", + help=""" d : Display OSM file i : Display information about OSM file ie : Display extended information about OSM file @@ -69,41 +134,57 @@ def parse_args(arg_list: list[str] | None) -> argparse.Namespace: sr : Sort and renumber objects in OSM file b : Upload OSM file to PostgreSQL database using osm2pgsql u : Extract greatest bounding box from given relation ID of - input_file and upload to PostgreSQL database using osm2pgsql""" -) - parser.add_argument('input_file', nargs='?', help="Path to input OSM file") - parser.add_argument("-id", dest="relation_id", nargs="?", help="Relation ID (required for 'b' tag)") - parser.add_argument("-l", dest="style_file", default="resources/lua_styles/default.lua", help="Path to style file (optional for 'b', 'u' tag)") - parser.add_argument("-o", dest="output_file", help="Path to output file (required for 's', 'r', 'sr' tag)") + input_file and upload to PostgreSQL database using osm2pgsql""", + ) + parser.add_argument("input_file", nargs="?", help="Path to input OSM file") + parser.add_argument( + "-id", dest="relation_id", nargs="?", help="Relation ID (required for 'b' tag)" + ) + parser.add_argument( + "-l", + dest="style_file", + default="default.lua", + help="Path to style file (optional for 'b', 'u' tag)", + ) + parser.add_argument( + "-o", + dest="output_file", + help="Path to output file (required for 's', 'r', 'sr' tag)", + ) args = parser.parse_args(arg_list) return args + def main(arg_list: list[str] | None = None): args = parse_args(arg_list) if not os.path.exists(args.input_file): raise FileNotFoundError(f"File '{args.input_file}' does not exist.") elif not is_valid_extension(args.input_file): - raise InvalidInputError("File must have one of the following extensions: osm, osm.pbf, osm.bz2.") + raise InvalidInputError( + "File must have one of the following extensions: osm, osm.pbf, osm.bz2." + ) elif args.style_file: if not os.path.exists(args.style_file): raise FileNotFoundError(f"File '{args.style_file}' does not exist.") elif not args.style_file.endswith(".lua"): raise InvalidInputError("File must have the '.lua' extension.") - + match args.tag: - case 'd' | 'i' | 'ie': + case "d" | "i" | "ie": # Display content or (extended) information of OSM file run_osmium_cmd(args.tag, args.input_file) - case 's' | 'r' | 'sr': + case "s" | "r" | "sr": # Sort, renumber OSM file or do both if not args.output_file: - raise MissingInputError("An output file must be specified with '-o' tag.") + raise MissingInputError( + "An output file must be specified with '-o' tag." + ) run_osmium_cmd(args.tag, args.input_file, args.output_file) - + case "u": # Upload OSM file to PostgreSQL database run_osm2pgsql_cmd(config, args.input_file, args.style_file) @@ -116,6 +197,7 @@ def main(arg_list: list[str] | None = None): coords = f"{min_lon},{min_lat},{max_lon},{max_lat}" run_osm2pgsql_cmd(config, args.input_file, args.style_file, coords) - -if __name__ == '__main__': - main() \ No newline at end of file + + +if __name__ == "__main__": + main() From e566169c648b430b1ef9f3ebed9a0bb91f3474ee Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Tue, 3 Sep 2024 14:53:28 +0200 Subject: [PATCH 03/28] Updated .gitignore to ignore Poetry files --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1206405..3454acf 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,8 @@ resources/lua_styles/mydef.lua .pytest_cache config.ini resources/to_import.osm.pbf -resources/performance_report.json \ No newline at end of file +resources/performance_report.json + +# poetry environment exclusions +poetry.lock +pyproject.toml From 8afac5dd0e10680982b282e2b60b10d8f08ea179 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Tue, 3 Sep 2024 17:49:23 +0200 Subject: [PATCH 04/28] Added post-processing py wrapper --- python/roadgraphtool/db.py | 66 +++++++++----- python/scripts/install_sql.py | 165 ++++++++++++++++++---------------- python/scripts/main.py | 72 +++++++-------- python/scripts/process_osm.py | 39 +++++++- 4 files changed, 205 insertions(+), 137 deletions(-) diff --git a/python/roadgraphtool/db.py b/python/roadgraphtool/db.py index 3108924..8af93fd 100644 --- a/python/roadgraphtool/db.py +++ b/python/roadgraphtool/db.py @@ -1,17 +1,22 @@ import atexit import logging -import psycopg2 -import sshtunnel -import sqlalchemy +from pathlib import Path + +import geopandas as gpd import pandas as pd +import psycopg2 import psycopg2.errors -import geopandas as gpd -from pathlib import Path +import sqlalchemy +import sshtunnel from sqlalchemy.engine import Row from roadgraphtool.credentials_config import CREDENTIALS -logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%H:%M:%S') +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + datefmt="%H:%M:%S", +) def connect_db_if_required(db_function): @@ -21,7 +26,7 @@ def connect_db_if_required(db_function): def wrapper(*args, **kwargs): db = args[0] - if hasattr(db, 'server'): + if hasattr(db, "server"): db.start_or_restart_ssh_connection_if_needed() if not db.is_connected(): db.set_up_db_connections() @@ -39,6 +44,7 @@ class __Database: Import as: from db import db """ + config = CREDENTIALS def __init__(self): @@ -46,7 +52,7 @@ def __init__(self): self.db_server_port = self.config.db_server_port self.db_name = self.config.db_name self.ssh_tunnel_local_port = 1113 - if hasattr(self.config, 'server'): + if hasattr(self.config, "server"): self.server = self.config.server self.ssh_server = None self.host = self.config.host @@ -58,7 +64,9 @@ def __init__(self): self._sql_alchemy_engine_str = None def is_connected(self): - return (self._psycopg2_connection is not None) and (self._sqlalchemy_engine is not None) + return (self._psycopg2_connection is not None) and ( + self._sqlalchemy_engine is not None + ) def set_up_db_connections(self): # psycopg2 connection object @@ -75,22 +83,24 @@ def set_ssh_to_db_server_and_set_port(self): ssh_pkey=self.config.private_key_path, ssh_username=self.config.server_username, ssh_private_key_password=self.config.private_key_phrase, - remote_bind_address=('localhost', self.db_server_port), - local_bind_address=('localhost', self.ssh_tunnel_local_port) + remote_bind_address=("localhost", self.db_server_port), + local_bind_address=("localhost", self.ssh_tunnel_local_port), ) try: self.ssh_server = sshtunnel.open_tunnel(self.server, **ssh_kwargs) except sshtunnel.paramiko.SSHException as e: # sshtunnel dependency paramiko may attempt to use ssh-agent and crashes if it fails logging.warning(f"sshtunnel.paramiko.SSHException: '{e}'") - self.ssh_server = sshtunnel.open_tunnel(self.server, **ssh_kwargs, allow_agent=False) + self.ssh_server = sshtunnel.open_tunnel( + self.server, **ssh_kwargs, allow_agent=False + ) self.ssh_server.start() logging.info( "SSH tunnel established from %s to %s/%s", self.ssh_server.local_bind_address, self.ssh_server.ssh_host, - self.db_server_port + self.db_server_port, ) self.db_server_port = self.ssh_server.local_bind_port @@ -110,12 +120,15 @@ def start_or_restart_ssh_connection_if_needed(self): self.ssh_server.restart() def get_sql_alchemy_engine_str(self): - sql_alchemy_engine_str = 'postgresql+psycopg2://{user}:{password}@{host}:{port}/{dbname}'.format( - user=self.config.username, - password=self.config.db_password, - host=self.host, - port=self.db_server_port, - dbname=self.db_name) + sql_alchemy_engine_str = ( + "postgresql+psycopg2://{user}:{password}@{host}:{port}/{dbname}".format( + user=self.config.username, + password=self.config.db_password, + host=self.host, + port=self.db_server_port, + dbname=self.db_name, + ) + ) return sql_alchemy_engine_str @@ -129,7 +142,7 @@ def get_new_psycopg2_connection(self): password=self.config.db_password, host=self.config.db_host, port=self.db_server_port, - dbname=self.db_name + dbname=self.db_name, ) atexit.register(psycopg2_connection.close) @@ -157,18 +170,21 @@ def execute_sql_and_fetch_all_rows(self, query, *args) -> list[Row]: return result @connect_db_if_required - def execute_script(self, script_path: Path): + def execute_script(self, script_path: Path) -> int: with open(script_path) as f: script = f.read() cursor = self._psycopg2_connection.cursor() + retcode = 0 try: cursor.execute(script) self._psycopg2_connection.commit() except Exception as e: logging.error(f"Error executing script {script_path}: {e}") self._psycopg2_connection.rollback() + retcode = 1 finally: cursor.close() + return retcode @connect_db_if_required def execute_query_to_geopandas(self, sql: str, **kwargs) -> pd.DataFrame: @@ -203,7 +219,9 @@ def execute_query_to_pandas(self, sql: str, **kwargs) -> pd.DataFrame: return data @connect_db_if_required - def dataframe_to_db_table(self, df: pd.DataFrame, table_name: str, **kwargs) -> None: + def dataframe_to_db_table( + self, df: pd.DataFrame, table_name: str, **kwargs + ) -> None: """ Save DataFrame to a new table in the database @@ -212,7 +230,9 @@ def dataframe_to_db_table(self, df: pd.DataFrame, table_name: str, **kwargs) -> https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_sql.html """ - df.to_sql(table_name, con=self._sqlalchemy_engine, if_exists='append', index=False) + df.to_sql( + table_name, con=self._sqlalchemy_engine, if_exists="append", index=False + ) @connect_db_if_required def db_table_to_pandas(self, table_name: str, **kwargs) -> pd.DataFrame: diff --git a/python/scripts/install_sql.py b/python/scripts/install_sql.py index f93d91e..f43deca 100644 --- a/python/scripts/install_sql.py +++ b/python/scripts/install_sql.py @@ -5,89 +5,100 @@ from roadgraphtool.db import db db_name = db.db_name -sql_dir = Path(__file__).parent.parent.parent / "SQL" +SQL_DIR = Path(__file__).parent.parent.parent / "SQL" -def execute_sql_file(sql_file: Path, multistatement: bool = False): +def execute_sql_file(sql_file: Path, multistatement: bool = False) -> int: logging.info(f"Executing {sql_file}") + retcode = 0 if multistatement: - db.execute_script(sql_file) + retcode = db.execute_script(sql_file) else: with sql_file.open() as f: sql = f.read() db.execute_sql(sql) - - -logging.basicConfig(level=logging.INFO) - -# Check availability status of extensions in db -logging.info("Checking availability of extensions in the database") -extension_list = ["postgis", "pgrouting", "hstore", "pgtap"] - -sql = f""" -WITH extension_list(name) AS ( - VALUES - {','.join(f"('{ext}')" for ext in extension_list)} -) -SELECT - el.name AS extension_name, - CASE - WHEN ae.name IS NOT NULL THEN 'Available' - ELSE 'Not Available' - END AS status -FROM extension_list el -LEFT JOIN pg_available_extensions ae ON el.name = ae.name -ORDER BY el.name; -""" - -extensions = {} -for extension_name, status in db.execute_sql_and_fetch_all_rows(sql): - logging.info(f"{extension_name}: {status}") - extensions[extension_name] = status == "Available" - -# Assert that critically needed extensions are present -if not extensions["postgis"] or not extensions["pgrouting"] or not extensions["hstore"]: - raise Exception("Missing critical extensions") - -# initialize database if it's empty -sql = f"""SELECT EXISTS ( - SELECT FROM information_schema.tables - WHERE table_schema = 'public' AND table_name = 'areas' - ); - """ -if not db.execute_sql_and_fetch_all_rows(sql)[0][0]: - main_sql_path = sql_dir / "main.sql" - logging.info("Initializing the database") - execute_sql_file(main_sql_path, multistatement=True) - -# enable pgtap if it's not enabled -sql = """SELECT * FROM pg_extension WHERE extname = 'pgtap'""" -if extensions["pgtap"] and not db.execute_sql_and_fetch_all_rows(sql): - logging.info("Enabling pgtap") - execute_sql_file(sql_dir / "testing_extension.sql", multistatement=True) -else: - if not extensions["pgtap"]: - logging.warning("pgtap extension is not available") + return retcode + + +def main(): + logging.basicConfig(level=logging.INFO) + + # Check availability status of extensions in db + logging.info("Checking availability of extensions in the database") + extension_list = ["postgis", "pgrouting", "hstore", "pgtap"] + + sql = f""" + WITH extension_list(name) AS ( + VALUES + {','.join(f"('{ext}')" for ext in extension_list)} + ) + SELECT + el.name AS extension_name, + CASE + WHEN ae.name IS NOT NULL THEN 'Available' + ELSE 'Not Available' + END AS status + FROM extension_list el + LEFT JOIN pg_available_extensions ae ON el.name = ae.name + ORDER BY el.name; + """ + + extensions = {} + for extension_name, status in db.execute_sql_and_fetch_all_rows(sql): + logging.info(f"{extension_name}: {status}") + extensions[extension_name] = status == "Available" + + # Assert that critically needed extensions are present + if ( + not extensions["postgis"] + or not extensions["pgrouting"] + or not extensions["hstore"] + ): + raise Exception("Missing critical extensions") + + # initialize database if it's empty + sql = f"""SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = 'areas' + ); + """ + if not db.execute_sql_and_fetch_all_rows(sql)[0][0]: + main_sql_path = SQL_DIR / "main.sql" + logging.info("Initializing the database") + execute_sql_file(main_sql_path, multistatement=True) + + # enable pgtap if it's not enabled + sql = """SELECT * FROM pg_extension WHERE extname = 'pgtap'""" + if extensions["pgtap"] and not db.execute_sql_and_fetch_all_rows(sql): + logging.info("Enabling pgtap") + execute_sql_file(SQL_DIR / "testing_extension.sql", multistatement=True) else: - logging.info("pgtap extension is already enabled") - -# Create test schema if it doesn't exist -sql = """SELECT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'test_env')""" -if not db.execute_sql_and_fetch_all_rows(sql)[0][0]: - logging.info("Creating test schema") - db.execute_sql("CREATE SCHEMA test_env") - -functions_dir = sql_dir / "functions" -logging.info("Importing functions from %s", functions_dir) -for sql_function_file in functions_dir.rglob("*.sql"): - execute_sql_file(sql_function_file) - -procedures_dir = sql_dir / "procedures" -logging.info("Importing procedures from %s", procedures_dir) -for sql_procedure_file in procedures_dir.rglob("*.sql"): - execute_sql_file(sql_procedure_file) - -test_dir = sql_dir / "tests" -logging.info("Importing test functions from %s", test_dir) -for sql_test_file in test_dir.rglob("*.sql"): - execute_sql_file(sql_test_file, multistatement=True) + if not extensions["pgtap"]: + logging.warning("pgtap extension is not available") + else: + logging.info("pgtap extension is already enabled") + + # Create test schema if it doesn't exist + sql = """SELECT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'test_env')""" + if not db.execute_sql_and_fetch_all_rows(sql)[0][0]: + logging.info("Creating test schema") + db.execute_sql("CREATE SCHEMA test_env") + + functions_dir = SQL_DIR / "functions" + logging.info("Importing functions from %s", functions_dir) + for sql_function_file in functions_dir.rglob("*.sql"): + execute_sql_file(sql_function_file) + + procedures_dir = SQL_DIR / "procedures" + logging.info("Importing procedures from %s", procedures_dir) + for sql_procedure_file in procedures_dir.rglob("*.sql"): + execute_sql_file(sql_procedure_file) + + test_dir = SQL_DIR / "tests" + logging.info("Importing test functions from %s", test_dir) + for sql_test_file in test_dir.rglob("*.sql"): + execute_sql_file(sql_test_file, multistatement=True) + + +if __name__ == "__main__": + exit(main()) diff --git a/python/scripts/main.py b/python/scripts/main.py index 106eed7..d9fea80 100644 --- a/python/scripts/main.py +++ b/python/scripts/main.py @@ -71,47 +71,47 @@ def main(arg_list: list[str] | None = None): area_srid = args.area_srid fill_speed = args.fill_speed - logging.info("selecting nodes") - nodes = select_network_nodes_in_area(area_id) - logging.info("selected network nodes in area_id = {}".format(area_id)) - print(nodes) + # logging.info("selecting nodes") + # nodes = select_network_nodes_in_area(area_id) + # logging.info("selected network nodes in area_id = {}".format(area_id)) + # print(nodes) - logging.info("contracting graph") - contract_graph_in_area(area_id, area_srid, fill_speed) + # logging.info("contracting graph") + # contract_graph_in_area(area_id, area_srid, fill_speed) - logging.info("computing strong components for area_id = {}".format(area_id)) - compute_strong_components(area_id) - logging.info("storing the results in the component_data table") + # logging.info("computing strong components for area_id = {}".format(area_id)) + # compute_strong_components(area_id) + # logging.info("storing the results in the component_data table") # insert_area("test1", []) - area = get_area_for_demand( - 4326, - [1, 2, 3], - [1, 2, 3], - 1000, - 5, - "2023-01-01 00:00:00", - "2023-12-31 23:59:59", - (50.0, 10.0), - 5000, - ) - print(area) - - logging.info("Execution of assign_average_speeds_to_all_segments_in_area") - try: - assign_average_speed_to_all_segments_in_area(area_id, area_srid) - except psycopg2.errors.InvalidParameterValue as e: - logging.info("Expected Error: ", e) - - nodes = get_map_nodes_from_db(area_id) - print(nodes) - - logging.info("Execution of compute_speeds_for_segments") - compute_speeds_for_segments(area_id, 1, 12, 1) - - logging.info("Execution of compute_speeds_from_neighborhood_segments") - compute_speeds_from_neighborhood_segments(area_id, area_srid) + # area = get_area_for_demand( + # 4326, + # [1, 2, 3], + # [1, 2, 3], + # 1000, + # 5, + # "2023-01-01 00:00:00", + # "2023-12-31 23:59:59", + # (50.0, 10.0), + # 5000, + # ) + # print(area) + + # logging.info("Execution of assign_average_speeds_to_all_segments_in_area") + # try: + # assign_average_speed_to_all_segments_in_area(area_id, area_srid) + # except psycopg2.errors.InvalidParameterValue as e: + # logging.info("Expected Error: ", e) + + # nodes = get_map_nodes_from_db(area_id) + # print(nodes) + + # logging.info("Execution of compute_speeds_for_segments") + # compute_speeds_for_segments(area_id, 1, 12, 1) + + # logging.info("Execution of compute_speeds_from_neighborhood_segments") + # compute_speeds_from_neighborhood_segments(area_id, area_srid) return 0 diff --git a/python/scripts/process_osm.py b/python/scripts/process_osm.py index f336c34..0060fb0 100644 --- a/python/scripts/process_osm.py +++ b/python/scripts/process_osm.py @@ -3,11 +3,13 @@ import subprocess from pathlib import Path +import roadgraphtool.log from roadgraphtool.credentials_config import CREDENTIALS as config from roadgraphtool.credentials_config import CredentialsConfig from scripts.filter_osm import (InvalidInputError, MissingInputError, is_valid_extension, load_multipolygon_by_id) from scripts.find_bbox import find_min_max +from scripts.install_sql import SQL_DIR, execute_sql_file, logging RESOURCES_DIR = Path(__file__).parent.parent.parent / "resources" STYLES_DIR = RESOURCES_DIR / "lua_styles" @@ -74,6 +76,39 @@ def run_osm2pgsql_cmd( return subprocess.run(command).returncode +def post_process_osm_import(style_filename: str) -> int: + post_proc_dict = {"pipeline.lua": "after_import.sql"} + + if not post_proc_dict[style_filename]: + logging.warning(f"No post-processing defined for style {style_filename}") + return 0 + + sql_filepath = SQL_DIR / post_proc_dict[style_filename] + + logging.info("Post-processing OSM import...") + retcode = subprocess.run( + [ + "psql", + "-d", + config.db_name, + "-U", + config.username, + "-p", + str(config.db_server_port), + "-h", + config.db_host, + "-f", + sql_filepath, + ] + ).returncode + if retcode != 0: + logging.error(f"Error during post-processing. Return code: {retcode}") + return retcode + logging.info("Post-processing done.") + + return 0 + + def import_osm_to_db(filename: str | None = None, style_filename: str = "pipeline.lua"): """Function to import OSM file specified in config.ini file to database. The function expects the OSM file to be saved as resources/to_import.*. @@ -109,7 +144,9 @@ def import_osm_to_db(filename: str | None = None, style_filename: str = "pipelin style_file_path = str(STYLES_DIR / style_filename) - return run_osm2pgsql_cmd(config, input_file, style_file_path) + return run_osm2pgsql_cmd( + config, input_file, style_file_path + ) or post_process_osm_import(style_filename) # Main flow of the current file, including functions used only within this file From 3f40923df9f20e00bf399ca0b8be0583efb7df00 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Mon, 9 Sep 2024 11:29:33 +0200 Subject: [PATCH 05/28] Updated insert_area function in main.py --- python/scripts/main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/scripts/main.py b/python/scripts/main.py index d9fea80..cd59cff 100644 --- a/python/scripts/main.py +++ b/python/scripts/main.py @@ -10,6 +10,7 @@ select_network_nodes_in_area) from roadgraphtool.export import get_map_nodes_from_db # from roadgraphtool.credentials_config import CREDENTIALS +from roadgraphtool.insert_area import read_json_file as read_area_file from scripts.process_osm import import_osm_to_db @@ -42,6 +43,7 @@ def configure_arg_parser() -> argparse.ArgumentParser: dest="importing", action="store_true", help="Import OSM data to database specified in config.ini", + required=False, ) parser.add_argument( "-S", @@ -71,6 +73,13 @@ def main(arg_list: list[str] | None = None): area_srid = args.area_srid fill_speed = args.fill_speed + insert_area( + area_id, + "Insertion_area", + "Description of the area", + read_area_file("testarea.json"), + ) + # logging.info("selecting nodes") # nodes = select_network_nodes_in_area(area_id) # logging.info("selected network nodes in area_id = {}".format(area_id)) From d719dd0e1434064612548875f6335525a5f71d92 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Mon, 9 Sep 2024 11:29:52 +0200 Subject: [PATCH 06/28] Updated default path for style file --- python/scripts/process_osm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/scripts/process_osm.py b/python/scripts/process_osm.py index 0060fb0..0ca4360 100644 --- a/python/scripts/process_osm.py +++ b/python/scripts/process_osm.py @@ -180,7 +180,7 @@ def parse_args(arg_list: list[str] | None) -> argparse.Namespace: parser.add_argument( "-l", dest="style_file", - default="default.lua", + default=str(STYLES_DIR / "default.lua"), help="Path to style file (optional for 'b', 'u' tag)", ) parser.add_argument( From a365b1d0e5d5b9ed048203906f89fd7968946918 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Mon, 9 Sep 2024 11:30:53 +0200 Subject: [PATCH 07/28] Reformatted test_filter_osm.py --- python/tests/test_filter_osm.py | 158 +++++++++++++++++++++++--------- 1 file changed, 116 insertions(+), 42 deletions(-) diff --git a/python/tests/test_filter_osm.py b/python/tests/test_filter_osm.py index dbf6dcf..9433aec 100644 --- a/python/tests/test_filter_osm.py +++ b/python/tests/test_filter_osm.py @@ -1,57 +1,74 @@ import os import pathlib import tempfile -import pytest import xml.etree.ElementTree as ET -from scripts.filter_osm import check_strategy, extract_id, is_valid_extension, load_multipolygon_by_id, extract_bbox, main, InvalidInputError, MissingInputError +import pytest + +from scripts.filter_osm import (InvalidInputError, MissingInputError, + check_strategy, extract_bbox, extract_id, + is_valid_extension, load_multipolygon_by_id, + main) + def test_check_strategy(): assert check_strategy("simple") == None assert check_strategy("complete_ways") == None assert check_strategy("smart") == None - with pytest.raises(InvalidInputError, match="Invalid strategy type. Call filter_osm.py -h/--help to display help."): + with pytest.raises( + InvalidInputError, + match="Invalid strategy type. Call filter_osm.py -h/--help to display help.", + ): check_strategy("invalid_strategy") - + + @pytest.fixture def expected_multipolygon_id(): parent_dir = pathlib.Path(__file__).parent file_path = str(parent_dir) + "/data/expected_multipolygon_id.osm" - with open(file_path, 'rb') as f: + with open(file_path, "rb") as f: return f.read() + @pytest.fixture def mock_subprocess_run(mocker): return mocker.patch("subprocess.run") + @pytest.fixture def mock_os_path_isfile(mocker): return mocker.patch("os.path.isfile", return_value=True) + @pytest.fixture def mock_open(mocker): return mocker.patch("builtins.open", mocker.mock_open()) + @pytest.fixture def mock_remove(mocker): return mocker.patch("os.remove") + # TESTS: + def test_is_valid_extension_valid(): valid_file = "test.osm" - assert is_valid_extension(valid_file) == True + assert is_valid_extension(valid_file) + def test_is_valid_extension_invalid(): invalid_file = "test.pdf" - assert is_valid_extension(invalid_file) == False + assert not is_valid_extension(invalid_file) + -def test_load_multipolygon_by_id_url(mocker,expected_multipolygon_id): +def test_load_multipolygon_by_id_url(mocker, expected_multipolygon_id): relation_id = 5986438 url = f"https://www.openstreetmap.org/api/0.6/relation/{relation_id}/full" # Check HTTP handling - mock_get = mocker.patch('requests.get') + mock_get = mocker.patch("requests.get") mock_get.return_value.status_code = 200 mock_get.return_value.content = expected_multipolygon_id @@ -59,10 +76,11 @@ def test_load_multipolygon_by_id_url(mocker,expected_multipolygon_id): mock_get.assert_called_once_with(url) assert result == expected_multipolygon_id + def test_load_multipolygon_by_id_contains_id(mocker, expected_multipolygon_id): relation_id = 5986438 - mock_get = mocker.patch('requests.get') + mock_get = mocker.patch("requests.get") mock_get.return_value.status_code = 200 mock_get.return_value.content = expected_multipolygon_id @@ -74,17 +92,20 @@ def test_load_multipolygon_by_id_contains_id(mocker, expected_multipolygon_id): relation_id_str = str(relation_id) contains_relation_id = any( - relation.attrib.get('id') == relation_id_str - for relation in root.findall('relation') + relation.attrib.get("id") == relation_id_str + for relation in root.findall("relation") ) - assert contains_relation_id, f"Result does not contain the relation_id {relation_id}" - + assert ( + contains_relation_id + ), f"Result does not contain the relation_id {relation_id}" + + def test_extract_id_contains_id(): relation_id = 5986438 parent_dir = pathlib.Path(__file__).parent.parent.parent input_file = str(parent_dir) + "/python/tests/data/id_test.osm" - output_file = str(parent_dir) + "/resources/id_extract.osm" + output_file = str(parent_dir) + "/resources/id_extract.osm" extract_id(input_file, relation_id) @@ -92,38 +113,50 @@ def test_extract_id_contains_id(): assert os.path.exists(output_file), "Output file was not created" # Check that output file contains relation_id - with open(output_file, 'rb') as f: + with open(output_file, "rb") as f: tree = ET.ElementTree(ET.fromstring(f.read())) root = tree.getroot() relation_id_str = str(relation_id) contains_relation_id = any( - relation.attrib.get('id') == relation_id_str - for relation in root.findall('relation') + relation.attrib.get("id") == relation_id_str + for relation in root.findall("relation") ) - assert contains_relation_id, f"File content does not contain the relation_id {relation_id_str}." + assert ( + contains_relation_id + ), f"File content does not contain the relation_id {relation_id_str}." os.remove(output_file) + def test_extract_bbox_valid_coords(mock_subprocess_run): coords = "12.3456,78.9012,34.5678,90.1234" input_file = "test.osm.pbf" extract_bbox(input_file, coords) expected_command = [ - "osmium", "extract", "-b", coords, input_file, "-o", "extracted-bbox.osm.pbf" + "osmium", + "extract", + "-b", + coords, + input_file, + "-o", + "extracted-bbox.osm.pbf", ] mock_subprocess_run.assert_called_once_with(expected_command) + def test_extract_bbox_config_valid(mock_subprocess_run, mock_os_path_isfile): coords = "path/to/config/file.geojson" input_file = "test.osm.pbf" mock_os_path_isfile.return_value = True extract_bbox(input_file, coords) - expected_command = [ - "osmium", "extract", "-c", coords, input_file] + expected_command = ["osmium", "extract", "-c", coords, input_file] mock_subprocess_run.assert_called_once_with(expected_command) -def test_extract_bbox_coords_and_config_invalid(mock_subprocess_run, mock_os_path_isfile): + +def test_extract_bbox_coords_and_config_invalid( + mock_subprocess_run, mock_os_path_isfile +): coords = "invalid,coords,string" input_file = "test.osm.pbf" mock_os_path_isfile.return_value = False @@ -131,99 +164,140 @@ def test_extract_bbox_coords_and_config_invalid(mock_subprocess_run, mock_os_pat extract_bbox(input_file, coords) mock_subprocess_run.assert_not_called() + def test_main_inputfile_invalid(): arg_list = ["id", "invalid_file.osm"] - with pytest.raises(FileNotFoundError, match="File 'invalid_file.osm' does not exist."): + with pytest.raises( + FileNotFoundError, match="File 'invalid_file.osm' does not exist." + ): main(arg_list) + def test_main_inputfile_extension_invalid(): with tempfile.NamedTemporaryFile(suffix=".txt") as tmp_file: arg_list = ["id", tmp_file.name] - with pytest.raises(InvalidInputError, match="File must have one of the following extensions: osm, osm.pbf, osm.bz2"): + with pytest.raises( + InvalidInputError, + match="File must have one of the following extensions: osm, osm.pbf, osm.bz2", + ): main(arg_list) + def test_main_id_missing(): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["id", tmp_file.name] - with pytest.raises(MissingInputError, match="Existing relation ID must be specified."): + with pytest.raises( + MissingInputError, match="Existing relation ID must be specified." + ): main(arg_list) + def test_main_id_invalid_strategy(): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["id", tmp_file.name, "-rid", "1234", "-s", "invalid"] - with pytest.raises(InvalidInputError, match="Invalid strategy type. Call filter_osm.py -h/--help to display help."): + with pytest.raises( + InvalidInputError, + match="Invalid strategy type. Call filter_osm.py -h/--help to display help.", + ): main(arg_list) + def test_main_id_valid(mocker): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["id", tmp_file.name, "-rid", "1234"] - mock_extract_id = mocker.patch('scripts.filter_osm.extract_id') + mock_extract_id = mocker.patch("scripts.filter_osm.extract_id") main(arg_list) mock_extract_id.assert_called_once_with(arg_list[1], arg_list[3], None) + def test_main_id_startegy_valid(mocker): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["id", tmp_file.name, "-rid", "1234", "-s", "simple"] - mock_extract_id = mocker.patch('scripts.filter_osm.extract_id') + mock_extract_id = mocker.patch("scripts.filter_osm.extract_id") main(arg_list) mock_extract_id.assert_called_once_with(arg_list[1], arg_list[3], "simple") + def test_main_b_missing_coord(): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["b", tmp_file.name] - with pytest.raises(MissingInputError, match="Coordinates or config file need to be specified with the 'b' tag."): + with pytest.raises( + MissingInputError, + match="Coordinates or config file need to be specified with the 'b' tag.", + ): main(arg_list) + def test_main_b_strategy_invalid(): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["b", tmp_file.name, "-c", "10,20,30,40", "-s", "invalid"] - with pytest.raises(InvalidInputError, match="Invalid strategy type. Call filter_osm.py -h/--help to display help."): + with pytest.raises( + InvalidInputError, + match="Invalid strategy type. Call filter_osm.py -h/--help to display help.", + ): main(arg_list) + def test_main_b_coords_valid(mocker): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["b", tmp_file.name, "-c", "10,20,30,40"] - mock_extract_bbox = mocker.patch('scripts.filter_osm.extract_bbox') + mock_extract_bbox = mocker.patch("scripts.filter_osm.extract_bbox") main(arg_list) mock_extract_bbox.assert_called_once_with(arg_list[1], arg_list[3], None) + def test_main_b_config_valid(mocker): - with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file, tempfile.NamedTemporaryFile(suffix=".json") as config_file: + with tempfile.NamedTemporaryFile( + suffix=".osm" + ) as tmp_file, tempfile.NamedTemporaryFile(suffix=".json") as config_file: arg_list = ["b", tmp_file.name, "-c", config_file.name] - mock_extract_bbox = mocker.patch('scripts.filter_osm.extract_bbox') + mock_extract_bbox = mocker.patch("scripts.filter_osm.extract_bbox") main(arg_list) mock_extract_bbox.assert_called_once_with(arg_list[1], arg_list[3], None) + def test_main_b_strategy_valid(mocker): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["b", tmp_file.name, "-c", "10,20,30,40", "-s", "simple"] - mock_extract_bbox = mocker.patch('scripts.filter_osm.extract_bbox') + mock_extract_bbox = mocker.patch("scripts.filter_osm.extract_bbox") main(arg_list) mock_extract_bbox.assert_called_once_with(arg_list[1], arg_list[3], "simple") + def test_main_f_missing(): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["f", tmp_file.name] - with pytest.raises(MissingInputError, match="Expression file needs to be specified."): + with pytest.raises( + MissingInputError, match="Expression file needs to be specified." + ): main(arg_list) + def test_main_f_expressionfile_invalid(): invalid_expression = "invalid_expression.txt" with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["f", tmp_file.name, "-e", invalid_expression] - with pytest.raises(FileNotFoundError, match=f"File '{invalid_expression}' does not exist."): + with pytest.raises( + FileNotFoundError, match=f"File '{invalid_expression}' does not exist." + ): main(arg_list) + def test_main_f_valid(mocker): - with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file, tempfile.NamedTemporaryFile(suffix=".txt") as expression_file: + with tempfile.NamedTemporaryFile( + suffix=".osm" + ) as tmp_file, tempfile.NamedTemporaryFile(suffix=".txt") as expression_file: arg_list = ["f", tmp_file.name, "-e", expression_file.name] - mock_run_osmium_filter = mocker.patch('scripts.filter_osm.run_osmium_filter') + mock_run_osmium_filter = mocker.patch("scripts.filter_osm.run_osmium_filter") main(arg_list) mock_run_osmium_filter.assert_called_once_with(arg_list[1], arg_list[3], False) + def test_main_f_valid_R(mocker): - with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file, tempfile.NamedTemporaryFile(suffix=".txt") as expression_file: + with tempfile.NamedTemporaryFile( + suffix=".osm" + ) as tmp_file, tempfile.NamedTemporaryFile(suffix=".txt") as expression_file: arg_list = ["f", tmp_file.name, "-e", expression_file.name, "-R"] - mock_run_osmium_filter = mocker.patch('scripts.filter_osm.run_osmium_filter') + mock_run_osmium_filter = mocker.patch("scripts.filter_osm.run_osmium_filter") main(arg_list) - mock_run_osmium_filter.assert_called_once_with(arg_list[1], arg_list[3], True) \ No newline at end of file + mock_run_osmium_filter.assert_called_once_with(arg_list[1], arg_list[3], True) From 846c7fd269ec6d680b1b116fe5a0f1386961be7f Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Mon, 9 Sep 2024 11:32:18 +0200 Subject: [PATCH 08/28] test_process_osm.py: reformatted, resolved tests issue with pathlib usage --- python/tests/test_process_osm.py | 172 ++++++++++++++++++++++--------- 1 file changed, 122 insertions(+), 50 deletions(-) diff --git a/python/tests/test_process_osm.py b/python/tests/test_process_osm.py index c2907b7..146331e 100644 --- a/python/tests/test_process_osm.py +++ b/python/tests/test_process_osm.py @@ -1,31 +1,37 @@ +import os import pathlib -import tempfile -import pytest import subprocess -import os -import psycopg2 +import tempfile import xml.etree.ElementTree as ET +import psycopg2 +import pytest + from roadgraphtool.credentials_config import CREDENTIALS as config -from scripts.process_osm import run_osmium_cmd, main, import_osm_to_db +from scripts.filter_osm import InvalidInputError, MissingInputError from scripts.find_bbox import find_min_max -from scripts.filter_osm import MissingInputError, InvalidInputError +from scripts.process_osm import (RESOURCES_DIR, STYLES_DIR, import_osm_to_db, + main, run_osmium_cmd) + @pytest.fixture def mock_subprocess_run(mocker): return mocker.patch("subprocess.run") + @pytest.fixture def mock_os_path_isfile(mocker): return mocker.patch("os.path.isfile") + @pytest.fixture def bounding_box(): parent_dir = pathlib.Path(__file__).parent file_path = str(parent_dir) + "/data/bbox_test.osm" - with open(file_path, 'rb') as f: + with open(file_path, "rb") as f: return f.read() + @pytest.fixture(scope="module") def db_connection(): conn = psycopg2.connect( @@ -33,11 +39,12 @@ def db_connection(): user=config.username, password=config.db_password, host=config.db_host, - port=config.db_server_port + port=config.db_server_port, ) yield conn conn.close() + @pytest.fixture def teardown_db(db_connection, request): def cleanup(): @@ -47,14 +54,17 @@ def cleanup(): cursor.execute("DROP TABLE IF EXISTS mockrelations;") db_connection.commit() cursor.close() + request.addfinalizer(cleanup) + @pytest.fixture def renumber_test_files(): parent_dir = pathlib.Path(__file__).parent input_file = str(parent_dir) + "/data/renumber_test.osm" output_file = str(parent_dir) + "/data/renumber_test_output.osm" - return input_file, output_file + return input_file, output_file + @pytest.fixture def sort_test_files(): @@ -63,29 +73,33 @@ def sort_test_files(): output_file = str(parent_dir) + "/data/sort_test_output.osm" return input_file, output_file + def is_renumbered_by_id(content, obj_type): """Function to check if the obj_type IDs are renumbered in ascending order""" root = ET.fromstring(content) ids = [] for object in root.findall(obj_type): - obj_id = int(object.get('id')) + obj_id = int(object.get("id")) ids.append(obj_id) expected_ids = list(range(1, len(ids) + 1)) return ids == expected_ids + def is_sorted_by_id(content, obj_type): """Function to check if the obj_type IDs are sorted in ascending order""" root = ET.fromstring(content) ids = [] for object in root.findall(obj_type): - obj_id = int(object.get('id')) + obj_id = int(object.get("id")) ids.append(obj_id) return ids == sorted(ids) + # TESTS: + def test_find_mix_max(bounding_box): min_lon, min_lat, max_lon, max_lat = find_min_max(bounding_box) assert min_lon == 15.0 @@ -93,150 +107,208 @@ def test_find_mix_max(bounding_box): assert max_lon == 30.0 assert max_lat == 15.0 + @pytest.mark.usefixtures("teardown_db") def test_run_osm2pgsql_cmd(db_connection): parent_dir = pathlib.Path(__file__).parent style_file_path = str(parent_dir) + "/data/mock_default.lua" - input_file = str(parent_dir) + "/data/test.osm" + input_file = str(parent_dir) + "/data/bbox_test.osm" db_username = config.username db_host = config.db_host db_name = config.db_name db_server_port = config.db_server_port - command = ["osm2pgsql", "-d", db_name, "-U", db_username, "-H", db_host, "-P", str(db_server_port), - "--output=flex", "-S", style_file_path, input_file, "-x"] + command = [ + "osm2pgsql", + "-d", + db_name, + "-U", + db_username, + "-H", + db_host, + "-P", + str(db_server_port), + "--output=flex", + "-S", + style_file_path, + input_file, + "-x", + ] subprocess.run(command, check=True) cursor = db_connection.cursor() - cursor.execute('SELECT COUNT(*) FROM mocknodes;') + cursor.execute("SELECT COUNT(*) FROM mocknodes;") nodes_count = cursor.fetchone()[0] assert nodes_count == 6 - cursor.execute('SELECT COUNT(*) FROM mockways;') + cursor.execute("SELECT COUNT(*) FROM mockways;") ways_count = cursor.fetchone()[0] assert ways_count == 0 - cursor.execute('SELECT COUNT(*) FROM mockrelations;') + cursor.execute("SELECT COUNT(*) FROM mockrelations;") relations_count = cursor.fetchone()[0] assert relations_count == 1 - cursor.execute('SELECT * FROM mocknodes WHERE node_id=1;') + cursor.execute("SELECT * FROM mocknodes WHERE node_id=1;") node = cursor.fetchone() assert node is not None - cursor.execute('SELECT * FROM mocknodes WHERE node_id=7;') + cursor.execute("SELECT * FROM mocknodes WHERE node_id=7;") node = cursor.fetchone() assert node is None cursor.close() + def test_run_osmium_cmd_renumber(renumber_test_files): input_file, output_file = renumber_test_files - run_osmium_cmd('r', input_file, output_file) + run_osmium_cmd("r", input_file, output_file) assert os.path.exists(output_file) - with open(output_file, 'r') as f: + with open(output_file, "r") as f: content = f.read() - assert is_renumbered_by_id(content, 'node') == True - assert is_renumbered_by_id(content, 'way') == True - assert is_renumbered_by_id(content, 'relation') == True + assert is_renumbered_by_id(content, "node") == True + assert is_renumbered_by_id(content, "way") == True + assert is_renumbered_by_id(content, "relation") == True os.remove(output_file) + def test_run_osmium_cmd_sort(sort_test_files): input_file, output_file = sort_test_files - run_osmium_cmd('s', str(input_file), str(output_file)) + run_osmium_cmd("s", str(input_file), str(output_file)) assert os.path.exists(output_file) - with open(output_file, 'r') as f: + with open(output_file, "r") as f: content = f.read() - assert is_sorted_by_id(content, 'node') == True - assert is_sorted_by_id(content, 'way') == True - assert is_sorted_by_id(content, 'relation') == True + assert is_sorted_by_id(content, "node") == True + assert is_sorted_by_id(content, "way") == True + assert is_sorted_by_id(content, "relation") == True os.remove(output_file) + def test_import_to_db_valid(mocker): - mocker.patch('os.path.exists', side_effect=lambda path: path == "resources/to_import.osm") - mock_run_osm2pgsql_cmd = mocker.patch('scripts.process_osm.run_osm2pgsql_cmd') + mocker.patch( + "os.path.exists", side_effect=lambda path: path.endswith("to_import.osm") + ) + mock_run_osm2pgsql_cmd = mocker.patch("scripts.process_osm.run_osm2pgsql_cmd") import_osm_to_db() - mock_run_osm2pgsql_cmd.assert_called_once_with(config, 'resources/to_import.osm', 'resources/lua_styles/default.lua') + mock_run_osm2pgsql_cmd.assert_called_once_with( + config, str(RESOURCES_DIR / "to_import.osm"), str(STYLES_DIR / "pipeline.lua") + ) + def test_main_invalid_inputfile(): arg_list = ["d", "invalid_file.osm"] - with pytest.raises(FileNotFoundError, match="File 'invalid_file.osm' does not exist."): + with pytest.raises( + FileNotFoundError, match="File 'invalid_file.osm' does not exist." + ): main(arg_list) + def test_invalid_inputfile_extension(): with tempfile.NamedTemporaryFile(suffix=".txt") as tmp_file: arg_list = ["d", tmp_file.name] - with pytest.raises(InvalidInputError, match="File must have one of the following extensions: osm, osm.pbf, osm.bz2"): + with pytest.raises( + InvalidInputError, + match="File must have one of the following extensions: osm, osm.pbf, osm.bz2", + ): main(arg_list) + @pytest.mark.parametrize("test_input", ["d", "i", "ie"]) def test_main_diie_valid(mocker, test_input): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = [test_input, tmp_file.name] - mock_run_osmium_cmd = mocker.patch('scripts.process_osm.run_osmium_cmd') + mock_run_osmium_cmd = mocker.patch("scripts.process_osm.run_osmium_cmd") main(arg_list) mock_run_osmium_cmd.assert_called_once_with(arg_list[0], arg_list[1]) + @pytest.mark.parametrize("test_input", ["s", "r", "sr"]) def test_main_srsr_invalid(test_input): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = [test_input, tmp_file.name] - with pytest.raises(MissingInputError, match="An output file must be specified with '-o' tag."): + with pytest.raises( + MissingInputError, match="An output file must be specified with '-o' tag." + ): main(arg_list) + @pytest.mark.parametrize("test_input", ["s", "r", "sr"]) def test_main_srsr_valid(mocker, test_input): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = [test_input, tmp_file.name, "-o", " output.osm"] - mock_run_osmium_cmd = mocker.patch('scripts.process_osm.run_osmium_cmd') + mock_run_osmium_cmd = mocker.patch("scripts.process_osm.run_osmium_cmd") main(arg_list) - mock_run_osmium_cmd.assert_called_once_with(arg_list[0], arg_list[1], arg_list[3]) + mock_run_osmium_cmd.assert_called_once_with( + arg_list[0], arg_list[1], arg_list[3] + ) + def test_main_default_style_valid(mocker): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["u", tmp_file.name] - mock_run_osm2pgsql_cmd = mocker.patch('scripts.process_osm.run_osm2pgsql_cmd') + mock_run_osm2pgsql_cmd = mocker.patch("scripts.process_osm.run_osm2pgsql_cmd") main(arg_list) - mock_run_osm2pgsql_cmd.assert_called_once_with(config, arg_list[1], "resources/lua_styles/default.lua") + mock_run_osm2pgsql_cmd.assert_called_once_with( + config, arg_list[1], str(STYLES_DIR / "default.lua") + ) + def test_main_input_style_valid(mocker): - with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_input, tempfile.NamedTemporaryFile(suffix=".lua") as tmp_lua: + with tempfile.NamedTemporaryFile( + suffix=".osm" + ) as tmp_input, tempfile.NamedTemporaryFile(suffix=".lua") as tmp_lua: arg_list = ["u", tmp_input.name, "-l", tmp_lua.name] - mock_run_osm2pgsql_cmd = mocker.patch('scripts.process_osm.run_osm2pgsql_cmd') + mock_run_osm2pgsql_cmd = mocker.patch("scripts.process_osm.run_osm2pgsql_cmd") main(arg_list) mock_run_osm2pgsql_cmd.assert_called_once_with(config, arg_list[1], arg_list[3]) + def test_main_style_file_invalid(): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["u", tmp_file.name, "-l", "invalid_style.lua"] - with pytest.raises(FileNotFoundError, match="File 'invalid_style.lua' does not exist."): + with pytest.raises( + FileNotFoundError, match="File 'invalid_style.lua' does not exist." + ): main(arg_list) + def test_main_style_file_extension_invalid(): - with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file, tempfile.NamedTemporaryFile(suffix=".txt") as invalid_lua: + with tempfile.NamedTemporaryFile( + suffix=".osm" + ) as tmp_file, tempfile.NamedTemporaryFile(suffix=".txt") as invalid_lua: arg_list = ["u", tmp_file.name, "-l", invalid_lua.name] - with pytest.raises(InvalidInputError, match="File must have the '.lua' extension."): + with pytest.raises( + InvalidInputError, match="File must have the '.lua' extension." + ): main(arg_list) + def test_main_bbox_valid(mocker): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["b", tmp_file.name, "-id", "1234"] - mock_extract_bbox = mocker.patch('scripts.process_osm.extract_bbox', return_value=(10, 20, 30, 40)) - mock_run_osm2pgsql_cmd = mocker.patch('scripts.process_osm.run_osm2pgsql_cmd') + mock_extract_bbox = mocker.patch( + "scripts.process_osm.extract_bbox", return_value=(10, 20, 30, 40) + ) + mock_run_osm2pgsql_cmd = mocker.patch("scripts.process_osm.run_osm2pgsql_cmd") main(arg_list) mock_extract_bbox.assert_called_once_with(arg_list[3]) - mock_run_osm2pgsql_cmd.assert_called_once_with(config, arg_list[1], "resources/lua_styles/default.lua", "10,20,30,40") + mock_run_osm2pgsql_cmd.assert_called_once_with( + config, arg_list[1], str(STYLES_DIR / "default.lua"), "10,20,30,40" + ) + # relation_id missing def test_main_bbox_id_missing(): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["b", tmp_file.name] - with pytest.raises(MissingInputError, match="Existing relation ID must be specified."): - main(arg_list) \ No newline at end of file + with pytest.raises( + MissingInputError, match="Existing relation ID must be specified." + ): + main(arg_list) From a6c147984bbfb1574ec50e2d3e1152c6d0684e4f Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Thu, 19 Sep 2024 16:06:32 +0300 Subject: [PATCH 09/28] Fixed insert_area.py, process_osm.py --- python/roadgraphtool/insert_area.py | 2 +- python/scripts/main.py | 15 ++++++--------- python/scripts/process_osm.py | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/python/roadgraphtool/insert_area.py b/python/roadgraphtool/insert_area.py index c3d97c9..565ba03 100644 --- a/python/roadgraphtool/insert_area.py +++ b/python/roadgraphtool/insert_area.py @@ -2,7 +2,7 @@ import json import sys -from .db import db +from roadgraphtool.db import db def insert_area(id: int | None, name: str, description: str | None, geom: dict): diff --git a/python/scripts/main.py b/python/scripts/main.py index cd59cff..3074eae 100644 --- a/python/scripts/main.py +++ b/python/scripts/main.py @@ -41,7 +41,6 @@ def configure_arg_parser() -> argparse.ArgumentParser: "-i", "--import", dest="importing", - action="store_true", help="Import OSM data to database specified in config.ini", required=False, ) @@ -73,12 +72,12 @@ def main(arg_list: list[str] | None = None): area_srid = args.area_srid fill_speed = args.fill_speed - insert_area( - area_id, - "Insertion_area", - "Description of the area", - read_area_file("testarea.json"), - ) + # insert_area( + # area_id, + # "Insertion_area", + # "Description of the area", + # read_area_file("testarea.json"), + # ) # logging.info("selecting nodes") # nodes = select_network_nodes_in_area(area_id) @@ -92,8 +91,6 @@ def main(arg_list: list[str] | None = None): # compute_strong_components(area_id) # logging.info("storing the results in the component_data table") - # insert_area("test1", []) - # area = get_area_for_demand( # 4326, # [1, 2, 3], diff --git a/python/scripts/process_osm.py b/python/scripts/process_osm.py index 0ca4360..b73b13b 100644 --- a/python/scripts/process_osm.py +++ b/python/scripts/process_osm.py @@ -136,8 +136,8 @@ def import_osm_to_db(filename: str | None = None, style_filename: str = "pipelin for file in input_files: if os.path.exists(file) and is_valid_extension(file): input_file = file - elif os.path.exists(RESOURCES_DIR / filename): - input_file = str(RESOURCES_DIR / filename) + elif os.path.exists(filename): + input_file = filename if not input_file: raise FileNotFoundError("There is no valid file to import.") From a822faa1dae449339c37310c473264521779f66a Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Thu, 19 Sep 2024 17:21:55 +0300 Subject: [PATCH 10/28] Added test data for integration tests --- python/tests/data/integration_area.json | 29 +++++++++++++++++++ python/tests/data/integration_test.osm | 37 +++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 python/tests/data/integration_area.json create mode 100644 python/tests/data/integration_test.osm diff --git a/python/tests/data/integration_area.json b/python/tests/data/integration_area.json new file mode 100644 index 0000000..88e08ea --- /dev/null +++ b/python/tests/data/integration_area.json @@ -0,0 +1,29 @@ +{ + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + 13.403920115445864, + 52.51912951075764 + ], + [ + 13.404591964637179, + 52.518789787533855 + ], + [ + 13.409067805024401, + 52.51991834946755 + ], + [ + 13.407071182779786, + 52.5223884170219 + ], + [ + 13.403920115445864, + 52.51912951075764 + ] + ] + ] + ] +} diff --git a/python/tests/data/integration_test.osm b/python/tests/data/integration_test.osm new file mode 100644 index 0000000..e2b9879 --- /dev/null +++ b/python/tests/data/integration_test.osm @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 2c1e8299cd443ac426baebd4403e7a53824eba0d Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Thu, 19 Sep 2024 17:37:02 +0300 Subject: [PATCH 11/28] Updated test data to support contraction --- python/tests/data/integration_test.osm | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/python/tests/data/integration_test.osm b/python/tests/data/integration_test.osm index e2b9879..d38f753 100644 --- a/python/tests/data/integration_test.osm +++ b/python/tests/data/integration_test.osm @@ -3,14 +3,20 @@ - - - - + + + + + + + + + + From c208cddcda3e3bf7288273a60b53fd8bb6d98acd Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Fri, 20 Sep 2024 13:27:18 +0300 Subject: [PATCH 12/28] Added integration tests --- python/tests/integrations_test.py | 67 +++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 python/tests/integrations_test.py diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py new file mode 100644 index 0000000..58ce62c --- /dev/null +++ b/python/tests/integrations_test.py @@ -0,0 +1,67 @@ +from pathlib import Path + +import pytest + +from roadgraphtool.db import db +from roadgraphtool.db_operations import (compute_strong_components, + contract_graph_in_area) +from roadgraphtool.insert_area import insert_area +from roadgraphtool.insert_area import read_json_file as read_area +from scripts.install_sql import main as pre_pocessing +from scripts.process_osm import import_osm_to_db + +TEST_DATA_PATH = Path(__file__).parent / "data" + + +@pytest.fixture(scope="module") +def setup(): + pre_pocessing() + + import_osm_to_db(str(TEST_DATA_PATH / "integration_test.osm")) + + insert_area( + 1, + "Deutschland", + "test area", + read_area(str(TEST_DATA_PATH / "integration_area.json")), + ) + + +def test_integration_contraction(setup): + contract_graph_in_area(1, 4326, fill_speed=False) + + contracted_nodes = db.execute_sql_and_fetch_all_rows( + "SELECT id FROM nodes WHERE contracted;" + ) + assert all(num in [(6,), (7,), (8,)] for num in contracted_nodes) + + edges = db.execute_sql_and_fetch_all_rows('SELECT "from", "to" FROM edges;') + + assert all( + pair in [(3, 4), (2, 1), (1, 2), (5, 3), (5, 2), (4, 3), (2, 3), (3, 2)] + for pair in edges + ) + + +def test_integration_strongly_connected_components(setup): + compute_strong_components(1) + + component_data = db.execute_sql_and_fetch_all_rows( + "SELECT component_id, node_id FROM component_data" + ) + + assert all( + pair in [(0, 1), (0, 2), (0, 3), (0, 4), (1, 5)] for pair in component_data + ) + + +# The flow is +# install_sql.py - import main as "pre_processing" +# import_osm_to_db, post-processing +# insert_area +# Graph operations: +# - contraction of the graph +# - computing strongly connected components + +# QAs: +# Is there a way in our python project to remove everything from db From a19b884362ad62d074b3cedaf93663959171e7a1 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Fri, 20 Sep 2024 13:41:28 +0300 Subject: [PATCH 13/28] Test checks if the pre-processing was already done --- python/tests/integrations_test.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py index 58ce62c..eb705e7 100644 --- a/python/tests/integrations_test.py +++ b/python/tests/integrations_test.py @@ -13,9 +13,19 @@ TEST_DATA_PATH = Path(__file__).parent / "data" +def check_setup_of_database(): + query = """SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'areas' +);""" + return db.execute_count_query(query) + + @pytest.fixture(scope="module") def setup(): - pre_pocessing() + if not check_setup_of_database(): + pre_pocessing() import_osm_to_db(str(TEST_DATA_PATH / "integration_test.osm")) From c79afaf7eab36dfb2391f2f6764f9b734616cd4f Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Fri, 20 Sep 2024 14:25:05 +0300 Subject: [PATCH 14/28] Removed dev notes --- python/tests/integrations_test.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py index eb705e7..601d9f9 100644 --- a/python/tests/integrations_test.py +++ b/python/tests/integrations_test.py @@ -63,15 +63,3 @@ def test_integration_strongly_connected_components(setup): assert all( pair in [(0, 1), (0, 2), (0, 3), (0, 4), (1, 5)] for pair in component_data ) - - -# The flow is -# install_sql.py - import main as "pre_processing" -# import_osm_to_db, post-processing -# insert_area -# Graph operations: -# - contraction of the graph -# - computing strongly connected components - -# QAs: -# Is there a way in our python project to remove everything from db From 16048dbdbdd5bab3df63cdb9e7a740e0b9b3cfaa Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Fri, 20 Sep 2024 14:35:18 +0300 Subject: [PATCH 15/28] Updated assertion method --- python/tests/integrations_test.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py index 601d9f9..93a643b 100644 --- a/python/tests/integrations_test.py +++ b/python/tests/integrations_test.py @@ -43,13 +43,12 @@ def test_integration_contraction(setup): contracted_nodes = db.execute_sql_and_fetch_all_rows( "SELECT id FROM nodes WHERE contracted;" ) - assert all(num in [(6,), (7,), (8,)] for num in contracted_nodes) + assert set([(6,), (7,), (8,)]) == set(contracted_nodes) edges = db.execute_sql_and_fetch_all_rows('SELECT "from", "to" FROM edges;') - assert all( - pair in [(3, 4), (2, 1), (1, 2), (5, 3), (5, 2), (4, 3), (2, 3), (3, 2)] - for pair in edges + assert set([(3, 4), (2, 1), (1, 2), (5, 3), (5, 2), (4, 3), (2, 3), (3, 2)]) == set( + edges ) @@ -60,6 +59,4 @@ def test_integration_strongly_connected_components(setup): "SELECT component_id, node_id FROM component_data" ) - assert all( - pair in [(0, 1), (0, 2), (0, 3), (0, 4), (1, 5)] for pair in component_data - ) + assert set([(0, 1), (0, 2), (0, 3), (0, 4), (1, 5)]) == set(component_data) From b5c6ee389f98ef3a808e7ca47bc98f8fc869ef6e Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Sat, 21 Sep 2024 16:06:26 +0300 Subject: [PATCH 16/28] Updated integration tests to run in test_env of db --- SQL/after_import.sql | 32 +++++++++++---------- SQL/main.sql | 8 +++--- SQL/testing_extension.sql | 8 +++--- python/roadgraphtool/db.py | 4 ++- python/scripts/process_osm.py | 46 +++++++++++++++++++++---------- python/tests/integrations_test.py | 13 ++++++++- python/tests/test_process_osm.py | 5 +++- 7 files changed, 76 insertions(+), 40 deletions(-) diff --git a/SQL/after_import.sql b/SQL/after_import.sql index dd3fff5..f7f3ffa 100644 --- a/SQL/after_import.sql +++ b/SQL/after_import.sql @@ -1,65 +1,69 @@ -\echo 'Altering column nodes.id to integer...' +DO $$ +BEGIN +RAISE NOTICE 'Altering column nodes.id to integer...'; ALTER TABLE nodes ALTER COLUMN id TYPE integer; -\echo 'Altering column ways.id to integer...' +RAISE NOTICE 'Altering column ways.id to integer...'; ALTER TABLE ways ALTER COLUMN id TYPE integer; -\echo 'Altering column relations.id to integer...' +RAISE NOTICE 'Altering column relations.id to integer...'; ALTER TABLE relations ALTER COLUMN id TYPE integer; -\echo 'Altering column nodes_ways.id to integer...' +RAISE NOTICE 'Altering column nodes_ways.id to integer...'; ALTER TABLE nodes_ways ALTER COLUMN way_id TYPE integer; -- PRIMARY KEYS -\echo 'Adding PRIMARY KEY constraints to table nodes...' +RAISE NOTICE 'Adding PRIMARY KEY constraints to table nodes...'; ALTER TABLE nodes ADD CONSTRAINT pk_nodes PRIMARY KEY (id); -\echo 'Adding PRIMARY KEY constraints to table nodes_ways' +RAISE NOTICE 'Adding PRIMARY KEY constraints to table nodes_ways'; ALTER TABLE nodes_ways ADD CONSTRAINT nodes_ways_pk PRIMARY KEY (id); -\echo 'Adding PRIMARY KEY constraints to table ways...' +RAISE NOTICE 'Adding PRIMARY KEY constraints to table ways...'; ALTER TABLE ways ADD CONSTRAINT pk_ways PRIMARY KEY (id); -\echo 'Adding PRIMARY KEY constraints to table relations...' +RAISE NOTICE 'Adding PRIMARY KEY constraints to table relations...'; ALTER TABLE relations ADD CONSTRAINT pk_relations PRIMARY KEY (id); -- FOREIGN KEYS -- ways table foreign keys -\echo 'Adding FOREIGN KEY constraints to table ways...' +RAISE NOTICE 'Adding FOREIGN KEY constraints to table ways...'; ALTER TABLE ways ADD CONSTRAINT fk_ways_from FOREIGN KEY ("from") REFERENCES nodes(id); ALTER TABLE ways ADD CONSTRAINT fk_ways_to FOREIGN KEY ("to") REFERENCES nodes(id); ALTER TABLE ways ADD CONSTRAINT fk_ways_area FOREIGN KEY (area) REFERENCES areas(id); -- nodes table foreign key -\echo 'Adding FOREIGN KEY constraints to table nodes...' +RAISE NOTICE 'Adding FOREIGN KEY constraints to table nodes...'; ALTER TABLE nodes ADD CONSTRAINT fk_nodes_area FOREIGN KEY (area) REFERENCES areas(id); -- edges table foreign keys -\echo 'Adding FOREIGN KEY constraints to table edges...' +RAISE NOTICE 'Adding FOREIGN KEY constraints to table edges...'; ALTER TABLE edges ADD CONSTRAINT fk_edges_from FOREIGN KEY ("from") REFERENCES nodes(id); ALTER TABLE edges ADD CONSTRAINT fk_edges_to FOREIGN KEY ("to") REFERENCES nodes(id); -- trip_locations table foreign keys -\echo 'Adding FOREIGN KEY constraints to table trip_location...' +RAISE NOTICE 'Adding FOREIGN KEY constraints to table trip_location...'; ALTER TABLE trip_locations ADD CONSTRAINT fk_trip_locations_destination FOREIGN KEY (destination) REFERENCES nodes(id); ALTER TABLE trip_locations ADD CONSTRAINT fk_trip_locations_origin FOREIGN KEY (origin) REFERENCES nodes(id); -- nodes_ways table foreign key -\echo 'Adding FOREIGN KEY constraints to table nodes_ways...' +RAISE NOTICE 'Adding FOREIGN KEY constraints to table nodes_ways...'; ALTER TABLE nodes_ways ADD CONSTRAINT fk_nodes_ways_area FOREIGN KEY (area) REFERENCES areas(id); -- nodes_ways_speeds table foreign keys -\echo 'Adding FOREIGN KEY constraints to table nodes_ways_speeds...' +RAISE NOTICE 'Adding FOREIGN KEY constraints to table nodes_ways_speeds...'; ALTER TABLE nodes_ways_speeds ADD CONSTRAINT fk_nodes_ways_speeds_from FOREIGN KEY (from_node_ways_id) REFERENCES nodes_ways(id); ALTER TABLE nodes_ways_speeds ADD CONSTRAINT fk_nodes_ways_speeds_to FOREIGN KEY (to_node_ways_id) REFERENCES nodes_ways(id); +END +$$; diff --git a/SQL/main.sql b/SQL/main.sql index 6615a52..98849f6 100644 --- a/SQL/main.sql +++ b/SQL/main.sql @@ -155,7 +155,7 @@ CREATE TABLE IF NOT EXISTS public.demand ( -- CREATE TABLE IF NOT EXISTS public.nodes ( - id bigint NOT NULL, + id integer NOT NULL, geom public.geometry(Point,4326) NOT NULL, area integer, contracted boolean DEFAULT false NOT NULL @@ -523,12 +523,12 @@ CREATE TABLE IF NOT EXISTS public.trip_times ( -- CREATE TABLE IF NOT EXISTS public.ways ( - id bigint NOT NULL, + id integer NOT NULL, tags public.hstore, geom public.geometry(Geometry,4326) NOT NULL, area integer, - "from" bigint NOT NULL, - "to" bigint NOT NULL, + "from" integer NOT NULL, + "to" integer NOT NULL, oneway boolean NOT NULL ); diff --git a/SQL/testing_extension.sql b/SQL/testing_extension.sql index 6728d96..476d469 100644 --- a/SQL/testing_extension.sql +++ b/SQL/testing_extension.sql @@ -262,25 +262,25 @@ BEGIN -- drop every sequence in test_scheme_name FOR tmp IN (SELECT sequence_name FROM information_schema.sequences WHERE sequence_schema = test_scheme_name) LOOP - EXECUTE format('DROP SEQUENCE %I.%I', test_scheme_name, tmp); + EXECUTE format('DROP SEQUENCE %I.%I CASCADE', test_scheme_name, tmp); END LOOP; -- drop every view in test_scheme_name FOR table_name_i IN (SELECT table_name FROM information_schema.views WHERE table_schema = test_scheme_name) LOOP - EXECUTE format('DROP VIEW %I.%I', test_scheme_name, table_name_i); + EXECUTE format('DROP VIEW %I.%I CASCADE', test_scheme_name, table_name_i); END LOOP; -- drop every table in test_scheme_name FOR table_name_i IN (SELECT table_name FROM information_schema.tables WHERE table_schema = test_scheme_name) LOOP - EXECUTE format('DROP TABLE %I.%I', test_scheme_name, table_name_i); + EXECUTE format('DROP TABLE %I.%I CASCADE', test_scheme_name, table_name_i); END LOOP; -- drop routines in test_scheme_name FOR table_name_i IN (SELECT routine_name FROM information_schema.routines WHERE routine_schema = test_scheme_name) LOOP - EXECUTE format('DROP FUNCTION %I.%I', test_scheme_name, table_name_i); + EXECUTE format('DROP FUNCTION %I.%I CASCADE', test_scheme_name, table_name_i); END LOOP; -- update search path diff --git a/python/roadgraphtool/db.py b/python/roadgraphtool/db.py index 8af93fd..4578c20 100644 --- a/python/roadgraphtool/db.py +++ b/python/roadgraphtool/db.py @@ -179,7 +179,9 @@ def execute_script(self, script_path: Path) -> int: cursor.execute(script) self._psycopg2_connection.commit() except Exception as e: - logging.error(f"Error executing script {script_path}: {e}") + logging.error( + f"Error executing script {script_path}: {e}. Search_path: {self.execute_sql_and_fetch_all_rows('SHOW search_path;')}" + ) self._psycopg2_connection.rollback() retcode = 1 finally: diff --git a/python/scripts/process_osm.py b/python/scripts/process_osm.py index b73b13b..c4a20f1 100644 --- a/python/scripts/process_osm.py +++ b/python/scripts/process_osm.py @@ -6,6 +6,7 @@ import roadgraphtool.log from roadgraphtool.credentials_config import CREDENTIALS as config from roadgraphtool.credentials_config import CredentialsConfig +from roadgraphtool.db import db from scripts.filter_osm import (InvalidInputError, MissingInputError, is_valid_extension, load_multipolygon_by_id) from scripts.find_bbox import find_min_max @@ -51,6 +52,7 @@ def run_osm2pgsql_cmd( input_file: str, style_file_path: str, coords: str | list[int] | None = None, + schema: str | None = None, ) -> int: """Function to run osm2pgsl command. Returns return code of the subprocess.""" @@ -73,10 +75,12 @@ def run_osm2pgsql_cmd( ] if coords: command.extend(["-b", coords]) + if schema: + command.extend(["--schema", schema]) return subprocess.run(command).returncode -def post_process_osm_import(style_filename: str) -> int: +def post_process_osm_import(style_filename: str, schema: str | None = None) -> int: post_proc_dict = {"pipeline.lua": "after_import.sql"} if not post_proc_dict[style_filename]: @@ -86,21 +90,29 @@ def post_process_osm_import(style_filename: str) -> int: sql_filepath = SQL_DIR / post_proc_dict[style_filename] logging.info("Post-processing OSM import...") - retcode = subprocess.run( + # retcode = db.execute_script(sql_filepath) + command = [ + "psql", + "-d", + config.db_name, + "-U", + config.username, + "-h", + config.host, + "-p", + str(config.db_server_port), + ] + if schema: + print(f"\n\nSEARCHPATH = {schema}") + command.extend(["-c", f"SET search_path TO {schema};"]) + command.extend( [ - "psql", - "-d", - config.db_name, - "-U", - config.username, - "-p", - str(config.db_server_port), - "-h", - config.db_host, "-f", sql_filepath, ] - ).returncode + ) + retcode = subprocess.run(command).returncode + if retcode != 0: logging.error(f"Error during post-processing. Return code: {retcode}") return retcode @@ -109,7 +121,11 @@ def post_process_osm_import(style_filename: str) -> int: return 0 -def import_osm_to_db(filename: str | None = None, style_filename: str = "pipeline.lua"): +def import_osm_to_db( + filename: str | None = None, + style_filename: str = "pipeline.lua", + schema: str | None = None, +) -> int: """Function to import OSM file specified in config.ini file to database. The function expects the OSM file to be saved as resources/to_import.*. The pipeline.lua style file is used as default style. @@ -145,8 +161,8 @@ def import_osm_to_db(filename: str | None = None, style_filename: str = "pipelin style_file_path = str(STYLES_DIR / style_filename) return run_osm2pgsql_cmd( - config, input_file, style_file_path - ) or post_process_osm_import(style_filename) + config, input_file, style_file_path, schema=schema + ) or post_process_osm_import(style_filename, schema=schema) # Main flow of the current file, including functions used only within this file diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py index 93a643b..b365929 100644 --- a/python/tests/integrations_test.py +++ b/python/tests/integrations_test.py @@ -27,7 +27,10 @@ def setup(): if not check_setup_of_database(): pre_pocessing() - import_osm_to_db(str(TEST_DATA_PATH / "integration_test.osm")) + # change the main schema + db.execute_sql("CALL test_env_constructor();") + + import_osm_to_db(str(TEST_DATA_PATH / "integration_test.osm"), schema="test_env") insert_area( 1, @@ -37,6 +40,14 @@ def setup(): ) +@pytest.fixture(scope="module", autouse=True) +def destructor(): + yield # to act as a destructor + + # test environment desctructor + db.execute_sql("CALL test_env_destructor();") + + def test_integration_contraction(setup): contract_graph_in_area(1, 4326, fill_speed=False) diff --git a/python/tests/test_process_osm.py b/python/tests/test_process_osm.py index 146331e..c5034e5 100644 --- a/python/tests/test_process_osm.py +++ b/python/tests/test_process_osm.py @@ -198,7 +198,10 @@ def test_import_to_db_valid(mocker): mock_run_osm2pgsql_cmd = mocker.patch("scripts.process_osm.run_osm2pgsql_cmd") import_osm_to_db() mock_run_osm2pgsql_cmd.assert_called_once_with( - config, str(RESOURCES_DIR / "to_import.osm"), str(STYLES_DIR / "pipeline.lua") + config, + str(RESOURCES_DIR / "to_import.osm"), + str(STYLES_DIR / "pipeline.lua"), + schema=None, ) From 5ffb4a27273f77b3ebe1617d0ecbd86dc2210135 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Sat, 21 Sep 2024 16:09:36 +0300 Subject: [PATCH 17/28] Removed debugging lines --- python/roadgraphtool/db.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/roadgraphtool/db.py b/python/roadgraphtool/db.py index 4578c20..8af93fd 100644 --- a/python/roadgraphtool/db.py +++ b/python/roadgraphtool/db.py @@ -179,9 +179,7 @@ def execute_script(self, script_path: Path) -> int: cursor.execute(script) self._psycopg2_connection.commit() except Exception as e: - logging.error( - f"Error executing script {script_path}: {e}. Search_path: {self.execute_sql_and_fetch_all_rows('SHOW search_path;')}" - ) + logging.error(f"Error executing script {script_path}: {e}") self._psycopg2_connection.rollback() retcode = 1 finally: From e4f9548b8ec70df1cbac0e155f8de2ac321ec017 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Sun, 22 Sep 2024 15:10:04 +0300 Subject: [PATCH 18/28] Added import state test --- python/tests/integrations_test.py | 132 ++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py index b365929..448aef9 100644 --- a/python/tests/integrations_test.py +++ b/python/tests/integrations_test.py @@ -1,3 +1,4 @@ +import xml.etree.ElementTree as ET from pathlib import Path import pytest @@ -12,6 +13,8 @@ TEST_DATA_PATH = Path(__file__).parent / "data" +# Helper functions + def check_setup_of_database(): query = """SELECT EXISTS ( @@ -22,6 +25,63 @@ def check_setup_of_database(): return db.execute_count_query(query) +def parse_osm_file(file_path): + # Parse the XML file + tree = ET.parse(file_path) + root = tree.getroot() + + # Initialize the dictionary to store the OSM data + osm_data = {"nodes": {}, "ways": {}} + + # Parse nodes + for node in root.findall("node"): + node_id = node.get("id") + node_data = { + "id": node_id, + "lat": node.get("lat"), + "lon": node.get("lon"), + "version": node.get("version"), + "timestamp": node.get("timestamp"), + "uid": node.get("uid"), + "user": node.get("user"), + "tags": {}, + } + + # Parse tags for nodes + for tag in node.findall("tag"): + node_data["tags"][tag.get("k")] = tag.get("v") + + osm_data["nodes"][node_id] = node_data + + # Parse ways + for way in root.findall("way"): + way_id = way.get("id") + way_data = { + "id": way_id, + "version": way.get("version"), + "timestamp": way.get("timestamp"), + "uid": way.get("uid"), + "user": way.get("user"), + "nodes": [], + "tags": {}, + } + + # Parse node references for ways + for nd in way.findall("nd"): + way_data["nodes"].append(nd.get("ref")) + + # Parse tags for ways + for tag in way.findall("tag"): + way_data["tags"][tag.get("k")] = tag.get("v") + + osm_data["ways"][way_id] = way_data + + return osm_data + + +# Testing functions + + @pytest.fixture(scope="module") def setup(): if not check_setup_of_database(): @@ -48,6 +108,78 @@ def destructor(): db.execute_sql("CALL test_env_destructor();") +def test_importing(setup): + # read osm file to dictionary + osm_dict = parse_osm_file(TEST_DATA_PATH / "integration_test.osm") + print(osm_dict) + # Test that importing osm data and area was done successfully + nodes = db.execute_sql_and_fetch_all_rows( + "SELECT id, ST_X(geom), ST_Y(geom) FROM nodes;" + ) + nodes_set = set(nodes) + + expected_nodes = osm_dict["nodes"] + expected_nodes_set = set( + [ + (int(key), float(value["lon"]), float(value["lat"])) + for key, value in expected_nodes.items() + ] + ) + + assert nodes_set == expected_nodes_set + + ways = db.execute_sql_and_fetch_all_rows( + 'SELECT id, tags, "from", "to", oneway FROM ways;' + ) + + ways_set = set( + [ + (way[0], str(dict(sorted(way[1].items()))), way[2], way[3], way[4]) + for way in ways + ] + ) + + expected_ways = osm_dict["ways"] + expected_ways_set = set( + [ + ( + int(key), + str( + { + k: v for k, v in value["tags"].items() if k != "oneway" + } # the same tags, but without "oneway" tag + ), + int(value["nodes"][0]), + int(value["nodes"][-1]), + value["tags"]["oneway"] == "yes", + ) + for key, value in expected_ways.items() + ] + ) + + assert expected_ways_set == ways_set + + nodes_ways = db.execute_sql_and_fetch_all_rows( + "SELECT way_id, node_id FROM nodes_ways;" + ) + nodes_ways_set = set(nodes_ways) + + expected_nodes_ways = set() + for way_id, way in expected_ways.items(): + for node_id in way["nodes"]: + expected_nodes_ways.add((int(way_id), int(node_id))) + + assert nodes_ways_set == expected_nodes_ways + + area = db.execute_sql_and_fetch_all_rows( + "SELECT id, name, description FROM areas;" + )[0] + + expected_area = (1, "Deutschland", "test area") + + assert area == expected_area + + def test_integration_contraction(setup): contract_graph_in_area(1, 4326, fill_speed=False) From ee9cd8bd09e01d4577995b5b5e761b965cb37fe8 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Sun, 22 Sep 2024 15:29:20 +0300 Subject: [PATCH 19/28] Removed debugging lines --- python/scripts/process_osm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/scripts/process_osm.py b/python/scripts/process_osm.py index c4a20f1..a49276a 100644 --- a/python/scripts/process_osm.py +++ b/python/scripts/process_osm.py @@ -103,7 +103,6 @@ def post_process_osm_import(style_filename: str, schema: str | None = None) -> i str(config.db_server_port), ] if schema: - print(f"\n\nSEARCHPATH = {schema}") command.extend(["-c", f"SET search_path TO {schema};"]) command.extend( [ From e7ba1da36718dc9fb8cacb7b5e03b3a67f24854b Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Tue, 1 Oct 2024 15:11:47 +0200 Subject: [PATCH 20/28] Updated structure of integration test of the base flow --- python/tests/integrations_test.py | 45 ++++++++++++++++++------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py index 448aef9..ed5f42c 100644 --- a/python/tests/integrations_test.py +++ b/python/tests/integrations_test.py @@ -1,8 +1,6 @@ import xml.etree.ElementTree as ET from pathlib import Path -import pytest - from roadgraphtool.db import db from roadgraphtool.db_operations import (compute_strong_components, contract_graph_in_area) @@ -82,8 +80,21 @@ def parse_osm_file(file_path): # Testing functions -@pytest.fixture(scope="module") -def setup(): +def test_integration_base_flow(): + """ + Integration test: + Base flow: + 1) Import test data: + i) set up basic tables (optionally) + ii) import osm file + iii) add test area + 2) Execute contraction of graph in the area + 3) Execute computation of strong components in the area + 4) Export data to files + 5) Destroy testing environment + """ + + # 1) setting up the database with needed information if not check_setup_of_database(): pre_pocessing() @@ -99,19 +110,10 @@ def setup(): read_area(str(TEST_DATA_PATH / "integration_area.json")), ) - -@pytest.fixture(scope="module", autouse=True) -def destructor(): - yield # to act as a destructor - - # test environment desctructor - db.execute_sql("CALL test_env_destructor();") - - -def test_importing(setup): # read osm file to dictionary osm_dict = parse_osm_file(TEST_DATA_PATH / "integration_test.osm") print(osm_dict) + # Test that importing osm data and area was done successfully nodes = db.execute_sql_and_fetch_all_rows( "SELECT id, ST_X(geom), ST_Y(geom) FROM nodes;" @@ -179,10 +181,10 @@ def test_importing(setup): assert area == expected_area - -def test_integration_contraction(setup): + # 2) 2) Call contraction contract_graph_in_area(1, 4326, fill_speed=False) + # GET updated data from db and assert contracted_nodes = db.execute_sql_and_fetch_all_rows( "SELECT id FROM nodes WHERE contracted;" ) @@ -194,12 +196,19 @@ def test_integration_contraction(setup): edges ) - -def test_integration_strongly_connected_components(setup): + # 3) Call Compute Strong Components compute_strong_components(1) + # GET updated data from db and assert component_data = db.execute_sql_and_fetch_all_rows( "SELECT component_id, node_id FROM component_data" ) assert set([(0, 1), (0, 2), (0, 3), (0, 4), (1, 5)]) == set(component_data) + + # 4) Export data to files + # TODO: export + + # 5) Environment destruction + # test environment desctructor + db.execute_sql("CALL test_env_destructor();") From 2ae800066ba7320b4864ecadf6336e802cd76ea6 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Thu, 3 Oct 2024 15:45:43 +0200 Subject: [PATCH 21/28] Added config to base integration test --- python/tests/integrations_test.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py index ed5f42c..eb9ca2b 100644 --- a/python/tests/integrations_test.py +++ b/python/tests/integrations_test.py @@ -1,3 +1,4 @@ +import os import xml.etree.ElementTree as ET from pathlib import Path @@ -6,6 +7,7 @@ contract_graph_in_area) from roadgraphtool.insert_area import insert_area from roadgraphtool.insert_area import read_json_file as read_area +from roadgraphtool.map import get_map as export_nodes_edges from scripts.install_sql import main as pre_pocessing from scripts.process_osm import import_osm_to_db @@ -181,7 +183,7 @@ def test_integration_base_flow(): assert area == expected_area - # 2) 2) Call contraction + # 2) Call contraction contract_graph_in_area(1, 4326, fill_speed=False) # GET updated data from db and assert @@ -207,7 +209,24 @@ def test_integration_base_flow(): assert set([(0, 1), (0, 2), (0, 3), (0, 4), (1, 5)]) == set(component_data) # 4) Export data to files - # TODO: export + TMP_DIR = Path(__file__).parent / "TMP" + os.mkdir(str(TMP_DIR)) + map_path = TMP_DIR / "map" + area_dir = TMP_DIR / "area_dir" + os.mkdir(str(map_path)) + os.mkdir(str(area_dir)) + + config = { + "map": { + "path": str(map_path), + "SRID_plane": 4326, + }, + "area_dir": str(area_dir), + "area": "Deutschland", + "area_id": 1, + } + _ = export_nodes_edges(config) + assert False # 5) Environment destruction # test environment desctructor From a9364aea0ca296bf589cb6d8a87ef138cab50115 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Wed, 9 Oct 2024 15:02:37 +0200 Subject: [PATCH 22/28] added todos --- python/tests/integrations_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py index eb9ca2b..d342300 100644 --- a/python/tests/integrations_test.py +++ b/python/tests/integrations_test.py @@ -228,6 +228,9 @@ def test_integration_base_flow(): _ = export_nodes_edges(config) assert False + # TODO: assert created files + # 5) Environment destruction # test environment desctructor db.execute_sql("CALL test_env_destructor();") + # TODO: remove tables from public scheme From 7fe71d2efb57e8f8503cc7bc32948825f1fbf3b5 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Wed, 9 Oct 2024 16:03:24 +0200 Subject: [PATCH 23/28] Added assertions of the created files of the base flow --- python/tests/integrations_test.py | 56 +++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py index d342300..05e0d66 100644 --- a/python/tests/integrations_test.py +++ b/python/tests/integrations_test.py @@ -1,7 +1,10 @@ +import hashlib import os import xml.etree.ElementTree as ET from pathlib import Path +import pytest + from roadgraphtool.db import db from roadgraphtool.db_operations import (compute_strong_components, contract_graph_in_area) @@ -13,6 +16,20 @@ TEST_DATA_PATH = Path(__file__).parent / "data" +# Fixtures + + +@pytest.fixture() +def cleanup_for_base_flow(): + yield # wait for the call after test + + # 5) Environment destruction + # remove all created files + # test environment desctructor + db.execute_sql("CALL test_env_destructor();") + # TODO: remove tables from public scheme + + # Helper functions @@ -79,6 +96,11 @@ def parse_osm_file(file_path): return osm_data +def get_file_md5_hash(file_path): + with open(file_path, "rb") as f: + return hashlib.md5(f.read()).hexdigest() + + # Testing functions @@ -226,11 +248,33 @@ def test_integration_base_flow(): "area_id": 1, } _ = export_nodes_edges(config) - assert False - # TODO: assert created files + # assert created files by expected hash + expected_hashes = { + str(map_path / "edges.csv"): "96b1bc002e872cc13ba6d83823889070", + str(map_path / "nodes.csv"): "7d582c1906dd68bd1c38b99baa6da191", + str(map_path / "shapefiles/edges.cpg"): "ae3b3df9970b49b6523e608759bc957d", + str(map_path / "shapefiles/edges.dbf"): "0d327b70d2394bed6a6fd412d7886bc8", + str(map_path / "shapefiles/edges.prj"): "c742bee3d4edfc2948a2ad08de1790a5", + str(map_path / "shapefiles/edges.shp"): "5854b916fbaf5dadef0d18891c8aa4c8", + str(map_path / "shapefiles/edges.shx"): "0544fbe8da9bfff9360bae6b2f1fefe9", + str(map_path / "shapefiles/nodes.cpg"): "ae3b3df9970b49b6523e608759bc957d", + str(map_path / "shapefiles/nodes.dbf"): "f774112b1580f9d93c7afe45e2a5ee9f", + str(map_path / "shapefiles/nodes.prj"): "c742bee3d4edfc2948a2ad08de1790a5", + str(map_path / "shapefiles/nodes.shp"): "6a5b95f286089ad19ba943410c25d5f5", + str(map_path / "shapefiles/nodes.shx"): "499035dc9713ee99510385119f067710", + } + passage = True + result_dict = dict() + + for file_path, expected_hash in expected_hashes.items(): + actual_hash = get_file_md5_hash(file_path) + result = expected_hash != actual_hash + result_dict[file_path] = { + "assertion_result": result, + "message": "Ok" if result else f"File content mismatch for {file_path}", + } + if not result: + passage = result - # 5) Environment destruction - # test environment desctructor - db.execute_sql("CALL test_env_destructor();") - # TODO: remove tables from public scheme + assert passage, f"Result dictionary of assertion: {result_dict}" From a4bd82698dcccabb0aeebe08ce7b1551c2507715 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Wed, 9 Oct 2024 16:07:34 +0200 Subject: [PATCH 24/28] Added cleanup after base_flow --- python/tests/integrations_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py index 05e0d66..7290f08 100644 --- a/python/tests/integrations_test.py +++ b/python/tests/integrations_test.py @@ -1,5 +1,6 @@ import hashlib import os +import shutil import xml.etree.ElementTree as ET from pathlib import Path @@ -15,6 +16,7 @@ from scripts.process_osm import import_osm_to_db TEST_DATA_PATH = Path(__file__).parent / "data" +TMP_DIR = Path(__file__).parent / "TMP" # Fixtures @@ -25,6 +27,11 @@ def cleanup_for_base_flow(): # 5) Environment destruction # remove all created files + if TMP_DIR.is_file() or TMP_DIR.is_symlink(): + TMP_DIR.unlink() + elif TMP_DIR.is_dir(): + shutil.rmtree(TMP_DIR) + # test environment desctructor db.execute_sql("CALL test_env_destructor();") # TODO: remove tables from public scheme @@ -231,7 +238,6 @@ def test_integration_base_flow(): assert set([(0, 1), (0, 2), (0, 3), (0, 4), (1, 5)]) == set(component_data) # 4) Export data to files - TMP_DIR = Path(__file__).parent / "TMP" os.mkdir(str(TMP_DIR)) map_path = TMP_DIR / "map" area_dir = TMP_DIR / "area_dir" From ffb5144cba0e5ff242999b1226bb9d0a4701a9e2 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Wed, 9 Oct 2024 16:24:17 +0200 Subject: [PATCH 25/28] Fixed call of cleanup in base_flow_integration_test --- python/tests/integrations_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py index 7290f08..6d7cec7 100644 --- a/python/tests/integrations_test.py +++ b/python/tests/integrations_test.py @@ -111,7 +111,7 @@ def get_file_md5_hash(file_path): # Testing functions -def test_integration_base_flow(): +def test_integration_base_flow(cleanup_for_base_flow): """ Integration test: Base flow: From 2cd2acc1ae9ee6ff7ed29243c73f615fe65eaf2c Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Mon, 28 Oct 2024 15:22:40 +0100 Subject: [PATCH 26/28] Merged main into pipeline-integration-testing. Additional files --- python/scripts/find_bbox.py | 16 +-- python/scripts/main.py | 45 ++++--- python/scripts/process_osm.py | 179 ++++++++++++++++++------- python/tests/integrations_test.py | 2 +- python/tests/test_filter_osm.py | 22 +++- python/tests/test_process_osm.py | 208 +++++++++++++++++++++--------- 6 files changed, 328 insertions(+), 144 deletions(-) diff --git a/python/scripts/find_bbox.py b/python/scripts/find_bbox.py index e2941c0..51588e3 100644 --- a/python/scripts/find_bbox.py +++ b/python/scripts/find_bbox.py @@ -1,19 +1,19 @@ -import xml.etree.ElementTree as ET import sys +import xml.etree.ElementTree as ET def find_min_max(xml) -> tuple[float, float, float, float]: """Return tuple of floats representing bounding box borders.""" root = ET.fromstring(xml) - min_lon = float('inf') - min_lat = float('inf') - max_lon = float('-inf') - max_lat = float('-inf') + min_lon = float("inf") + min_lat = float("inf") + max_lon = float("-inf") + max_lat = float("-inf") - for node in root.findall('.//node'): - lat = float(node.get('lat')) - lon = float(node.get('lon')) + for node in root.findall(".//node"): + lat = float(node.get("lat")) + lon = float(node.get("lon")) min_lon = min(min_lon, lon) min_lat = min(min_lat, lat) max_lon = max(max_lon, lon) diff --git a/python/scripts/main.py b/python/scripts/main.py index 13bc5ca..9ca9783 100644 --- a/python/scripts/main.py +++ b/python/scripts/main.py @@ -1,5 +1,6 @@ import argparse import logging + import psycopg2.errors from roadgraphtool.db_operations import ( @@ -36,37 +37,39 @@ def configure_arg_parser() -> argparse.ArgumentParser: required=False, ) parser.add_argument( - '-i', - '--import', - dest='importing', - action="store_true", - help='Import OSM data to database specified in config.ini' + "-i", + "--import", + dest="importing", + action="store_true", + help="Import OSM data to database specified in config.ini", ) parser.add_argument( - '-if', - '--input-file', - dest='input_file', + "-if", + "--input-file", + dest="input_file", required=True, - help='Input OSM file path for -i/--import.' + help="Input OSM file path for -i/--import.", ) parser.add_argument( - '-sf', '--style-file', - dest='style_file', + "-sf", + "--style-file", + dest="style_file", help="Optional style file path for -i/--import. Default is 'default.lua' otherwise.", - required=False + required=False, ) parser.add_argument( - '-sch', '--schema', - dest='schema', + "-sch", + "--schema", + dest="schema", help="Optional schema argument for -i/--import. Default is 'public' otherwise.", - required=False + required=False, ) parser.add_argument( - '--force', - dest='force', + "--force", + dest="force", action="store_true", help="Force overwrite of data in existing tables in schema.", - required=False + required=False, ) return parser @@ -78,7 +81,7 @@ def main(arg_list: list[str] | None = None): if args.importing: import_osm_to_db(args.input_file, args.force, args.style_file, args.schema) - + area_id = args.area_id area_srid = args.area_srid fill_speed = args.fill_speed @@ -126,5 +129,5 @@ def main(arg_list: list[str] | None = None): compute_speeds_from_neighborhood_segments(area_id, area_srid) -if __name__ == '__main__': - main() \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/python/scripts/process_osm.py b/python/scripts/process_osm.py index 19b84c8..c4df25c 100644 --- a/python/scripts/process_osm.py +++ b/python/scripts/process_osm.py @@ -1,20 +1,23 @@ import argparse +import logging import os import subprocess from pathlib import Path -import logging from roadgraphtool.credentials_config import CREDENTIALS, CredentialsConfig -from scripts.filter_osm import InvalidInputError, MissingInputError, load_multipolygon_by_id, is_valid_extension, setup_logger -from scripts.find_bbox import find_min_max -from scripts.install_sql import SQL_DIR from roadgraphtool.db import db from roadgraphtool.schema import * +from scripts.filter_osm import (InvalidInputError, MissingInputError, + is_valid_extension, load_multipolygon_by_id, + setup_logger) +from scripts.find_bbox import find_min_max +from scripts.install_sql import SQL_DIR STYLES_DIR = Path(__file__).parent.parent.parent / "resources" / "lua_styles" DEFAULT_STYLE_FILE = STYLES_DIR / "pipeline.lua" -logger = setup_logger('process_osm') +logger = setup_logger("process_osm") + def extract_bbox(relation_id: int) -> tuple[float, float, float, float]: """Return tuple of floats based on bounding box coordinations.""" @@ -23,10 +26,13 @@ def extract_bbox(relation_id: int) -> tuple[float, float, float, float]: logger.debug(f"Bounding box found: {min_lon},{min_lat},{max_lon},{max_lat}.") return min_lon, min_lat, max_lon, max_lat + def run_osmium_cmd(flag: str, input_file: str, output_file: str = None): """Run osmium command based on flag.""" if output_file and not is_valid_extension(output_file): - raise InvalidInputError("File must have one of the following extensions: osm, osm.pbf, osm.bz2") + raise InvalidInputError( + "File must have one of the following extensions: osm, osm.pbf, osm.bz2" + ) match flag: case "d": subprocess.run(["osmium", "show", input_file]) @@ -34,46 +40,74 @@ def run_osmium_cmd(flag: str, input_file: str, output_file: str = None): subprocess.run(["osmium", "fileinfo", input_file]) case "ie": subprocess.run(["osmium", "fileinfo", "-e", input_file]) - case 'r': + case "r": res = subprocess.run(["osmium", "renumber", input_file, "-o", output_file]) if not res.returncode: logger.info("Renumbering of OSM data completed.") - case 's': + case "s": res = subprocess.run(["osmium", "sort", input_file, "-o", output_file]) if not res.returncode: logger.info("Sorting of OSM data completed.") - case 'sr': - tmp_file = 'tmp.osm' + case "sr": + tmp_file = "tmp.osm" res = subprocess.run(["osmium", "sort", input_file, "-o", tmp_file]) if not res.returncode: logger.info("Sorting of OSM data completed.") - res = subprocess.run(["osmium", "renumber", tmp_file, "-o", output_file]) + res = subprocess.run( + ["osmium", "renumber", tmp_file, "-o", output_file] + ) if not res.returncode: logger.info("Renumbering of OSM data completed.") os.remove(tmp_file) -def run_osm2pgsql_cmd(config: CredentialsConfig, input_file: str, style_file_path: str, schema: str, force: bool, coords: str| list[int] = None): + +def run_osm2pgsql_cmd( + config: CredentialsConfig, + input_file: str, + style_file_path: str, + schema: str, + force: bool, + coords: str | list[int] = None, +): """Import data from input_file to database specified in config using osm2pgsql tool.""" - if hasattr(config, "server"): # remote connection + if hasattr(config, "server") and config.server is not None: # remote connection ssh_tunnel_port = db.ssh_tunnel_local_port db.set_ssh_to_db_server_and_set_port() else: # local connection ssh_tunnel_port = config.db_server_port if not force and not check_empty_or_nonexistent_tables(schema, config): - raise TableNotEmptyError("Attempt to overwrite non-empty tables. Use '--force' flag to proceed.") + raise TableNotEmptyError( + "Attempt to overwrite non-empty tables. Use '--force' flag to proceed." + ) create_schema(schema, config) add_postgis_extension(schema, config) - cmd = ["osm2pgsql", "-d", config.db_name, "-U", config.username, "-W", "-H", config.db_host, - "-P", str(ssh_tunnel_port), "--output=flex", "-S", style_file_path, input_file, "-x", f"--schema={schema}"] + cmd = [ + "osm2pgsql", + "-d", + config.db_name, + "-U", + config.username, + "-W", + "-H", + config.db_host, + "-P", + str(ssh_tunnel_port), + "--output=flex", + "-S", + style_file_path, + input_file, + "-x", + f"--schema={schema}", + ] if coords: cmd.extend(["-b", coords]) if logger.level == logging.DEBUG: - cmd.extend(['--log-level=debug']) + cmd.extend(["--log-level=debug"]) logger.debug(f"Begin importing with: '{' '.join(cmd)}'") else: logger.info(f"Begin importing with: '{' '.join(cmd)}'") @@ -81,6 +115,7 @@ def run_osm2pgsql_cmd(config: CredentialsConfig, input_file: str, style_file_pat if not res.returncode: logger.info("Importing completed.") + def post_process_osm_import(style_filename: str, schema: str | None = None) -> int: post_proc_dict = {"pipeline.lua": "after_import.sql"} @@ -95,13 +130,13 @@ def post_process_osm_import(style_filename: str, schema: str | None = None) -> i command = [ "psql", "-d", - CredentialsConfig.db_name, + CREDENTIALS.db_name, "-U", - CredentialsConfig.username, + CREDENTIALS.username, "-h", - CredentialsConfig.host, + CREDENTIALS.host, "-p", - str(CredentialsConfig.db_server_port), + str(CREDENTIALS.db_server_port), ] if schema: command.extend(["-c", f"SET search_path TO {schema};"]) @@ -120,7 +155,10 @@ def post_process_osm_import(style_filename: str, schema: str | None = None) -> i return 0 -def import_osm_to_db(input_file: str, force: bool, style_file_path: str = None, schema: str = "public") -> int: + +def import_osm_to_db( + input_file: str, force: bool, style_file_path: str = None, schema: str = "public" +) -> int: """Return the size of OSM file in bytes if file found and imports OSM file do database specified in config.ini file. The **default.lua** style file is used if not specified or set otherwise. @@ -141,11 +179,18 @@ def import_osm_to_db(input_file: str, force: bool, style_file_path: str = None, post_process_osm_import(style_filename, schema) return file_size + def parse_args(arg_list: list[str] | None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Process OSM files and interact with PostgreSQL database.", formatter_class=argparse.RawTextHelpFormatter) + parser = argparse.ArgumentParser( + description="Process OSM files and interact with PostgreSQL database.", + formatter_class=argparse.RawTextHelpFormatter, + ) - parser.add_argument("flag", choices=["d", "i", "ie", "s", "r", "sr", "b", "u"], metavar="flag", - help=""" + parser.add_argument( + "flag", + choices=["d", "i", "ie", "s", "r", "sr", "b", "u"], + metavar="flag", + help=""" d : Display OSM file i : Display information about OSM file ie : Display extended information about OSM file @@ -154,15 +199,44 @@ def parse_args(arg_list: list[str] | None) -> argparse.Namespace: sr : Sort and renumber objects in OSM file u : Upload OSM file to PostgreSQL database using osm2pgsql b : Extract greatest bounding box from given relation ID of - input_file and upload to PostgreSQL database using osm2pgsql""" -) - parser.add_argument('input_file', help="Path to input OSM file") - parser.add_argument("-id", dest="relation_id", help="Relation ID (required for 'b' flag)") - parser.add_argument("-l", dest="style_file", nargs='?', default="resources/lua_styles/default.lua", help="Path to style file (optional for 'b', 'u' flag)") - parser.add_argument("-o", dest="output_file", help="Path to output file (required for 's', 'r', 'sr' flag)") - parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Enable verbose output (DEBUG level logging)") - parser.add_argument("-sch", "--schema", dest="schema", default="public", help="Database schema (for 'b', 'u' flag)") - parser.add_argument("--force", dest="force", action="store_true", help="Force overwrite of data in existing tables in schema (for 'b', 'u' flag)") + input_file and upload to PostgreSQL database using osm2pgsql""", + ) + parser.add_argument("input_file", help="Path to input OSM file") + parser.add_argument( + "-id", dest="relation_id", help="Relation ID (required for 'b' flag)" + ) + parser.add_argument( + "-l", + dest="style_file", + nargs="?", + default="resources/lua_styles/default.lua", + help="Path to style file (optional for 'b', 'u' flag)", + ) + parser.add_argument( + "-o", + dest="output_file", + help="Path to output file (required for 's', 'r', 'sr' flag)", + ) + parser.add_argument( + "-v", + "--verbose", + dest="verbose", + action="store_true", + help="Enable verbose output (DEBUG level logging)", + ) + parser.add_argument( + "-sch", + "--schema", + dest="schema", + default="public", + help="Database schema (for 'b', 'u' flag)", + ) + parser.add_argument( + "--force", + dest="force", + action="store_true", + help="Force overwrite of data in existing tables in schema (for 'b', 'u' flag)", + ) args = parser.parse_args(arg_list) @@ -173,33 +247,40 @@ def parse_args(arg_list: list[str] | None) -> argparse.Namespace: return args + def main(arg_list: list[str] | None = None): args = parse_args(arg_list) if not os.path.exists(args.input_file): raise FileNotFoundError(f"File '{args.input_file}' does not exist.") elif not is_valid_extension(args.input_file): - raise InvalidInputError("File must have one of the following extensions: osm, osm.pbf, osm.bz2.") + raise InvalidInputError( + "File must have one of the following extensions: osm, osm.pbf, osm.bz2." + ) elif args.style_file: if not os.path.exists(args.style_file): raise FileNotFoundError(f"File '{args.style_file}' does not exist.") elif not args.style_file.endswith(".lua"): raise InvalidInputError("File must have the '.lua' extension.") - + match args.flag: - case 'd' | 'i' | 'ie': + case "d" | "i" | "ie": # Display content or (extended) information of OSM file run_osmium_cmd(args.flag, args.input_file) - case 's' | 'r' | 'sr': + case "s" | "r" | "sr": # Sort, renumber OSM file or do both if not args.output_file: - raise MissingInputError("An output file must be specified with '-o' flag.") + raise MissingInputError( + "An output file must be specified with '-o' flag." + ) run_osmium_cmd(args.flag, args.input_file, args.output_file) - + case "u": # Upload OSM file to PostgreSQL database - run_osm2pgsql_cmd(CREDENTIALS, args.input_file, args.style_file, args.schema, args.force) + run_osm2pgsql_cmd( + CREDENTIALS, args.input_file, args.style_file, args.schema, args.force + ) case "b": # Extract bounding box based on relation ID and import to PostgreSQL if not args.relation_id: @@ -208,7 +289,15 @@ def main(arg_list: list[str] | None = None): min_lon, min_lat, max_lon, max_lat = extract_bbox(args.relation_id) coords = f"{min_lon},{min_lat},{max_lon},{max_lat}" - run_osm2pgsql_cmd(CREDENTIALS, args.input_file, args.style_file, args.schema, args.force, coords) - -if __name__ == '__main__': - main() \ No newline at end of file + run_osm2pgsql_cmd( + CREDENTIALS, + args.input_file, + args.style_file, + args.schema, + args.force, + coords, + ) + + +if __name__ == "__main__": + main() diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py index 6d7cec7..3dd7aa4 100644 --- a/python/tests/integrations_test.py +++ b/python/tests/integrations_test.py @@ -132,7 +132,7 @@ def test_integration_base_flow(cleanup_for_base_flow): # change the main schema db.execute_sql("CALL test_env_constructor();") - import_osm_to_db(str(TEST_DATA_PATH / "integration_test.osm"), schema="test_env") + import_osm_to_db(str(TEST_DATA_PATH / "integration_test.osm"), force=True, schema="test_env") insert_area( 1, diff --git a/python/tests/test_filter_osm.py b/python/tests/test_filter_osm.py index f13d6a5..448eb98 100644 --- a/python/tests/test_filter_osm.py +++ b/python/tests/test_filter_osm.py @@ -1,10 +1,15 @@ import os import pathlib import tempfile -import pytest import xml.etree.ElementTree as ET -from scripts.filter_osm import check_strategy, extract_id, is_valid_extension, load_multipolygon_by_id, extract_bbox, main, InvalidInputError, MissingInputError +import pytest + +from scripts.filter_osm import (InvalidInputError, MissingInputError, + check_strategy, extract_bbox, extract_id, + is_valid_extension, load_multipolygon_by_id, + main) + @pytest.fixture def expected_multipolygon_id(): @@ -51,10 +56,14 @@ def test_check_strategy(): assert check_strategy("simple") == None assert check_strategy("complete_ways") == None assert check_strategy("smart") == None - with pytest.raises(InvalidInputError, match="Invalid strategy type. Call filter_osm.py -h/--help to display help."): + with pytest.raises( + InvalidInputError, + match="Invalid strategy type. Call filter_osm.py -h/--help to display help.", + ): check_strategy("invalid_strategy") -def test_load_multipolygon_by_id_url(mocker,expected_multipolygon_id): + +def test_load_multipolygon_by_id_url(mocker, expected_multipolygon_id): relation_id = 5986438 url = f"https://www.openstreetmap.org/api/0.6/relation/{relation_id}/full" @@ -212,7 +221,10 @@ def test_main_id_startegy_valid(mocker): def test_main_b_missing_coord(): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["b", tmp_file.name] - with pytest.raises(MissingInputError, match="Coordinates or config file need to be specified with the 'b' flag."): + with pytest.raises( + MissingInputError, + match="Coordinates or config file need to be specified with the 'b' flag.", + ): main(arg_list) diff --git a/python/tests/test_process_osm.py b/python/tests/test_process_osm.py index 4676cfb..72d2c17 100644 --- a/python/tests/test_process_osm.py +++ b/python/tests/test_process_osm.py @@ -1,31 +1,37 @@ +import os import pathlib -import tempfile -import pytest import subprocess -import os -import psycopg2 +import tempfile import xml.etree.ElementTree as ET +import psycopg2 +import pytest + from roadgraphtool.credentials_config import CREDENTIALS as config -from scripts.process_osm import run_osmium_cmd, main, import_osm_to_db, run_osm2pgsql_cmd +from scripts.filter_osm import InvalidInputError, MissingInputError from scripts.find_bbox import find_min_max -from scripts.filter_osm import MissingInputError, InvalidInputError +from scripts.process_osm import (import_osm_to_db, main, run_osm2pgsql_cmd, + run_osmium_cmd) + @pytest.fixture def mock_subprocess_run(mocker): return mocker.patch("subprocess.run") + @pytest.fixture def mock_os_path_isfile(mocker): return mocker.patch("os.path.isfile") + @pytest.fixture def bounding_box(): parent_dir = pathlib.Path(__file__).parent file_path = str(parent_dir) + "/data/bbox_test.osm" - with open(file_path, 'rb') as f: + with open(file_path, "rb") as f: return f.read() + @pytest.fixture(scope="module") def db_connection(): conn = psycopg2.connect( @@ -33,11 +39,12 @@ def db_connection(): user=config.username, password=config.db_password, host=config.db_host, - port=config.db_server_port + port=config.db_server_port, ) yield conn conn.close() + @pytest.fixture def teardown_db(db_connection, request): def cleanup(): @@ -47,14 +54,17 @@ def cleanup(): cursor.execute("DROP TABLE IF EXISTS mockrelations;") db_connection.commit() cursor.close() + request.addfinalizer(cleanup) + @pytest.fixture def renumber_test_files(): parent_dir = pathlib.Path(__file__).parent input_file = str(parent_dir) + "/data/renumber_test.osm" output_file = str(parent_dir) + "/data/renumber_test_output.osm" - return input_file, output_file + return input_file, output_file + @pytest.fixture def sort_test_files(): @@ -63,29 +73,33 @@ def sort_test_files(): output_file = str(parent_dir) + "/data/sort_test_output.osm" return input_file, output_file + def is_renumbered_by_id(content, obj_type): """Function to check if the obj_type IDs are renumbered in ascending order""" root = ET.fromstring(content) ids = [] for object in root.findall(obj_type): - obj_id = int(object.get('id')) + obj_id = int(object.get("id")) ids.append(obj_id) expected_ids = list(range(1, len(ids) + 1)) return ids == expected_ids + def is_sorted_by_id(content, obj_type): """Function to check if the obj_type IDs are sorted in ascending order""" root = ET.fromstring(content) ids = [] for object in root.findall(obj_type): - obj_id = int(object.get('id')) + obj_id = int(object.get("id")) ids.append(obj_id) return ids == sorted(ids) + # TESTS: + def test_find_mix_max(bounding_box): min_lon, min_lat, max_lon, max_lat = find_min_max(bounding_box) assert min_lon == 15.0 @@ -93,169 +107,235 @@ def test_find_mix_max(bounding_box): assert max_lon == 30.0 assert max_lat == 15.0 + @pytest.mark.usefixtures("teardown_db") def test_run_osm2pgsql_cmd(db_connection): parent_dir = pathlib.Path(__file__).parent style_file_path = str(parent_dir) + "/data/mock_default.lua" input_file = str(parent_dir) + "/data/bbox_test.osm" - run_osm2pgsql_cmd(config, input_file, style_file_path, 'osm_testing', True) + run_osm2pgsql_cmd(config, input_file, style_file_path, "osm_testing", True) cursor = db_connection.cursor() - cursor.execute('SELECT COUNT(*) FROM mocknodes;') + cursor.execute("SELECT COUNT(*) FROM mocknodes;") nodes_count = cursor.fetchone()[0] assert nodes_count == 6 - cursor.execute('SELECT COUNT(*) FROM mockways;') + cursor.execute("SELECT COUNT(*) FROM mockways;") ways_count = cursor.fetchone()[0] assert ways_count == 0 - cursor.execute('SELECT COUNT(*) FROM mockrelations;') + cursor.execute("SELECT COUNT(*) FROM mockrelations;") relations_count = cursor.fetchone()[0] assert relations_count == 1 - cursor.execute('SELECT * FROM mocknodes WHERE node_id=1;') + cursor.execute("SELECT * FROM mocknodes WHERE node_id=1;") node = cursor.fetchone() assert node is not None - cursor.execute('SELECT * FROM mocknodes WHERE node_id=7;') + cursor.execute("SELECT * FROM mocknodes WHERE node_id=7;") node = cursor.fetchone() assert node is None cursor.close() + def test_run_osmium_cmd_renumber(renumber_test_files): input_file, output_file = renumber_test_files - run_osmium_cmd('r', input_file, output_file) + run_osmium_cmd("r", input_file, output_file) assert os.path.exists(output_file) - with open(output_file, 'r') as f: + with open(output_file, "r") as f: content = f.read() - assert is_renumbered_by_id(content, 'node') == True - assert is_renumbered_by_id(content, 'way') == True - assert is_renumbered_by_id(content, 'relation') == True + assert is_renumbered_by_id(content, "node") == True + assert is_renumbered_by_id(content, "way") == True + assert is_renumbered_by_id(content, "relation") == True os.remove(output_file) + def test_run_osmium_cmd_sort(sort_test_files): input_file, output_file = sort_test_files - run_osmium_cmd('s', str(input_file), str(output_file)) + run_osmium_cmd("s", str(input_file), str(output_file)) assert os.path.exists(output_file) - with open(output_file, 'r') as f: + with open(output_file, "r") as f: content = f.read() - assert is_sorted_by_id(content, 'node') == True - assert is_sorted_by_id(content, 'way') == True - assert is_sorted_by_id(content, 'relation') == True + assert is_sorted_by_id(content, "node") == True + assert is_sorted_by_id(content, "way") == True + assert is_sorted_by_id(content, "relation") == True os.remove(output_file) + # not working def test_run_osmium_cmd_sort_renumber(mocker): - mock_subprocess_run = mocker.patch('subprocess.run') - mock_subprocess_run.side_effect = [subprocess.CompletedProcess(args=[], returncode=0), # for sort - subprocess.CompletedProcess(args=[], returncode=0)] # for renumber - - mock_remove = mocker.patch('os.remove') - input_file = 'test_input.osm' - output_file = 'test_output.osm' - tmp_file = 'tmp.osm' - - run_osmium_cmd('sr', input_file, output_file) + mock_subprocess_run = mocker.patch("subprocess.run") + mock_subprocess_run.side_effect = [ + subprocess.CompletedProcess(args=[], returncode=0), # for sort + subprocess.CompletedProcess(args=[], returncode=0), + ] # for renumber + + mock_remove = mocker.patch("os.remove") + input_file = "test_input.osm" + output_file = "test_output.osm" + tmp_file = "tmp.osm" + + run_osmium_cmd("sr", input_file, output_file) # both sort and renumbering occurred assert mock_subprocess_run.call_count == 2 mock_subprocess_run.assert_any_call(["osmium", "sort", input_file, "-o", tmp_file]) - mock_subprocess_run.assert_any_call(["osmium", "renumber", tmp_file, "-o", output_file]) + mock_subprocess_run.assert_any_call( + ["osmium", "renumber", tmp_file, "-o", output_file] + ) # tmp_file was deleted mock_remove.assert_called_once_with(tmp_file) + def test_import_to_db_valid(mocker): - mocker.patch('scripts.process_osm.os.path.exists', side_effect=lambda path: path in ["resources/to_import.osm", 'resources/lua_styles/default.lua']) - mocker.patch('os.path.getsize', return_value=1) - mock_run_osm2pgsql_cmd = mocker.patch('scripts.process_osm.run_osm2pgsql_cmd') - file_size = import_osm_to_db('resources/to_import.osm', True, schema='osm_testing') - mock_run_osm2pgsql_cmd.assert_called_once_with(config, 'resources/to_import.osm', 'resources/lua_styles/default.lua', 'osm_testing', True) + mocker.patch( + "scripts.process_osm.os.path.exists", + side_effect=lambda path: path + in ["resources/to_import.osm", "resources/lua_styles/default.lua"], + ) + mocker.patch("os.path.getsize", return_value=1) + mock_run_osm2pgsql_cmd = mocker.patch("scripts.process_osm.run_osm2pgsql_cmd") + file_size = import_osm_to_db("resources/to_import.osm", True, schema="osm_testing") + mock_run_osm2pgsql_cmd.assert_called_once_with( + config, + "resources/to_import.osm", + "resources/lua_styles/default.lua", + "osm_testing", + True, + ) assert file_size == 1 + def test_import_to_db_invalid_file(mocker): - mocker.patch('scripts.process_osm.os.path.exists', side_effect=lambda path: path == 'resources/lua_styles/default.lua') + mocker.patch( + "scripts.process_osm.os.path.exists", + side_effect=lambda path: path == "resources/lua_styles/default.lua", + ) with pytest.raises(FileNotFoundError, match="No valid file to import was found."): - import_osm_to_db('resources/to_import.osm', False) + import_osm_to_db("resources/to_import.osm", False) + def test_main_invalid_inputfile(): arg_list = ["d", "invalid_file.osm"] - with pytest.raises(FileNotFoundError, match="File 'invalid_file.osm' does not exist."): + with pytest.raises( + FileNotFoundError, match="File 'invalid_file.osm' does not exist." + ): main(arg_list) + def test_invalid_inputfile_extension(): with tempfile.NamedTemporaryFile(suffix=".txt") as tmp_file: arg_list = ["d", tmp_file.name] - with pytest.raises(InvalidInputError, match="File must have one of the following extensions: osm, osm.pbf, osm.bz2"): + with pytest.raises( + InvalidInputError, + match="File must have one of the following extensions: osm, osm.pbf, osm.bz2", + ): main(arg_list) + @pytest.mark.parametrize("test_input", ["d", "i", "ie"]) def test_main_diie_valid(mocker, test_input): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = [test_input, tmp_file.name] - mock_run_osmium_cmd = mocker.patch('scripts.process_osm.run_osmium_cmd') + mock_run_osmium_cmd = mocker.patch("scripts.process_osm.run_osmium_cmd") main(arg_list) mock_run_osmium_cmd.assert_called_once_with(arg_list[0], arg_list[1]) + @pytest.mark.parametrize("test_input", ["s", "r", "sr"]) def test_main_srsr_invalid(test_input): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = [test_input, tmp_file.name] - with pytest.raises(MissingInputError, match="An output file must be specified with '-o' flag."): + with pytest.raises( + MissingInputError, match="An output file must be specified with '-o' flag." + ): main(arg_list) + @pytest.mark.parametrize("test_input", ["s", "r", "sr"]) def test_main_srsr_valid(mocker, test_input): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = [test_input, tmp_file.name, "-o", " output.osm"] - mock_run_osmium_cmd = mocker.patch('scripts.process_osm.run_osmium_cmd') + mock_run_osmium_cmd = mocker.patch("scripts.process_osm.run_osmium_cmd") main(arg_list) - mock_run_osmium_cmd.assert_called_once_with(arg_list[0], arg_list[1], arg_list[3]) + mock_run_osmium_cmd.assert_called_once_with( + arg_list[0], arg_list[1], arg_list[3] + ) + def test_main_default_style_valid(mocker): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["u", tmp_file.name] - mock_run_osm2pgsql_cmd = mocker.patch('scripts.process_osm.run_osm2pgsql_cmd') + mock_run_osm2pgsql_cmd = mocker.patch("scripts.process_osm.run_osm2pgsql_cmd") main(arg_list) - mock_run_osm2pgsql_cmd.assert_called_once_with(config, arg_list[1], "resources/lua_styles/default.lua", "public", False) + mock_run_osm2pgsql_cmd.assert_called_once_with( + config, arg_list[1], "resources/lua_styles/default.lua", "public", False + ) + def test_main_input_style_valid(mocker): - with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_input, tempfile.NamedTemporaryFile(suffix=".lua") as tmp_lua: + with tempfile.NamedTemporaryFile( + suffix=".osm" + ) as tmp_input, tempfile.NamedTemporaryFile(suffix=".lua") as tmp_lua: arg_list = ["u", tmp_input.name, "-l", tmp_lua.name] - mock_run_osm2pgsql_cmd = mocker.patch('scripts.process_osm.run_osm2pgsql_cmd') + mock_run_osm2pgsql_cmd = mocker.patch("scripts.process_osm.run_osm2pgsql_cmd") main(arg_list) - mock_run_osm2pgsql_cmd.assert_called_once_with(config, arg_list[1], arg_list[3], "public", False) + mock_run_osm2pgsql_cmd.assert_called_once_with( + config, arg_list[1], arg_list[3], "public", False + ) + def test_main_style_file_invalid(): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["u", tmp_file.name, "-l", "invalid_style.lua"] - with pytest.raises(FileNotFoundError, match="File 'invalid_style.lua' does not exist."): + with pytest.raises( + FileNotFoundError, match="File 'invalid_style.lua' does not exist." + ): main(arg_list) + def test_main_style_file_extension_invalid(): - with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file, tempfile.NamedTemporaryFile(suffix=".txt") as invalid_lua: + with tempfile.NamedTemporaryFile( + suffix=".osm" + ) as tmp_file, tempfile.NamedTemporaryFile(suffix=".txt") as invalid_lua: arg_list = ["u", tmp_file.name, "-l", invalid_lua.name] - with pytest.raises(InvalidInputError, match="File must have the '.lua' extension."): + with pytest.raises( + InvalidInputError, match="File must have the '.lua' extension." + ): main(arg_list) + def test_main_bbox_valid(mocker): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["b", tmp_file.name, "-id", "1234"] - mock_extract_bbox = mocker.patch('scripts.process_osm.extract_bbox', return_value=(10, 20, 30, 40)) - mock_run_osm2pgsql_cmd = mocker.patch('scripts.process_osm.run_osm2pgsql_cmd') + mock_extract_bbox = mocker.patch( + "scripts.process_osm.extract_bbox", return_value=(10, 20, 30, 40) + ) + mock_run_osm2pgsql_cmd = mocker.patch("scripts.process_osm.run_osm2pgsql_cmd") main(arg_list) mock_extract_bbox.assert_called_once_with(arg_list[3]) - mock_run_osm2pgsql_cmd.assert_called_once_with(config, arg_list[1], "resources/lua_styles/default.lua", "public", False, "10,20,30,40") + mock_run_osm2pgsql_cmd.assert_called_once_with( + config, + arg_list[1], + "resources/lua_styles/default.lua", + "public", + False, + "10,20,30,40", + ) + # relation_id missing def test_main_bbox_id_missing(): with tempfile.NamedTemporaryFile(suffix=".osm") as tmp_file: arg_list = ["b", tmp_file.name] - with pytest.raises(MissingInputError, match="Existing relation ID must be specified."): - main(arg_list) \ No newline at end of file + with pytest.raises( + MissingInputError, match="Existing relation ID must be specified." + ): + main(arg_list) From 55f18f3246563fe796c6bf63954098e8a51c0686 Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Mon, 28 Oct 2024 16:41:06 +0100 Subject: [PATCH 27/28] Resolved issue #97. Wrong overwrite of db port --- python/scripts/process_osm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/scripts/process_osm.py b/python/scripts/process_osm.py index d5ce9b1..01fc010 100644 --- a/python/scripts/process_osm.py +++ b/python/scripts/process_osm.py @@ -64,7 +64,7 @@ def run_osmium_cmd(flag: str, input_file: str, output_file: str = None): def setup_ssh_tunnel(config: CredentialsConfig) -> int: """Set up SSH tunnel if needed and returns port number.""" - if hasattr(config, "server"): # remote connection + if hasattr(config, "server") and config.server is not None: # remote connection db.start_or_restart_ssh_connection_if_needed() config.db_server_port = db.ssh_tunnel_local_port return db.ssh_tunnel_local_port @@ -109,7 +109,7 @@ def run_osm2pgsql_cmd(config: CredentialsConfig, input_file: str, style_file_pat if res: raise SubprocessError(f"Error during import: {res}") -logger.info("Importing completed.") + logger.info("Importing completed.") def postprocess_osm_import(config: CredentialsConfig, style_file_path: str, schema: str): """Apply postprocessing SQL associated with **style_file_path** to data in **schema** after importing. From c73b67ad6d63644ac434927065f4a432f52c351f Mon Sep 17 00:00:00 2001 From: Vladyslav Zlochevskyi Date: Mon, 18 Nov 2024 13:27:21 +0100 Subject: [PATCH 28/28] Added support of dynamic schema in multiple functions --- python/roadgraphtool/db.py | 16 +++++++++--- python/roadgraphtool/db_operations.py | 8 +++--- python/roadgraphtool/insert_area.py | 11 +++++--- python/roadgraphtool/map.py | 13 +++++----- python/tests/integrations_test.py | 37 ++++++++++++++++----------- 5 files changed, 53 insertions(+), 32 deletions(-) diff --git a/python/roadgraphtool/db.py b/python/roadgraphtool/db.py index 9e34e61..fd67589 100644 --- a/python/roadgraphtool/db.py +++ b/python/roadgraphtool/db.py @@ -161,14 +161,22 @@ def execute_sql(self, query, *args, schema='public', use_transactions=True) -> N if not use_transactions: connection.execution_options(isolation_level="AUTOCOMMIT") with connection.begin(): - connection.execute(sqlalchemy.text(f"SET search_path TO {schema};")) + search_path = connection.execute(sqlalchemy.text("SHOW search_path;")).all()[0][0] + connection.execute(sqlalchemy.text(f"SET search_path TO {','.join([schema, search_path])};")) connection.execute(sqlalchemy.text(query), *args) - connection.execute(sqlalchemy.text(f"SET search_path TO public;")) + connection.execute(sqlalchemy.text(f"SET search_path TO {search_path};")) @connect_db_if_required - def execute_sql_and_fetch_all_rows(self, query, *args) -> list[Row]: + def execute_sql_and_fetch_all_rows(self, query, *args, schema=None) -> list[Row]: with self._sqlalchemy_engine.connect() as conn: - result = conn.execute(sqlalchemy.text(query), *args).all() + result = None + if schema: + search_path = conn.execute(sqlalchemy.text("SHOW search_path;")).all()[0][0] + conn.execute(sqlalchemy.text(f"SET search_path TO {','.join([schema, search_path])};")) + result = conn.execute(sqlalchemy.text(query), *args).all() + conn.execute(sqlalchemy.text(f"SET search_path TO {search_path};")) + else: + result = conn.execute(sqlalchemy.text(query), *args).all() return result @connect_db_if_required diff --git a/python/roadgraphtool/db_operations.py b/python/roadgraphtool/db_operations.py index ac1215d..968e69c 100644 --- a/python/roadgraphtool/db_operations.py +++ b/python/roadgraphtool/db_operations.py @@ -42,10 +42,10 @@ def get_area_for_demand( def contract_graph_in_area( - target_area_id: int, target_area_srid: int, fill_speed: bool = True + target_area_id: int, target_area_srid: int, fill_speed: bool = True, schema=None ): sql_query = f'call public.contract_graph_in_area({target_area_id}::smallint, {target_area_srid}::int{", FALSE" if not fill_speed else ""})' - db.execute_sql(sql_query) + db.execute_sql(sql_query, schema=schema) def select_network_nodes_in_area(target_area_id: int) -> list: @@ -65,9 +65,9 @@ def assign_average_speed_to_all_segments_in_area( db.execute_sql(sql_query) -def compute_strong_components(target_area_id: int): +def compute_strong_components(target_area_id: int, schema=None): sql_query = f"call public.compute_strong_components({target_area_id}::smallint)" - db.execute_sql(sql_query) + db.execute_sql(sql_query, schema=schema) def compute_speeds_for_segments( diff --git a/python/roadgraphtool/insert_area.py b/python/roadgraphtool/insert_area.py index 565ba03..5c6de6e 100644 --- a/python/roadgraphtool/insert_area.py +++ b/python/roadgraphtool/insert_area.py @@ -5,7 +5,7 @@ from roadgraphtool.db import db -def insert_area(id: int | None, name: str, description: str | None, geom: dict): +def insert_area(id: int | None, name: str, description: str | None, geom: dict, schema: str | None = None): """ Insert a new area into the areas table. @@ -18,16 +18,21 @@ def insert_area(id: int | None, name: str, description: str | None, geom: dict): Returns: None """ + schema = "public" if schema is None else schema + if description is None: description = "" + # result ignored as the pgsql function returns void if id is None: db.execute_sql( - f"SELECT insert_area('{name}', '{json.dumps(geom)}', NULL, '{description}')" + f"SELECT insert_area('{name}', '{json.dumps(geom)}', NULL, '{description}')", + schema=schema ) else: db.execute_sql( - f"SELECT insert_area('{name}', '{json.dumps(geom)}', {id}, '{description}')" + f"SELECT insert_area('{name}', '{json.dumps(geom)}', {id}, '{description}')", + schema=schema ) diff --git a/python/roadgraphtool/map.py b/python/roadgraphtool/map.py index 4fc7f86..953464e 100644 --- a/python/roadgraphtool/map.py +++ b/python/roadgraphtool/map.py @@ -22,10 +22,11 @@ def add_node_highway_tags(nodes, G): nodes.loc[nodes.index[[v]], 'highway'] = tag -def _get_map_from_db(config: dict) -> Tuple[gpd.GeoDataFrame, gpd.GeoDataFrame]: - nodes = get_map_nodes_from_db(config['area_id']) +def _get_map_from_db(config: dict, schema: str | None = None) -> Tuple[gpd.GeoDataFrame, gpd.GeoDataFrame]: + schema = schema if schema else 'public' + nodes = get_map_nodes_from_db(config['area_id'], schema) logging.info(f"{len(nodes)} nodes fetched from db") - edges = get_map_edges_from_db(config) + edges = get_map_edges_from_db(config, schema) logging.info(f"{len(edges)} edges fetched from db") return nodes, edges @@ -91,7 +92,7 @@ def _save_graph_shapefile(nodes: gpd.GeoDataFrame, edges: gpd.GeoDataFrame, shap edges.to_file(str(filepath_edges), driver="ESRI Shapefile", index=False, encoding="utf-8") -def get_map(config: Dict) -> Tuple[gpd.GeoDataFrame, gpd.GeoDataFrame]: +def get_map(config: Dict, schema: str | None = None) -> Tuple[gpd.GeoDataFrame, gpd.GeoDataFrame]: """ Loads filtered map nodes geodataframe. If th dataframe is not generated yet, then the map is downloaded and processed to obtain the filtered nodes dataframe @@ -121,7 +122,7 @@ def get_map(config: Dict) -> Tuple[gpd.GeoDataFrame, gpd.GeoDataFrame]: if 'place' in config['map']: nodes, edges = _get_map(config) else: - nodes, edges = _get_map_from_db(config) + nodes, edges = _get_map_from_db(config, schema=schema) # save map to shapefile (for visualising) map_dir = config["map"]["path"] @@ -135,7 +136,7 @@ def get_map(config: Dict) -> Tuple[gpd.GeoDataFrame, gpd.GeoDataFrame]: # Filter nodes by config/area. Only these nodes should be used for demand/vehicle generation/selection if 'area' in config: sql = f"""SELECT geom FROM areas WHERE name = '{config['area']}'""" - area_shape = db.execute_query_to_geopandas(sql) + area_shape = db.execute_query_to_geopandas(sql, schema=schema if schema else 'public') mask = nodes.within(area_shape.loc[0, 'geom']) nodes = nodes.loc[mask] diff --git a/python/tests/integrations_test.py b/python/tests/integrations_test.py index 3dd7aa4..e5eff55 100644 --- a/python/tests/integrations_test.py +++ b/python/tests/integrations_test.py @@ -17,6 +17,7 @@ TEST_DATA_PATH = Path(__file__).parent / "data" TMP_DIR = Path(__file__).parent / "TMP" +TEST_SCHEMA = "test_env" # Fixtures @@ -132,22 +133,23 @@ def test_integration_base_flow(cleanup_for_base_flow): # change the main schema db.execute_sql("CALL test_env_constructor();") - import_osm_to_db(str(TEST_DATA_PATH / "integration_test.osm"), force=True, schema="test_env") + import_osm_to_db(str(TEST_DATA_PATH / "integration_test.osm"), force=True, pgpass=False, schema=TEST_SCHEMA) insert_area( 1, "Deutschland", "test area", read_area(str(TEST_DATA_PATH / "integration_area.json")), + schema=f"{TEST_SCHEMA}, public" ) # read osm file to dictionary osm_dict = parse_osm_file(TEST_DATA_PATH / "integration_test.osm") - print(osm_dict) # Test that importing osm data and area was done successfully nodes = db.execute_sql_and_fetch_all_rows( - "SELECT id, ST_X(geom), ST_Y(geom) FROM nodes;" + "SELECT id, ST_X(geom), ST_Y(geom) FROM nodes;", + schema=TEST_SCHEMA ) nodes_set = set(nodes) @@ -162,7 +164,8 @@ def test_integration_base_flow(cleanup_for_base_flow): assert nodes_set == expected_nodes_set ways = db.execute_sql_and_fetch_all_rows( - 'SELECT id, tags, "from", "to", oneway FROM ways;' + 'SELECT id, tags, "from", "to", oneway FROM ways;', + schema=TEST_SCHEMA ) ways_set = set( @@ -193,7 +196,8 @@ def test_integration_base_flow(cleanup_for_base_flow): assert expected_ways_set == ways_set nodes_ways = db.execute_sql_and_fetch_all_rows( - "SELECT way_id, node_id FROM nodes_ways;" + "SELECT way_id, node_id FROM nodes_ways;", + schema=TEST_SCHEMA ) nodes_ways_set = set(nodes_ways) @@ -205,7 +209,8 @@ def test_integration_base_flow(cleanup_for_base_flow): assert nodes_ways_set == expected_nodes_ways area = db.execute_sql_and_fetch_all_rows( - "SELECT id, name, description FROM areas;" + "SELECT id, name, description FROM areas;", + schema=TEST_SCHEMA )[0] expected_area = (1, "Deutschland", "test area") @@ -213,29 +218,31 @@ def test_integration_base_flow(cleanup_for_base_flow): assert area == expected_area # 2) Call contraction - contract_graph_in_area(1, 4326, fill_speed=False) + contract_graph_in_area(1, 4326, fill_speed=False, schema=TEST_SCHEMA) # GET updated data from db and assert contracted_nodes = db.execute_sql_and_fetch_all_rows( - "SELECT id FROM nodes WHERE contracted;" + "SELECT id FROM nodes WHERE contracted;", + schema=TEST_SCHEMA ) - assert set([(6,), (7,), (8,)]) == set(contracted_nodes) + assert {(6,), (7,), (8,)} == set(contracted_nodes) - edges = db.execute_sql_and_fetch_all_rows('SELECT "from", "to" FROM edges;') + edges = db.execute_sql_and_fetch_all_rows('SELECT "from", "to" FROM edges;', schema=TEST_SCHEMA) - assert set([(3, 4), (2, 1), (1, 2), (5, 3), (5, 2), (4, 3), (2, 3), (3, 2)]) == set( + assert {(3, 4), (2, 1), (1, 2), (5, 3), (5, 2), (4, 3), (2, 3), (3, 2)} == set( edges ) # 3) Call Compute Strong Components - compute_strong_components(1) + compute_strong_components(1, schema=TEST_SCHEMA) # GET updated data from db and assert component_data = db.execute_sql_and_fetch_all_rows( - "SELECT component_id, node_id FROM component_data" + "SELECT component_id, node_id FROM component_data", + schema=TEST_SCHEMA ) - assert set([(0, 1), (0, 2), (0, 3), (0, 4), (1, 5)]) == set(component_data) + assert {(0, 1), (0, 2), (0, 3), (0, 4), (1, 5)} == set(component_data) # 4) Export data to files os.mkdir(str(TMP_DIR)) @@ -253,7 +260,7 @@ def test_integration_base_flow(cleanup_for_base_flow): "area": "Deutschland", "area_id": 1, } - _ = export_nodes_edges(config) + _ = export_nodes_edges(config, schema=f"{TEST_SCHEMA}, \"$user\"") # assert created files by expected hash expected_hashes = {