From 8a3f899ffd6615b546f9428d932562d2be44f9a0 Mon Sep 17 00:00:00 2001 From: Bob Torgerson Date: Thu, 20 Mar 2025 09:16:59 -0800 Subject: [PATCH 1/3] Adds SUMMARY to list of fields that are collected into our generated shape file. --- wildfire_map/fire_layers/get_current_fire_layers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wildfire_map/fire_layers/get_current_fire_layers.py b/wildfire_map/fire_layers/get_current_fire_layers.py index cec1722..a5acd13 100644 --- a/wildfire_map/fire_layers/get_current_fire_layers.py +++ b/wildfire_map/fire_layers/get_current_fire_layers.py @@ -9,9 +9,9 @@ data_dir = os.path.join(script_dir, "data", "debug") # All of the URLs used to generate the layers -active_fire_perimeters_url = "https://fire.ak.blm.gov/arcgis/rest/services/MapAndFeatureServices/Fires_Perimeters/MapServer/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=OBJECTID%2CNAME%2CACRES%2CIRWINID%2CPRESCRIBED%2CLASTUPDATEDATETIME&returnGeometry=true&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=¶meterValues=&rangeValues=&quantizationParameters=&featureEncoding=esriDefault&f=geojson" +active_fire_perimeters_url = "https://fire.ak.blm.gov/arcgis/rest/services/MapAndFeatureServices/Fires_Perimeters/MapServer/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=OBJECTID%2CNAME%2CACRES%2CIRWINID%2CPRESCRIBED%2CLASTUPDATEDATETIME%2CSUMMARY&returnGeometry=true&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=¶meterValues=&rangeValues=&quantizationParameters=&featureEncoding=esriDefault&f=geojson" active_fires_url = "https://fire.ak.blm.gov/arcgis/rest/services/MapAndFeatureServices/Fires/MapServer/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=GENERALCAUSE%2COBJECTID%2CNAME%2CLASTUPDATEDATETIME%2CLATITUDE%2CLONGITUDE%2CPRESCRIBEDFIRE%2CDISCOVERYDATETIME%2CESTIMATEDTOTALACRES%2CSUMMARY%2CIRWINID&returnGeometry=true&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=¶meterValues=&rangeValues=&quantizationParameters=&featureEncoding=esriDefault&f=geojson" -inactive_fire_perimeters_url = "https://fire.ak.blm.gov/arcgis/rest/services/MapAndFeatureServices/Fires_Perimeters/MapServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=OBJECTID%2CNAME%2CACRES%2CIRWINID%2CPRESCRIBED%2CLASTUPDATEDATETIME&returnGeometry=true&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=¶meterValues=&rangeValues=&quantizationParameters=&featureEncoding=esriDefault&f=geojson" +inactive_fire_perimeters_url = "https://fire.ak.blm.gov/arcgis/rest/services/MapAndFeatureServices/Fires_Perimeters/MapServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=OBJECTID%2CNAME%2CACRES%2CIRWINID%2CPRESCRIBED%2CLASTUPDATEDATETIME%2CSUMMARY&returnGeometry=true&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=¶meterValues=&rangeValues=&quantizationParameters=&featureEncoding=esriDefault&f=geojson" inactive_fires_url = "https://fire.ak.blm.gov/arcgis/rest/services/MapAndFeatureServices/Fires/MapServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=GENERALCAUSE%2COBJECTID%2CNAME%2CLASTUPDATEDATETIME%2CLATITUDE%2CLONGITUDE%2CDISCOVERYDATETIME%2CESTIMATEDTOTALACRES%2CSUMMARY%2COUTDATE%2CIRWINID&returnGeometry=true&returnTrueCurves=false&maxAllowableOffset=&geometryPrecision=&outSR=&having=&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&historicMoment=&returnDistinctValues=false&resultOffset=&resultRecordCount=&queryByDistance=&returnExtentOnly=false&datumTransformation=¶meterValues=&rangeValues=&quantizationParameters=&featureEncoding=esriDefault&f=geojson" viirs_12hr_url = "https://fire.gina.alaska.edu/arcgis/rest/services/afs/VIIRS_iBand_FireHeatPoints/MapServer/1/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&distance=&units=esriSRUnit_Foot&relationParam=&outFields=&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=&havingClause=&gdbVersion=&historicMoment=&returnDistinctValues=false&returnIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&multipatchOption=xyFootprint&resultOffset=&resultRecordCount=&returnTrueCurves=false&returnExceededLimitFeatures=false&quantizationParameters=&returnCentroid=false&sqlFormat=none&resultType=&featureEncoding=esriDefault&datumTransformation=&f=geojson" viirs_24hr_url = "https://fire.gina.alaska.edu/arcgis/rest/services/afs/VIIRS_iBand_FireHeatPoints/MapServer/2/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&distance=&units=esriSRUnit_Foot&relationParam=&outFields=&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=&havingClause=&gdbVersion=&historicMoment=&returnDistinctValues=false&returnIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&multipatchOption=xyFootprint&resultOffset=&resultRecordCount=&returnTrueCurves=false&returnExceededLimitFeatures=false&quantizationParameters=&returnCentroid=false&sqlFormat=none&resultType=&featureEncoding=esriDefault&datumTransformation=&f=geojson" @@ -223,6 +223,7 @@ def parse_acres(a): "updated": feature["properties"].get("updated", None), "OUTDATE": feature["properties"].get("OUTDATE", None), "discovered": feature["properties"].get("discovered", None), + "SUMMARY": feature["properties"].get("SUMMARY", None), } # This filters out null, NaN and "0" fire sizes From 8a77b8133e74ebca4e1d7d59dbf8ff5aa82bd14a Mon Sep 17 00:00:00 2001 From: Bob Torgerson Date: Thu, 20 Mar 2025 10:59:06 -0800 Subject: [PATCH 2/3] Updates shapefile creation to add SUMMARY. --- wildfire_map/fire_layers/fire_layer_tasks.py | 10 +++++----- wildfire_map/fire_layers/get_current_fire_layers.py | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/wildfire_map/fire_layers/fire_layer_tasks.py b/wildfire_map/fire_layers/fire_layer_tasks.py index d81675a..1af8f2c 100644 --- a/wildfire_map/fire_layers/fire_layer_tasks.py +++ b/wildfire_map/fire_layers/fire_layer_tasks.py @@ -33,7 +33,7 @@ def install_conda_environment(conda_env_name, conda_env_file, local_install=Fals # Check if the Conda environment already exists conda_env_exists = ( subprocess.run( - f"source $HOME/miniconda3/bin/activate && $HOME/miniconda3/bin/conda env list | grep {conda_env_name}", + f". $HOME/miniconda3/bin/activate && $HOME/miniconda3/bin/conda env list | grep {conda_env_name}", shell=True, ).returncode == 0 @@ -44,7 +44,7 @@ def install_conda_environment(conda_env_name, conda_env_file, local_install=Fals # Install the Conda environment from the environment file subprocess.run( - f"source $HOME/miniconda3/bin/activate && $HOME/miniconda3/bin/conda env create -n {conda_env_name} -f {conda_env_file}", + f". $HOME/miniconda3/bin/activate && $HOME/miniconda3/bin/conda env create -n {conda_env_name} -f {conda_env_file}", shell=True, check=True, ) @@ -69,7 +69,7 @@ def install_conda_environment(conda_env_name, conda_env_file, local_install=Fals # Check if the Conda environment already exists conda_env_exists = ( subprocess.run( - f"source /opt/miniconda3/bin/activate && /opt/miniconda3/bin/conda env list | grep {conda_env_name}", + f". /opt/miniconda3/bin/activate && /opt/miniconda3/bin/conda env list | grep {conda_env_name}", shell=True, ).returncode == 0 @@ -80,7 +80,7 @@ def install_conda_environment(conda_env_name, conda_env_file, local_install=Fals # Install the Conda environment from the environment file subprocess.run( - f"source /opt/miniconda3/bin/activate && /opt/miniconda3/bin/conda env create -n {conda_env_name} -f {conda_env_file}", + f". /opt/miniconda3/bin/activate && /opt/miniconda3/bin/conda env create -n {conda_env_name} -f {conda_env_file}", shell=True, check=True, ) @@ -95,7 +95,7 @@ def execute_local_script( ): # Execute the script on the local machine process = subprocess.Popen( - f". /opt/miniconda3/bin/activate {conda_env_name}; python {script_path} --out-dir {output_path}", + f". /opt/miniconda3/bin/activate {conda_env_name} && /home/snapdata/.conda/envs/{conda_env_name}/bin/python {script_path} --out-dir {output_path}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/wildfire_map/fire_layers/get_current_fire_layers.py b/wildfire_map/fire_layers/get_current_fire_layers.py index a5acd13..9782e57 100644 --- a/wildfire_map/fire_layers/get_current_fire_layers.py +++ b/wildfire_map/fire_layers/get_current_fire_layers.py @@ -220,10 +220,10 @@ def parse_acres(a): "NAME": feature["properties"]["NAME"], "acres": feature["properties"]["acres"], "CAUSE": feature["properties"]["CAUSE"], + "SUMMARY": feature["properties"]["SUMMARY"], "updated": feature["properties"].get("updated", None), "OUTDATE": feature["properties"].get("OUTDATE", None), "discovered": feature["properties"].get("discovered", None), - "SUMMARY": feature["properties"].get("SUMMARY", None), } # This filters out null, NaN and "0" fire sizes @@ -253,6 +253,7 @@ def convert_geojson_to_shapefile(geojson_features, out_shapefile, feature_type=" point_layer.CreateField(ogr.FieldDefn("updated", ogr.OFTString)) point_layer.CreateField(ogr.FieldDefn("CAUSE", ogr.OFTString)) point_layer.CreateField(ogr.FieldDefn("discovered", ogr.OFTString)) + point_layer.CreateField(ogr.FieldDefn("SUMMARY", ogr.OFTString)) polygon_layer.CreateField(ogr.FieldDefn("NAME", ogr.OFTString)) polygon_layer.CreateField(ogr.FieldDefn("acres", ogr.OFTReal)) @@ -261,6 +262,7 @@ def convert_geojson_to_shapefile(geojson_features, out_shapefile, feature_type=" polygon_layer.CreateField(ogr.FieldDefn("updated", ogr.OFTString)) polygon_layer.CreateField(ogr.FieldDefn("CAUSE", ogr.OFTString)) polygon_layer.CreateField(ogr.FieldDefn("discovered", ogr.OFTString)) + polygon_layer.CreateField(ogr.FieldDefn("SUMMARY", ogr.OFTString)) elif feature_type == "lightning": # Create a single layer for lightning points point_layer = datasource.CreateLayer("lightning_points", geom_type=ogr.wkbPoint) @@ -295,6 +297,7 @@ def convert_geojson_to_shapefile(geojson_features, out_shapefile, feature_type=" feat.SetField("updated", feature["properties"]["updated"]) feat.SetField("discovered", feature["properties"]["discovered"]) feat.SetField("CAUSE", feature["properties"]["CAUSE"]) + feat.SetField("SUMMARY", feature["properties"]["SUMMARY"]) elif feature_type == "lightning": feat.SetField("amplitude", feature["properties"]["amplitude"]) From dd5580faa71efa9aa048b8a9e9aa55151e9e7b32 Mon Sep 17 00:00:00 2001 From: Bob Torgerson Date: Fri, 21 Mar 2025 09:02:11 -0800 Subject: [PATCH 3/3] Changes fire points and polygons from a shapefile to a Geopackage to allow for longer strings of characters. --- .../fire_layers/get_current_fire_layers.py | 135 ++++++++++++------ 1 file changed, 95 insertions(+), 40 deletions(-) diff --git a/wildfire_map/fire_layers/get_current_fire_layers.py b/wildfire_map/fire_layers/get_current_fire_layers.py index 9782e57..5b118cc 100644 --- a/wildfire_map/fire_layers/get_current_fire_layers.py +++ b/wildfire_map/fire_layers/get_current_fire_layers.py @@ -3,7 +3,7 @@ from datetime import datetime, timezone import json import argparse -from osgeo import ogr +from osgeo import ogr, gdal script_dir = os.path.dirname(__file__) data_dir = os.path.join(script_dir, "data", "debug") @@ -238,32 +238,10 @@ def convert_geojson_to_shapefile(geojson_features, out_shapefile, feature_type=" driver = ogr.GetDriverByName("ESRI Shapefile") datasource = driver.CreateDataSource(out_shapefile) - if feature_type == "fire": - # Create separate layers for points and polygons - point_layer = datasource.CreateLayer("fire_points", geom_type=ogr.wkbPoint) - polygon_layer = datasource.CreateLayer( - "fire_polygons", geom_type=ogr.wkbPolygon - ) + # Fire can't be a shapefile anymore due to the length of the SUMMARY field. + # Use convert_geojson_to_geopackage instead. - # Define fields for both layers - point_layer.CreateField(ogr.FieldDefn("NAME", ogr.OFTString)) - point_layer.CreateField(ogr.FieldDefn("acres", ogr.OFTReal)) - point_layer.CreateField(ogr.FieldDefn("active", ogr.OFTString)) - point_layer.CreateField(ogr.FieldDefn("OUTDATE", ogr.OFTString)) - point_layer.CreateField(ogr.FieldDefn("updated", ogr.OFTString)) - point_layer.CreateField(ogr.FieldDefn("CAUSE", ogr.OFTString)) - point_layer.CreateField(ogr.FieldDefn("discovered", ogr.OFTString)) - point_layer.CreateField(ogr.FieldDefn("SUMMARY", ogr.OFTString)) - - polygon_layer.CreateField(ogr.FieldDefn("NAME", ogr.OFTString)) - polygon_layer.CreateField(ogr.FieldDefn("acres", ogr.OFTReal)) - polygon_layer.CreateField(ogr.FieldDefn("active", ogr.OFTString)) - polygon_layer.CreateField(ogr.FieldDefn("OUTDATE", ogr.OFTString)) - polygon_layer.CreateField(ogr.FieldDefn("updated", ogr.OFTString)) - polygon_layer.CreateField(ogr.FieldDefn("CAUSE", ogr.OFTString)) - polygon_layer.CreateField(ogr.FieldDefn("discovered", ogr.OFTString)) - polygon_layer.CreateField(ogr.FieldDefn("SUMMARY", ogr.OFTString)) - elif feature_type == "lightning": + if feature_type == "lightning": # Create a single layer for lightning points point_layer = datasource.CreateLayer("lightning_points", geom_type=ogr.wkbPoint) @@ -289,17 +267,7 @@ def convert_geojson_to_shapefile(geojson_features, out_shapefile, feature_type=" feat = ogr.Feature(layer.GetLayerDefn()) feat.SetGeometry(geom) try: - if feature_type == "fire": - feat.SetField("NAME", feature["properties"]["NAME"]) - feat.SetField("acres", feature["properties"]["acres"]) - feat.SetField("active", feature["properties"]["active"]) - feat.SetField("OUTDATE", feature["properties"]["OUTDATE"]) - feat.SetField("updated", feature["properties"]["updated"]) - feat.SetField("discovered", feature["properties"]["discovered"]) - feat.SetField("CAUSE", feature["properties"]["CAUSE"]) - feat.SetField("SUMMARY", feature["properties"]["SUMMARY"]) - - elif feature_type == "lightning": + if feature_type == "lightning": feat.SetField("amplitude", feature["properties"]["amplitude"]) feat.SetField("hoursago", feature["properties"]["hoursago"]) @@ -311,6 +279,93 @@ def convert_geojson_to_shapefile(geojson_features, out_shapefile, feature_type=" datasource = None +def convert_geojson_to_geopackage(geojson_features, out_gpkg, feature_type="fire"): + driver = ogr.GetDriverByName("GPKG") + if os.path.exists(out_gpkg): + driver.DeleteDataSource(out_gpkg) + datasource = driver.CreateDataSource(out_gpkg) + + spatial_ref = ogr.osr.SpatialReference() + spatial_ref.ImportFromEPSG(4326) + + if feature_type == "fire": + point_layer = datasource.CreateLayer( + "fire_points", + srs=spatial_ref, + geom_type=ogr.wkbPoint, + options=["GEOMETRY_NAME=the_geom"], + ) + polygon_layer = datasource.CreateLayer( + "fire_polygons", + srs=spatial_ref, + geom_type=ogr.wkbPolygon, + options=["GEOMETRY_NAME=the_geom"], + ) + + for layer in [point_layer, polygon_layer]: + layer.CreateField(ogr.FieldDefn("NAME", ogr.OFTString)) + layer.CreateField(ogr.FieldDefn("acres", ogr.OFTReal)) + layer.CreateField(ogr.FieldDefn("active", ogr.OFTString)) + layer.CreateField(ogr.FieldDefn("OUTDATE", ogr.OFTString)) + layer.CreateField(ogr.FieldDefn("updated", ogr.OFTString)) + layer.CreateField(ogr.FieldDefn("CAUSE", ogr.OFTString)) + layer.CreateField(ogr.FieldDefn("discovered", ogr.OFTString)) + field = ogr.FieldDefn("SUMMARY", ogr.OFTString) + field.SetWidth(1000) + layer.CreateField(field) + + elif feature_type == "lightning": + point_layer = datasource.CreateLayer( + "lightning_points", + srs=spatial_ref, + geom_type=ogr.wkbPoint, + options=["GEOMETRY_NAME=the_geom"], + ) + point_layer.CreateField(ogr.FieldDefn("amplitude", ogr.OFTReal)) + point_layer.CreateField(ogr.FieldDefn("hoursago", ogr.OFTReal)) + + elif feature_type == "viirs": + point_layer = datasource.CreateLayer( + "viirs_hotspots", + srs=spatial_ref, + geom_type=ogr.wkbPoint, + options=["GEOMETRY_NAME=the_geom"], + ) + + for feature in geojson_features: + geom_type = feature["geometry"]["type"] + geom = ogr.CreateGeometryFromJson(json.dumps(feature["geometry"])) + + if geom_type == "Point": + layer = point_layer + elif geom_type in ["Polygon", "MultiPolygon"]: + layer = polygon_layer + else: + continue + + feat = ogr.Feature(layer.GetLayerDefn()) + feat.SetGeometry(geom) + + try: + if feature_type == "fire": + feat.SetField("NAME", feature["properties"].get("NAME", "")) + feat.SetField("acres", feature["properties"].get("acres", 0)) + feat.SetField("active", feature["properties"].get("active", "")) + feat.SetField("OUTDATE", feature["properties"].get("OUTDATE", "")) + feat.SetField("updated", feature["properties"].get("updated", "")) + feat.SetField("discovered", feature["properties"].get("discovered", "")) + feat.SetField("CAUSE", feature["properties"].get("CAUSE", "N/A")) + feat.SetField("SUMMARY", feature["properties"].get("SUMMARY", "")) + elif feature_type == "lightning": + feat.SetField("amplitude", feature["properties"].get("amplitude", 0)) + feat.SetField("hoursago", feature["properties"].get("hoursago", 0)) + layer.CreateFeature(feat) + except Exception as e: + print(f"Error adding feature to the layer: {e}") + + datasource = None + + if __name__ == "__main__": # TODO Need a way to pass in the working directory to work with Prefect parser = argparse.ArgumentParser( @@ -324,11 +379,11 @@ def convert_geojson_to_shapefile(geojson_features, out_shapefile, feature_type=" ) args = parser.parse_args() - # Creates fire shapefiles - out_shapefile = os.path.join(args.out_dir, "fires.shp") + # Creates fire GeoPackage <-- required due to length of SUMMARY field + out_shapefile = os.path.join(args.out_dir, "fires.gpkg") fire_geojson = fetch_fire_geojson() if fire_layers_update_failed is False: - convert_geojson_to_shapefile(fire_geojson, out_shapefile, "fire") + convert_geojson_to_geopackage(fire_geojson, out_shapefile, "fire") # Creates lightning shapefile out_shapefile = os.path.join(args.out_dir, "lightning.shp")