From 385638d55362ac20ebcb16c63ca82aa44bef4ea4 Mon Sep 17 00:00:00 2001 From: davidwyld <24752124+davidwilby@users.noreply.github.com> Date: Thu, 18 Dec 2025 16:01:15 +0000 Subject: [PATCH] wip towards tile serving for meshes using martin --- CHANGELOG.md | 1 + Dockerfile | 6 +- compose.yml | 19 +- martin/config.yaml | 263 ++++++++++++++++++ martin/style/mesh.json | 76 +++++ polarrouteserver/route_api/admin.py | 7 +- polarrouteserver/route_api/apps.py | 3 + .../migrations/0017_mesh_geometry.py | 19 ++ .../0018_remove_mesh_geometry_meshpolygon.py | 31 +++ polarrouteserver/route_api/models.py | 49 +++- polarrouteserver/route_api/signals.py | 28 ++ .../route_api/templates/index.html | 189 +++++++++++++ polarrouteserver/settings/base.py | 70 +++-- polarrouteserver/urls.py | 2 + postgis-init-scripts/01_functions.sql | 29 ++ 15 files changed, 762 insertions(+), 30 deletions(-) create mode 100644 martin/config.yaml create mode 100644 martin/style/mesh.json create mode 100644 polarrouteserver/route_api/migrations/0017_mesh_geometry.py create mode 100644 polarrouteserver/route_api/migrations/0018_remove_mesh_geometry_meshpolygon.py create mode 100644 polarrouteserver/route_api/signals.py create mode 100644 polarrouteserver/route_api/templates/index.html create mode 100644 postgis-init-scripts/01_functions.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index dd0d43e8..0d764067 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - added ensure_adminuser command to add subtly more sophisticated behaviour to Django's createsuperuser - i.e. don't raise non-zero exit code if superuser already exists, add more useful output. - Use uv in the docker image. - Empty arrays to empty responses for a consistent response structure. +- Experimental feature: mesh tile serving using [Martin](https://github.com/maplibre/martin). ### Changed - Inappropriate use of 204 code: RecentRoutesView changed from 204 to 200 OK with an empty array and the original message ("No recent routes found for today."). diff --git a/Dockerfile b/Dockerfile index 859396c5..4c42f062 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,10 +6,12 @@ ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 ENV DJANGO_SETTINGS_MODULE=polarrouteserver.settings.development -# Install GDAL - used by Fiona RUN apt-get update && apt-get install -y \ gdal-bin \ libgdal-dev \ + binutils\ + libproj-dev \ + postgresql-client \ && rm -rf /var/lib/apt/lists/* ENV GDAL_CONFIG=/usr/bin/gdal-config @@ -22,4 +24,4 @@ COPY pyproject.toml manage.py /usr/src/app/ COPY polarrouteserver /usr/src/app/polarrouteserver RUN uv pip install --system -e . -RUN uv pip install --system django-debug-toolbar \ No newline at end of file +RUN uv pip install --system django-debug-toolbar diff --git a/compose.yml b/compose.yml index 88f74e6a..b36d8c20 100644 --- a/compose.yml +++ b/compose.yml @@ -11,6 +11,7 @@ services: POLARROUTE_DB_PORT: 5432 POLARROUTE_SUPERUSER_USERNAME: admin POLARROUTE_SUPERUSER_PASSWORD: password + POLARROUTE_LOG_LEVEL: INFO entrypoint: /usr/src/app/docker-entrypoint.sh command: ["python", "manage.py", "runserver", "0.0.0.0:8000"] volumes: @@ -22,7 +23,7 @@ services: - celery db: - image: postgres:16 + image: postgis/postgis restart: always environment: POSTGRES_USER: polarroute @@ -31,6 +32,7 @@ services: PGDATA: /var/lib/postgresql/data/pgdata volumes: - db-data:/var/lib/postgresql/data + - ./postgis-init-scripts/:/docker-entrypoint-initdb.d ports: - 5432:5432 @@ -55,6 +57,21 @@ services: - 5672:5672 - 15672:15672 + martin: + image: ghcr.io/maplibre/martin + restart: unless-stopped + command: ["--config", "/config/config.yaml"] + environment: + # DATABASE_URL: postgres://polarroute@db:5432/polarroute + PGPASSWORD: polarroute + RUST_LOG: debug + ports: + - "3000:3000" + volumes: + - ./martin:/config + depends_on: + - db + swagger: profiles: ['swagger'] image: swaggerapi/swagger-ui diff --git a/martin/config.yaml b/martin/config.yaml new file mode 100644 index 00000000..171ef3c1 --- /dev/null +++ b/martin/config.yaml @@ -0,0 +1,263 @@ +web_ui: enable-for-all +base_path: /tiles +postgres: + # Database connection string. + # + # You can use environment variables too, for example: + # connection_string: $DATABASE_URL + # connection_string: ${DATABASE_URL:-postgres://postgres@localhost/db} + connection_string: 'postgres://polarroute:polarroute@db:5432/polarroute?sslmode=disable' + + # Same as PGSSLCERT for psql + # ssl_cert: './postgresql.crt' + # Same as PGSSLKEY for psql + # ssl_key: './postgresql.key' + # Same as PGSSLROOTCERT for psql + # ssl_root_cert: './root.crt' + + # If a spatial table has SRID 0, then this SRID will be used as a fallback + default_srid: 4326 + + # Maximum Postgres connections pool size [default: 20] + pool_size: 20 + + # Limit the number of geo features per tile. + # + # If the source table has more features than set here, they will not be + # included in the tile and the result will look "cut off"/incomplete. + # This feature allows you to put a maximum latency bound on tiles with an + # extreme amount of detail at the cost of not returning all data. + # It is sensible to set this limit if you have user generated/untrusted + # geodata, e.g. a lot of data points at [Null Island] + # (https://en.wikipedia.org/wiki/Null_Island). + max_feature_count: null # either a positive integer, or null=unlimited (default) + + # Specify how bounds should be computed for the spatial PG tables [default: quick] + # + # Options: + # - `calc` compute table geometry bounds on startup. + # - `quick` same as 'calc', but the calculation will be aborted after 5 seconds. + # - `skip` does not compute table geometry bounds on startup. + auto_bounds: quick + + # Enable automatic discovery of tables and functions. + # You may set this to `false` to disable. + auto_publish: false + # # Optionally limit to just these schemas + # from_schemas: + # - public + # - my_schema + # Here we enable both tables and functions auto discovery. + # You can also enable just one of them by not mentioning the other, or + # setting it to false. Setting one to true disables the other one as well. + # E.g. `tables: false` enables just the functions auto-discovery. + # tables: + # Optionally set how source ID should be generated based on the table's name, + # schema, and geometry column + # source_id_format: 'table.{schema}.{table}.{column}' + # Add more schemas to the ones listed above + # from_schemas: my_other_schema + # A table column to use as the feature ID + # If a table has no column with this name, `id_column` will not be set for + # that table. + # If a list of strings is given, the first found column will be treated as a + # feature ID. + # id_columns: feature_id + # Controls if geometries should be clipped or encoded as is [default: true] + # clip_geom: true + # Buffer distance in tile coordinate space to optionally clip geometries, + # optional, default to 64 + # buffer: 64 + # Tile extent in tile coordinate space, optional, default to 4096 + # extent: 4096 + # functions: + # # Optionally limit to just these schemas + # from_schemas: + # - public + # - my_schema + # # Optionally set how source ID should be generated based on the function's + # # name and schema + # source_id_format: '{schema}.{function}' + + # Associative arrays of table sources + # tables: + # mesh: + # # ID of the MVT layer (optional, defaults to table name) + # layer_id: mesh + + # # Table schema (required) + # schema: public + + # # Table name (required) + # table: route_api_mesh + + # # Geometry SRID (required) + # srid: 4326 + + # # Geometry column name (required) + # geometry_column: geometry + + # # Feature id column name + # id_column: id + + # # An integer specifying the minimum zoom level + # minzoom: 0 + + # # An integer specifying the maximum zoom level. MUST be >= minzoom + # maxzoom: 30 + + # # The maximum extent of available map tiles. Bounds MUST define an area + # # covered by all zoom levels. The bounds are represented in WGS:84 latitude + # # and longitude values, in the order left, bottom, right, top. Values may + # # be integers or floating point numbers. + # bounds: [ -180.0, -90.0, 180.0, 90.0 ] + + # # Tile extent in tile coordinate space + # extent: 4096 + + # # Buffer distance in tile coordinate space to optionally clip geometries + # buffer: 64 + + # # Boolean to control if geometries should be clipped or encoded as is + # clip_geom: true + + # # Geometry type + # geometry_type: MULTIPOLYGON + + # # List of columns, that should be encoded as tile properties (required) + # # + # # Keys and values are the names and descriptions of attributes available in this layer. + # # Each value (description) must be a string that describes the underlying data. + # # If no fields (=just the geometry) should be encoded, an empty object is allowed. + # properties: + # id: int4 + + # Associative arrays of function sources + functions: + mesh: + # Schema name (required) + schema: public + + # Function name (required) + function: get_mesh_zxy + + # An integer specifying the minimum zoom level + minzoom: 0 + + # An integer specifying the maximum zoom level. MUST be >= minzoom + maxzoom: 30 + + # The maximum extent of available map tiles. Bounds MUST define an area + # covered by all zoom levels. The bounds are represented in WGS:84 + # latitude and longitude values, in the order left, bottom, right, top. + # Values may be integers or floating point numbers. + bounds: [ -180.0, -90.0, 180.0, 90.0 ] + +# Publish PMTiles files from local disk or proxy to a web server +# pmtiles: +# # Size of the directory cache (in MB). +# # Defaults to cache_size_mb / 4 +# # +# # Note: +# # Tile and directory caching are complementary. +# # For good performance, you want +# # - directory caching (to not resolve the directory on each request) and +# # - tile caching (for high access tiles) +# directory_cache_size_mb: 128 + +# # You can pass options for pmtiles files located on remote storages here. +# # +# # The avaliable options are documented here: +# # - local file sources don't have options +# # - Http(s) Source: https://docs.rs/object_store/latest/object_store/http/struct.HttpBuilder.html +# # - Amazon S3: https://docs.rs/object_store/latest/object_store/aws/struct.AmazonS3Builder.html +# # - Google Cloud Storage: https://docs.rs/object_store/latest/object_store/gcp/struct.GoogleCloudStorageBuilder.html +# # - Microsoft Azure: https://docs.rs/object_store/latest/object_store/azure/struct.MicrosoftAzureBuilder.html +# # +# # Example for configuring a source to allow http +# allow_http: true + +# paths: +# # scan this whole dir, matching all *.pmtiles files +# - /dir-path +# # specific pmtiles file will be published as a pmt source (filename without extension) +# - /path/to/pmt.pmtiles +# # A web server with a PMTiles file that supports range requests +# - https://example.org/path/tiles.pmtiles +# sources: +# # named source matching source name to a single file +# pm-src1: /path/to/pmt.pmtiles +# # A named source to a web server with a PMTiles file that supports range requests +# pm-web2: https://example.org/path/tiles.pmtiles + +# Publish MBTiles files +# mbtiles: +# paths: +# # scan this whole dir, matching all *.mbtiles files +# - /dir-path +# # specific mbtiles file will be published as mbtiles2 source +# - /path/to/mbtiles.mbtiles +# sources: +# # named source matching source name to a single file +# mb-src1: /path/to/mbtiles1.mbtiles + +# # Sprite configuration +# sprites: +# # Size of the sprite cache (in MB). +# # Defaults to cache_size_mb / 8 +# cache_size_mb: 64 + +# paths: +# # all SVG files in this dir will be published as a "my_images" sprite source +# - /path/to/my_images +# sources: +# # SVG images in this directory will be published as a "my_sprites" sprite source +# my_sprites: /path/to/some_dir + +# Font configuration +# fonts: +# # Size of the sprite cache (in MB). +# # Defaults to cache_size_mb / 4 +# cache_size_mb: 64 + +# # A list of *.otf, *.ttf, and *.ttc font files and dirs to search recursively. +# paths: +# - /path/to/font/file.ttf +# - /path/to/font_dir + +# Publish MapLibre style files +# In the future, the style files will be used for the server-side rendering as well +styles: + paths: + # publish all *.json files in this directory + # The name of the file will be used as the style name + - /config/style/ + # sources: + # # publish a JSON file found at this path as `some_style_name` + # # + # # Contrairy to paths, if directories are specified, Martin will print a + # # warning and ignore them. + # # To serve a style-directory, use the `paths` section above or name each + # # style individually. This prevents footguns with names being unclear. + # some_style_name: /path/to/this/style.json + # # Publish specific file as `other_style_name` + # other_style_name: /path/to/other_style.json + +# If set, the version of the tileset (as specified in the MBTiles or PMTiles metadata) +# will be embedded in the TileJSON `tiles` URL, with the set identifier. +# This is useful to give clients a better way to cache-bust a CDN: +# 1. maplibre requests tilejson, tilejson contains the tiles URL. This is always up-to-date. +# 2. maplibre requests each tile it requires, with the tiles URL in the tilejson. +# 3. Add `Control: public, max-age=..., immutable` on the tile responses +# optimize browser/CDN cache hit rates, while also making sure that +# old tiles aren't served when a new tileset is deployed. +# +# The CDN must handle query parameters for caching to work correctly. +# Many CDNs ignore them by default. +# +# For example, if +# - the setting here is `version`, and +# - the PMTiles tileset version is `1.0.0`, the +# TileJSON will be: +# { ..., "tiles": [".../{z}/{x}/{y}?version=1.0.0"], ... } +tilejson_url_version_param: null # a string, such as `version` or `v` diff --git a/martin/style/mesh.json b/martin/style/mesh.json new file mode 100644 index 00000000..c12d920f --- /dev/null +++ b/martin/style/mesh.json @@ -0,0 +1,76 @@ +{ + "version": 8, + "name": "PolarRoute Mesh Style", + "metadata": { + "mapbox:autocomposite": true + }, + "sources": { + "mesh": { + "type": "vector", + + "tiles": [ + "http://localhost:3000/mesh/{z}/{x}/{y}?mesh_id=5" + ], + + "minzoom": 0, + "maxzoom": 14 + } + }, + "layers": [ + + { + "id": "mesh-fill", + "type": "fill", + "source": "mesh", + "source-layer": "get_mesh_zxy", + "paint": { + "fill-color": [ + "case", + ["boolean", ["get", "land"], true], + "#646464", + ["step", + ["get", "SIC"], + "#4590f7", + 10, "#1e90ff", + 20, "#1efaa0", + 30, "#228b22", + 40, "#00fa00", + 50, "#7dfa00", + 60, "#adff2f", + 70, "#fafa00", + 80, "#fa7d00", + 85, "#fa0000", + 90, "#ba55d3", + 95, "#9400d3", + 99, "#78005a"] + ], + "fill-opacity": [ + "step", + ["get", "SIC"], + 0, 9, + 1.0 + ] + } + }, + { + "id": "mesh-outline", + "type": "line", + "source": "mesh", + "source-layer": "get_mesh_zxy", + "minzoom": 0, + "filter": ["==", "$type", "Polygon"], + "paint": { + "line-color": "#111111", + "line-width": [ + "interpolate", + ["linear"], + ["zoom"], + 10, 0.5, + 16, 2 + ] + } + } + + ] +} + diff --git a/polarrouteserver/route_api/admin.py b/polarrouteserver/route_api/admin.py index ddd590bd..a1612832 100644 --- a/polarrouteserver/route_api/admin.py +++ b/polarrouteserver/route_api/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from .models import Vehicle, Route, Mesh, Job, Location +from .models import Vehicle, Route, Mesh, Job, Location, MeshPolygon LIST_PER_PAGE = 20 @@ -95,6 +95,10 @@ def get_queryset(self, request): queryset = super().get_queryset(request) return queryset.defer("json") +class MeshPolygonAdmin(admin.ModelAdmin): + list_display = [ + "mesh_id", + ] class LocationAdmin(admin.ModelAdmin): list_display = [ @@ -113,3 +117,4 @@ class LocationAdmin(admin.ModelAdmin): admin.site.register(Mesh, MeshAdmin) admin.site.register(Job, JobAdmin) admin.site.register(Location, LocationAdmin) +admin.site.register(MeshPolygon, MeshPolygonAdmin) \ No newline at end of file diff --git a/polarrouteserver/route_api/apps.py b/polarrouteserver/route_api/apps.py index 81a2b89c..85cd2e0e 100644 --- a/polarrouteserver/route_api/apps.py +++ b/polarrouteserver/route_api/apps.py @@ -4,3 +4,6 @@ class BackendConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "polarrouteserver.route_api" + + def ready(self): + import polarrouteserver.route_api.signals diff --git a/polarrouteserver/route_api/migrations/0017_mesh_geometry.py b/polarrouteserver/route_api/migrations/0017_mesh_geometry.py new file mode 100644 index 00000000..7c604c0b --- /dev/null +++ b/polarrouteserver/route_api/migrations/0017_mesh_geometry.py @@ -0,0 +1,19 @@ +# Generated by Django 5.2.9 on 2025-12-03 13:31 + +import django.contrib.gis.db.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('route_api', '0016_alter_location_name'), + ] + + operations = [ + migrations.AddField( + model_name='mesh', + name='geometry', + field=django.contrib.gis.db.models.fields.MultiPolygonField(null=True, srid=4326), + ), + ] diff --git a/polarrouteserver/route_api/migrations/0018_remove_mesh_geometry_meshpolygon.py b/polarrouteserver/route_api/migrations/0018_remove_mesh_geometry_meshpolygon.py new file mode 100644 index 00000000..ac0f9095 --- /dev/null +++ b/polarrouteserver/route_api/migrations/0018_remove_mesh_geometry_meshpolygon.py @@ -0,0 +1,31 @@ +# Generated by Django 5.2.9 on 2025-12-16 15:15 + +import django.contrib.gis.db.models.fields +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('route_api', '0017_mesh_geometry'), + ] + + operations = [ + migrations.RemoveField( + model_name='mesh', + name='geometry', + ), + migrations.CreateModel( + name='MeshPolygon', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('properties', models.JSONField()), + ('geometry', django.contrib.gis.db.models.fields.GeometryField(srid=3857)), + ('mesh', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='route_api.mesh')), + ], + options={ + 'indexes': [models.Index(fields=['mesh'], name='route_api_m_mesh_id_80e8f5_idx')], + }, + ), + ] diff --git a/polarrouteserver/route_api/models.py b/polarrouteserver/route_api/models.py index dd095de3..0e867a20 100644 --- a/polarrouteserver/route_api/models.py +++ b/polarrouteserver/route_api/models.py @@ -1,7 +1,8 @@ import logging from celery.result import AsyncResult -from django.db import models +from django.contrib.gis.db import models +from django.contrib.gis.geos import GEOSGeometry from django.utils import timezone from polarrouteserver.celery import app @@ -32,6 +33,52 @@ def size(self) -> float: class Meta: verbose_name_plural = "Meshes" +class MeshPolygon(models.Model): + default_srid=3857 + mesh = models.ForeignKey(Mesh, on_delete=models.CASCADE) + properties = models.JSONField() + geometry = models.GeometryField(srid=default_srid) + # use 3857: web mercator as the default projection + # we will store based on the most appropriate projection for a given mesh, + # eg 3031: antarctic stereographic for south polar + # or 3431 for north polar + + @classmethod + def create_from_mesh(cls, mesh_id:int): + # NOTE: This method is currently only called from the corresponding signal from a mesh save, see signals.py module. + + source_srid = 4326 + + # get mesh object + try: + mesh = Mesh.objects.get(id=mesh_id) + except Mesh.DoesNotExist: + raise Exception + + # delete previous polygons for this mesh + try: + cls.objects.get(mesh=mesh).delete() + except MeshPolygon.DoesNotExist: + pass + + # create new mesh polygons + new_polygons = [] + for feature in mesh.json['cellboxes']: + geometry = feature.pop('geometry') + properties = feature + + new_polygons.append(cls( + mesh=mesh, + properties=properties, + geometry = GEOSGeometry(geometry, source_srid).transform(cls.default_srid, clone=True), + )) + + cls.objects.bulk_create(new_polygons) + + class Meta: + indexes = [ + models.Index(fields=['mesh']), + ] class Vehicle(models.Model): # Required properties diff --git a/polarrouteserver/route_api/signals.py b/polarrouteserver/route_api/signals.py new file mode 100644 index 00000000..1d0be6e8 --- /dev/null +++ b/polarrouteserver/route_api/signals.py @@ -0,0 +1,28 @@ +import logging + +from django.db.models.signals import post_save +from django.dispatch import receiver +from polarrouteserver.route_api.models import Mesh, MeshPolygon + +logger = logging.getLogger(__name__) + +@receiver(post_save, sender=Mesh) +def create_mesh_polygons_on_mesh_save(sender, instance, created, raw, using, update_fields, **kwargs): + + print( + f"sender: {sender} \n" + f"instance: {instance} \n" + f"created: {created} \n" + f"raw: {raw} \n" + f"using: {using} \n" + f"update_fields: {update_fields}" + ) + + + if not instance.json: + logger.warning(f"Mesh id: {instance.id} saved with no json. Won't create mesh polygons.") + return + + # if newly created, or json has been updated + if created is True or (update_fields is not None and "json" in update_fields): + MeshPolygon.create_from_mesh(instance.id) \ No newline at end of file diff --git a/polarrouteserver/route_api/templates/index.html b/polarrouteserver/route_api/templates/index.html new file mode 100644 index 00000000..71364ba4 --- /dev/null +++ b/polarrouteserver/route_api/templates/index.html @@ -0,0 +1,189 @@ + + + + +