diff --git a/data/observational/queries.py b/data/observational/queries.py index ffcad6a2c..8b58f3125 100644 --- a/data/observational/queries.py +++ b/data/observational/queries.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Callable, Dict, List, Optional, Tuple -from sqlalchemy import and_, func +from sqlalchemy import and_, func, select from sqlalchemy.orm import Session, joinedload from . import DataType, Platform, PlatformMetadata, Sample, Station, engine @@ -134,7 +134,7 @@ def get_platform_tracks( the optional query filters. """ funcs = __db_funcs() - query = session.query( + query = select( Platform.id, Platform.type, func.avg(Station.longitude), @@ -160,9 +160,11 @@ def get_platform_tracks( endtime=endtime, ) - query = query.group_by(Platform.id, funcs[quantum](Station.time)) + query = query.group_by(Platform.id, funcs[quantum](Station.time)).order_by( + Platform.id, funcs[quantum](Station.time) + ) - return query.order_by(Platform.id, funcs[quantum](Station.time)).all() + return session.execute(query).all() def get_platform_track( @@ -326,7 +328,13 @@ def __build_station_query( meta_key=None, meta_value=None, ): - query = session.query(Station) + query = select( + Platform.type, + Station.id, + Station.name, + Station.latitude, + Station.longitude, + ).join(Station) # Use index hint query = query.with_hint(Station, "USE INDEX (idx_stations_time)") @@ -378,7 +386,7 @@ def get_stations( """ Queries for stations, given the optional query filters. """ - return __build_station_query( + query = __build_station_query( session=session, variable=variable, mindepth=mindepth, @@ -392,7 +400,9 @@ def get_stations( platform_types=platform_types, meta_key=meta_key, meta_value=meta_value, - ).all() + ) + + return session.execute(query).all() def get_station_time_range(session: Session): diff --git a/oceannavigator/frontend/src/components/Map/Map.jsx b/oceannavigator/frontend/src/components/Map/Map.jsx index 05dd8d2a8..f8df40a11 100644 --- a/oceannavigator/frontend/src/components/Map/Map.jsx +++ b/oceannavigator/frontend/src/components/Map/Map.jsx @@ -827,14 +827,27 @@ const Map = forwardRef((props, ref) => { let url = ""; let extent = mapView.calculateExtent(map0.getSize()); let resolution = mapView.getResolution(); + let prevFeatures; switch (featureType) { case "observation_points": + prevFeatures = featureVectorSource.getFeatures(); + prevFeatures = prevFeatures.filter((feature) => feature.get("class") === "observation") + featureVectorSource.removeFeatures(prevFeatures) + url = `/api/v2.0/observation/point/` + `${featureId}.json`; break; case "observation_tracks": + prevFeatures = featureVectorSource.getFeatures(); + prevFeatures = prevFeatures.filter((feature) => feature.get("class") === "observation") + featureVectorSource.removeFeatures(prevFeatures) + url = `/api/v2.0/observation/track/` + `${featureId}.json`; break; case "class4": + prevFeatures = featureVectorSource.getFeatures(); + prevFeatures = prevFeatures.filter((feature) => feature.get("type") === "class4") + featureVectorSource.removeFeatures(prevFeatures) + url = `/api/v2.0/class4` + `/${props.class4Type}` + diff --git a/oceannavigator/frontend/src/components/Map/utils.js b/oceannavigator/frontend/src/components/Map/utils.js index 893473f15..d773e1e02 100644 --- a/oceannavigator/frontend/src/components/Map/utils.js +++ b/oceannavigator/frontend/src/components/Map/utils.js @@ -49,13 +49,17 @@ const I9 = require("../../images/s111/I9.svg").default; const arrowImages = [I0, I1, I2, I3, I4, I5, I6, I7, I8, I9]; +const OBS_COLORS = { + argo: [255, 0, 0], + mission: [255, 255, 0], + drifter: [0, 255, 0], + glider: [0, 255, 255], + animal: [0, 0, 255], +}; + const COLORS = [ - [0, 0, 255], [0, 128, 0], - [255, 0, 0], - [0, 255, 255], [255, 0, 255], - [255, 255, 0], [0, 0, 0], [255, 255, 255], ]; @@ -78,7 +82,7 @@ export const getBasemap = ( source, projection, attribution, - topoShadedRelief + topoShadedRelief, ) => { switch (source) { case "topo": @@ -129,13 +133,13 @@ export const createMap = ( layerFeatureVector, obsDrawSource, maxZoom, - mapRef + mapRef, ) => { const newLayerBasemap = getBasemap( mapSettings.basemap, mapSettings.projection, mapSettings.basemap_attribution, - mapSettings.topoShadedRelief + mapSettings.topoShadedRelief, ); const vectorTileGrid = new olTilegrid.createXYZ({ @@ -260,7 +264,7 @@ export const createMap = ( mapObject.getEventPixel(e.originalEvent), function (feature, layer) { return feature; - } + }, ); if (feature && feature.get("name")) { overlay.setPosition(e.coordinate); @@ -286,7 +290,7 @@ export const createMap = ( {bearing} )} - + , ); } else { popupElement.current.innerHTML = feature.get("name"); @@ -313,8 +317,8 @@ export const createMap = ( olProj.transform( f.get("centroid"), "EPSG:4326", - mapSettings.projection - ) + mapSettings.projection, + ), ), text: new Text({ text: f.get("name"), @@ -355,8 +359,8 @@ export const createMap = ( {response.data[key]} ))} - - ) + , + ), ); popupElement.current.innerHTML = feature.get("meta"); }) @@ -416,7 +420,7 @@ export const createFeatureVectorLayer = (source, mapSettings) => { style: function (feat, res) { if (feat.get("class") == "observation") { if (feat.getGeometry() instanceof olgeom.LineString) { - let color = drifter_color[feat.get("id")]; + let color = OBS_COLORS[feat.get("type")]; if (color === undefined) { color = COLORS[Object.keys(drifter_color).length % COLORS.length]; @@ -429,6 +433,12 @@ export const createFeatureVectorLayer = (source, mapSettings) => { width: 8, }), }), + new Style({ + stroke: new Stroke({ + color: "#555555", + width: isMobile ? 6 : 4, + }), + }), new Style({ stroke: new Stroke({ color: color, @@ -452,11 +462,12 @@ export const createFeatureVectorLayer = (source, mapSettings) => { }); let stroke = new Stroke({ color: "#000000", width: 1 }); let radius = isMobile ? 9 : 6; - switch (feat.get("type")) { + let featureType = feat.get("type"); + switch (featureType) { case "argo": image = new Circle({ radius: isMobile ? 6 : 4, - fill: new Fill({ color: "#ff0000" }), + fill: new Fill({ color: OBS_COLORS[featureType] }), stroke: stroke, }); break; @@ -464,7 +475,7 @@ export const createFeatureVectorLayer = (source, mapSettings) => { image = new RegularShape({ points: 3, radius: radius, - fill: new Fill({ color: "#ffff00" }), + fill: new Fill({ color: OBS_COLORS[featureType] }), stroke: stroke, }); break; @@ -472,7 +483,7 @@ export const createFeatureVectorLayer = (source, mapSettings) => { image = new RegularShape({ points: 4, radius: radius, - fill: new Fill({ color: "#00ff00" }), + fill: new Fill({ color: OBS_COLORS[featureType] }), stroke: stroke, }); break; @@ -480,7 +491,7 @@ export const createFeatureVectorLayer = (source, mapSettings) => { image = new RegularShape({ points: 5, radius: radius, - fill: new Fill({ color: "#00ffff" }), + fill: new Fill({ color: OBS_COLORS[featureType] }), stroke: stroke, }); break; @@ -488,7 +499,7 @@ export const createFeatureVectorLayer = (source, mapSettings) => { image = new RegularShape({ points: 6, radius: radius, - fill: new Fill({ color: "#0000ff" }), + fill: new Fill({ color: OBS_COLORS[featureType] }), stroke: stroke, }); break; @@ -520,7 +531,7 @@ export const createFeatureVectorLayer = (source, mapSettings) => { feat, "#000", "#ffffff", - mapSettings + mapSettings, ); if (textStyle) styles.push(textStyle); @@ -544,7 +555,7 @@ export const createFeatureVectorLayer = (source, mapSettings) => { feat, "#000", "#ffffff", - mapSettings + mapSettings, ); if (textStyle) styles.push(textStyle); return styles; @@ -568,7 +579,7 @@ export const createFeatureVectorLayer = (source, mapSettings) => { feat, "#000", "#ffffff", - mapSettings + mapSettings, ); if (textStyle) styles.push(textStyle); return styles; @@ -654,7 +665,7 @@ export const createFeatureVectorLayer = (source, mapSettings) => { const red = Math.min(255, 255 * (feat.get("error_norm") / 0.5)); const green = Math.min( 255, - (255 * (1 - feat.get("error_norm"))) / 0.5 + (255 * (1 - feat.get("error_norm"))) / 0.5, ); return new Style({ @@ -680,7 +691,7 @@ export const createFeatureTextStyle = ( feature, textColor = "#000", strokeColor = "#ffffff", - mapSettings = null + mapSettings = null, ) => { if (!feature.get("name")) return null; @@ -702,8 +713,8 @@ export const createFeatureTextStyle = ( olProj.transform( feature.get("centroid"), "EPSG:4326", - mapSettings.projection - ) + mapSettings.projection, + ), ); } @@ -821,7 +832,7 @@ export const createPlotData = (selected, projection) => { let id = selected[0].getId(); let name = selected.map((f) => f.get("name")); let coordinates = selected.map((feature) => - feature.getGeometry().getCoordinates() + feature.getGeometry().getCoordinates(), ); // Observations if (selected[0].get("class") === "observation") { @@ -853,7 +864,7 @@ export const createPlotData = (selected, projection) => { title = selected.map((feature, idx) => feature.get("name") ? feature.get("name") - : `${formatLatLon(coordinates[idx][0], coordinates[idx][1])}` + : `${formatLatLon(coordinates[idx][0], coordinates[idx][1])}`, ); title = "Point - " + title.join(", "); } else if (type === "LineString") { diff --git a/oceannavigator/frontend/src/components/data-selectors/TimeSlider.jsx b/oceannavigator/frontend/src/components/data-selectors/TimeSlider.jsx index ab268c0e4..363f08f9c 100644 --- a/oceannavigator/frontend/src/components/data-selectors/TimeSlider.jsx +++ b/oceannavigator/frontend/src/components/data-selectors/TimeSlider.jsx @@ -1,10 +1,4 @@ -import React, { - useState, - useEffect, - useRef, - useMemo, - memo, -} from "react"; +import React, { useState, useEffect, useRef, useMemo, memo } from "react"; import Button from "react-bootstrap/Button"; import OverlayTrigger from "react-bootstrap/OverlayTrigger"; import Tooltip from "react-bootstrap/Tooltip"; @@ -88,16 +82,6 @@ function TimeSlider({ id, dataset, timestamps, selected, onChange }) { return () => clearInterval(interval); }, [scrollSpeed]); - useEffect(() => { - document.addEventListener("mousemove", handleThumbMousemove); - document.addEventListener("mouseup", handleThumbMouseup); - document.addEventListener("mouseleave", handleThumbMouseup); - return () => { - document.removeEventListener("mousemove", handleThumbMousemove); - document.removeEventListener("mouseup", handleThumbMouseup); - document.removeEventListener("mouseleave", handleThumbMouseup); - }; - }, [timestamps, tickWidth]); useEffect(() => { if (contentRef.current && scrollTrackRef.current) { @@ -158,7 +142,7 @@ function TimeSlider({ id, dataset, timestamps, selected, onChange }) { const updateTickContainerWidth = () => { if (!scrollTrackRef.current || timestamps.length === 0) return; - let minTickWidth = 70; + let minTickWidth = 105; if (dataset.quantum === "hour" && dataset.id !== "giops_day") { minTickWidth = 35; } @@ -427,7 +411,12 @@ function TimeSlider({ id, dataset, timestamps, selected, onChange }) { )); return ( -
+
{leftButtons}
); - let observationVariableSelector = null; + let observationInputs = null; if (plotData.observation) { - if (typeof plotData.id === "number") { - observationVariableSelector = ( - setObservationVariable(value)} - alwaysShow={true} - /> - ); - } else { - const data = plotData.coordinates[0][2].datatypes.map((o, i) => ({ - id: i, - value: o.replace(/ \[.*\]/, ""), - })); - observationVariableSelector = ( - setObservationVariable(value)} - alwaysShow={true} - /> - ); - } + let obsOptions = + typeof plotData.id === "number" + ? observationVariables.data + : plotData.coordinates[0][2].datatypes.map((o, i) => ({ + id: i, + value: o.replace(/ \[.*\]/, ""), + })); + observationInputs = ( + + {_("Observation Settings")} + + setObservationVariable(value)} + alwaysShow={true} + /> + + + ); } const inputs = ( <> - {observationVariableSelector} - setShowMap(value)} - title={_("Show Location")} - > - {_("showmap_help")} - + {selected !== TabEnum.OBSERVATION && ( + setShowMap(value)} + title={_("Show Location")} + > + {_("showmap_help")} + + )} {/* {plotData.coordinates.length === 1 && ( v.id) + : plotDataset.variable.id, observation: [plotData.id], observation_variable: observationVariable, }; @@ -365,22 +367,34 @@ const PointWindow = ({
@@ -402,6 +418,7 @@ const PointWindow = ({ {_("Global Settings")} {inputs} + {observationInputs} GetComboBoxQuery(url), - }); - - return { data, status }; -} - export function prefetchAllVariables() { const queryClient = useQueryClient(); queryClient.prefetchQuery({ diff --git a/plotting/observation.py b/plotting/observation.py index 5f817942a..8c0ef6fba 100644 --- a/plotting/observation.py +++ b/plotting/observation.py @@ -8,7 +8,7 @@ import pint import pytz from babel.dates import format_datetime -from sqlalchemy import func +from sqlalchemy import func, select from sqlalchemy.orm import Session from data import open_dataset @@ -71,17 +71,19 @@ def load_data(self): data = [] for dt in datatypes: - data.append( - self.db.query(Sample.depth, Sample.value) - .filter(Sample.station == station, Sample.datatype == dt) - .all() + query = ( + select(Sample.depth, Sample.value) + .where(Sample.station_id == station.id) + .where(Sample.datatype_key == dt.key) ) + dt_values = self.db.execute(query).all() + data.append(np.ma.array(dt_values)) if idx == 0: self.observation_variable_names.append(dt.name) self.observation_variable_units.append(dt.unit) - observation["data"] = np.ma.array(data) # .transpose() + observation["data"] = data # .transpose() self.observation[idx] = observation self.points = [ @@ -176,10 +178,11 @@ def plot(self): data = [] for o in self.observation: - d = np.ma.MaskedArray(o["data"]) - d[np.where(d == "")] = np.ma.masked - d = np.ma.masked_invalid(d.filled(np.nan).astype(np.float32)) - data.append(d) + for obs_data in o["data"]: + d = np.ma.MaskedArray(obs_data) + d[np.where(d == "")] = np.ma.masked + d = np.ma.masked_invalid(d.filled(np.nan).astype(np.float32)) + data.append(d) ureg = pint.UnitRegistry() ax_idx = -1 @@ -187,13 +190,13 @@ def plot(self): unit_map = {} for idx in self.observation_variable: ax_idx += 1 - for d in data: - if d.shape[1] == 1: - style = "." - else: - style = "-" + d = data[idx] + if d.shape[0] == 1: + style = "." + else: + style = "-" - ax[ax_idx].plot(d[idx, :, 1], d[idx, :, 0], style) + ax[ax_idx].plot(d[:, 1], d[:, 0], style) ax[ax_idx].xaxis.set_label_position("top") ax[ax_idx].xaxis.set_ticks_position("top") ax[ax_idx].set_xlabel( @@ -212,7 +215,7 @@ def plot(self): u = self.observation_variable_units[idx].lower() unit_map[self.observation_variable_names[idx]] = ureg.parse_units(u) - except: + except Exception: unit_map[self.observation_variable_names[idx]] = ureg.dimensionless for k, v in list(unit_map.items()): @@ -233,7 +236,7 @@ def plot(self): if destunit == ureg.speed_of_light: destunit = ureg.celsius - except: + except Exception: destunit = ureg.dimensionless for j in range(0, self.data.shape[0]): @@ -243,7 +246,7 @@ def plot(self): u = ureg.celsius quan = ureg.Quantity(self.data[j, idx, :], u) - except: + except Exception: quan = ureg.Quantity(self.data[j, idx, :], ureg.dimensionless) axis.plot(quan.to(destunit).magnitude, self.depths[j, idx, :]) @@ -260,7 +263,7 @@ def plot(self): ) ) else: - l = [] + leg_items = [] for j in [ ( "Observed", @@ -277,9 +280,9 @@ def plot(self): else: name = name + " " - l.append("%s%s (%s)" % (name, j[0], format_datetime(j[1][i]))) + leg_items.append("%s%s (%s)" % (name, j[0], format_datetime(j[1][i]))) - leg = axis.legend(l, loc="best") + leg = axis.legend(leg_items, loc="best") for legobj in leg.legend_handles: legobj.set_linewidth(4.0) diff --git a/routes/api_v2_0.py b/routes/api_v2_0.py index 568a864f1..b9d900472 100644 --- a/routes/api_v2_0.py +++ b/routes/api_v2_0.py @@ -1383,25 +1383,36 @@ def observation_track( db, query_dict.get("quantum", "day"), **params ) - if len(coordinates) > 1: - df = pd.DataFrame(np.array(coordinates), columns=["id", "type", "lon", "lat"]) - df["id"] = df.id.astype(int) + df = pd.DataFrame(np.array(coordinates), columns=["id", "type", "lon", "lat"]) + df["id"] = df.id.astype(int) + df = df.groupby("id").filter(lambda id: len(id) > 1) + + if len(df) > 0: + df["coordinates"] = df[["lon", "lat"]].values.tolist() + df["type"] = df["type"].apply(lambda t: t.name) + df = ( + df[["id", "type", "coordinates"]] + .groupby(["id", "type"]) + .agg(list) + .reset_index() + ) - vc = df.id.value_counts() - for p_id in vc.where(vc > 1).dropna().index: - d = { + data = [ + { "type": "Feature", "geometry": { "type": "LineString", - "coordinates": df[["lon", "lat"]][df.id == p_id].values.tolist(), + "coordinates": row[2], }, "properties": { - "id": int(p_id), - "type": df.type[df.id == p_id].values[0].name, + "id": int(row[0]), + "type": row[1], "class": "observation", }, } - data.append(d) + for row in df.values + if len(row[2]) > 1 + ] result = { "type": "FeatureCollection", @@ -1488,23 +1499,38 @@ def observation_point( if len(stations) > 500: stations = stations[:: round(len(stations) / 500)] - for s in stations: - if checkpoly and not poly.contains(Point(s.latitude, s.longitude)): - continue + df = pd.DataFrame( + np.array(stations), + columns=["type", "id", "name", "lat", "lon"], + ) + df["id"] = df.id.astype(int) - d = { - "type": "Feature", - "geometry": {"type": "Point", "coordinates": [s.longitude, s.latitude]}, - "properties": { - "type": s.platform.type.name, - "id": s.id, - "class": "observation", - }, - } - if s.name: - d["properties"]["name"] = s.name + if len(df) > 0: + df["coordinates"] = df[["lon", "lat"]].values.tolist() + df["type"] = df["type"].apply(lambda t: t.name) + + if checkpoly: + df["in_poly"] = df["coordinates"].apply( + lambda c: poly.contains(Point(c[1], c[0])) + ) + df = df.loc[df["in_poly"]] - data.append(d) + data = [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": row["coordinates"], + }, + "properties": { + "id": int(row.id), + "type": row["type"], + "class": "observation", + **({"name": row["name"]} if row["name"] is not None else {}), + }, + } + for idx, row in df.iterrows() + ] result = { "type": "FeatureCollection", diff --git a/tests/test_api_v_2_0.py b/tests/test_api_v_2_0.py index feb8d5f75..101f6fadc 100644 --- a/tests/test_api_v_2_0.py +++ b/tests/test_api_v_2_0.py @@ -98,7 +98,7 @@ def test_variables_depth_only(self) -> None: "interp": None, "two_dimensional": False, "vector_variable": False, - "data_categories": False + "data_categories": False, } ] @@ -393,27 +393,32 @@ def test_observation_meta_values(self, patch_get_meta_values): data = json.loads(response.content) assert data[0] == "this is a test" - @patch("data.observational.queries.get_stations") + @patch("data.observational.queries.get_platform_tracks") def test_observation_track( self, - patch_get_stations, + patch_get_platform_tracks, ): platform_type = PropertyMock() platform_type.name = "platform_type" - station = PropertyMock( - platform=PropertyMock(type=platform_type), - latitude=0, - longitude=0, - id=0, - ) - station.name = "myname" - patch_get_stations.return_value = [station] - response = self.client.get(self.api_links["observation_point"]) + platform = PropertyMock() + platform.name = "platform_name" + + coordinates = [ + (1, platform, -120.4, 33.1), + (1, platform, -120.3, 32.9), + (1, platform, -120.1, 32.7), + ] + patch_get_platform_tracks.return_value = coordinates + response = self.client.get(self.api_links["observation_track"]) assert response.status_code == 200 data = json.loads(response.content) assert len(data["features"]) == 1 - assert data["features"][0]["geometry"]["coordinates"] == [0, 0] + assert data["features"][0]["geometry"]["coordinates"] == [ + [-120.4, 33.1], + [-120.3, 32.9], + [-120.1, 32.7], + ] @patch("sqlalchemy.orm.session.Session.query") def test_observation_variables(self, patch_query):