diff --git a/api/Dockerfile b/api/Dockerfile index 1a1588e2..e777f37e 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-bullseye +FROM python:3.12-slim-bookworm # Ensure all system packages are up to date to reduce vulnerabilities RUN apt-get update && apt-get upgrade -y && apt-get clean @@ -6,10 +6,6 @@ RUN apt-get update && apt-get upgrade -y && apt-get clean COPY requirements.txt requirements.txt RUN pip install -r requirements.txt -# remove requirements-dev installation for deployments -COPY requirements-dev.txt requirements-dev.txt -RUN pip install -r requirements-dev.txt - # https://www.digicert.com/kb/digicert-root-certificates.htm # Get the .pem file from digicert and add it to the bundle used by certifi @@ -23,24 +19,13 @@ RUN wget -O /usr/lib/ssl/certs/GeoTrustTLSRSACAG1.crt.pem https://cacerts.digice update-ca-certificates && \ cat /usr/lib/ssl/certs/GeoTrustTLSRSACAG1.crt.pem >> $(python -c "import requests; print(requests.certs.where())") -# Install system dependencies (required for Google Cloud SDK) -RUN apt-get update && apt-get install -y \ - curl \ - && rm -rf /var/lib/apt/lists/* - -#RUN python -m wget https://dl.min.io/client/mc/release/linux-amd64/mc -#RUN chmod +x mc -#RUN mv mc /usr/local/bin/mc -RUN curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && \ - chmod +x /usr/local/bin/mc - COPY ./com_res /com_res # (Optional) For local testing with a service account key: # COPY key.json /app/key.json # ENV GOOGLE_APPLICATION_CREDENTIALS="path/to/your/service-account-key.json" -ENV PYTHONPATH "/com_res/:${PYTHONPATH}" +ENV PYTHONPATH=/com_res/ EXPOSE 8000 -CMD uvicorn --host 0.0.0.0 --port 8000 --proxy-headers main:app +CMD ["uvicorn", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers", "main:app"] diff --git a/api/README.md b/api/README.md index db853e80..c1eaeac6 100644 --- a/api/README.md +++ b/api/README.md @@ -1,25 +1,35 @@ -# com_res FastAPI +# FloodSavvy API -A python FastAPI application +FastAPI service for FloodSavvy data endpoints. -The Dockerfile declares a python base image and installs the dependencies declared in `requirements.txt` and `requirements-dev.txt` and starts up the FastApi application at port 8000. +## Active Routers -`com_res/main.py` is the entrypoint to the FastAPI application, configures the routers. The file also contains a startup event hook that initialized the mongodb database with [beanie ODM](https://beanie-odm.dev/). The startup event hook also sets up a minio client for the [CUAHSI MinIO instance](https://console.minio.cuahsi.io). The minio client is used for synchronizing user specific access policies and keys/secrets. +The deployment now includes only the routers mounted in `com_res/main.py`: -API documentation is rendered at https://com_res-api-jbzfw6l52q-uc.a.run.app/redoc (This will be updated to https://api.com_res.cuahsi.io/redocs pending certificate creation). OpenAPI spec documentation is generated from the code defining the api endpoints (FastAPI) and input/output models (Pydantic). +- `timeseries` (`/timeseries/*`): National Water Model historical/forecast utilities. +- `fim` (`/fim` and `/historical-quantiles`): flood inundation mapping and quantile lookups. -User authentication is achieved by configuring the [fastapi_users](https://github.com/fastapi-users/fastapi-users) module with [CUAHSI SSO](https://auth.cuahsi.org/) using the `OpenID Connect` protocol. On registration a S3 bucket is created for the user on [CUAHSI MinIO](https://console.minio.cuahsi.io) (TODO: create a default quota of 5 GB). An admin may increase the quota on a case by case basis. +## Runtime Configuration -The com_res API is divided into 4 routers defined at `com_res/app/routers/`. +Required environment variables: -## Routers -### Access Control Router -The `access_control` router contains prototyped synchronization of view/edit access to paths on MinIO that have a HydroShare resource that references a path on the CUAHSI MinIO instance. In the [mongo_discovery-access-control](https://github.com/hydroshare/hydroshare/compare/develop...mongo-discovery-access-control) HydroShare branch, event hooks are created for exporting Resource and User access to a mongo database. This mongo database is accessed to look up the resources which a user has view/edit privileges and generates the view/edit policies that are assigned to the user on CUAHSI MinIO storage. This means a path in a user's bucket may be registered on HydroShare and enjoy the same access control capabilities of a HydroShare Composite Resource. +- `VITE_APP_API_URL` +- `ALLOW_ORIGINS` +- `NWM_BIGQUERY_KEY` +- `NWM_BIGQUERY_URL` +- `BIGQUERY_PROJECT_ID` (optional, defaults to `com-res`) +- `CLOUD_RUN_REGION` (optional, defaults to `us-central1`) +- `CLOUD_RUN_JOB_NAME` (optional, defaults to `fimserv`) +- `GCS_BUCKET_NAME` (optional, defaults to `com_res_fim_output`) +- `GOOGLE_APPLICATION_CREDENTIALS_PATH` (optional, for explicit service account key path) -### Discovery Router -A copy of the IGUIDE discovery router that includes endpoints for searching resource metadata. The com_res workflows run the hydroshare metadata extraction tool to extract metadata the same metadata that a HydroShare composite resource will extract from recognized file formats. The resulting metadata can then be written to the Discovery database on Atlas. TODO: collect the metadata extracted from com_res outputs into a discovery database. +## Local Run -### Storage Router -Contins the endpoints to generate presigned urls for PUT and GET of objects on S3. This is not currently used but could be used to create a resource landing page for resources stored on S3 equivalent to a resource on HydroShare. +From this `api` directory: +```bash +pip install -r requirements.txt +uvicorn --host 0.0.0.0 --port 8000 --proxy-headers main:app +``` +Or use the repository-level `docker-compose.yml` to run the API container (non-debug runtime). diff --git a/api/com_res/app/db.py b/api/com_res/app/db.py deleted file mode 100644 index ab697f89..00000000 --- a/api/com_res/app/db.py +++ /dev/null @@ -1,100 +0,0 @@ -import re -from enum import Enum -from functools import lru_cache -from typing import List, Optional, Tuple - -import httpx -import motor.motor_asyncio -from beanie import Document -from fastapi_users.db import BaseOAuthAccount, BeanieBaseUser, BeanieUserDatabase -from pydantic import BaseModel, Field - -from config import get_settings - -client = motor.motor_asyncio.AsyncIOMotorClient(get_settings().mongo_url, uuidRepresentation="standard") -db = client[get_settings().mongo_database] - -client_hydroshare = motor.motor_asyncio.AsyncIOMotorClient( - get_settings().hydroshare_mongo_url, uuidRepresentation="standard" -) -db_hydroshare = client_hydroshare[get_settings().hydroshare_mongo_database] - - -class OAuthAccount(BaseOAuthAccount): - pass - - -class PhaseEnum(str, Enum): - RUNNING = "Running" - SUCCEEDED = "Succeeded" - FAILED = "Failed" - PENDING = "Pending" - ERROR = "Error" - - -class Submission(BaseModel): - workflow_id: str - workflow_name: str - phase: Optional[PhaseEnum] = None - startedAt: Optional[str] = None - finishedAt: Optional[str] = None - estimatedDuration: Optional[int] = None - - def output_path(self, base_path): - return f"{base_path}/{self.workflow_name}/{self.workflow_id}" - - -class User(BeanieBaseUser, Document): - oauth_accounts: List[OAuthAccount] = Field(default_factory=list) - submissions: List[Submission] = Field(default_factory=list) - name: Optional[str] = None - username: Optional[str] = None - given_name: Optional[str] = None - family_name: Optional[str] = None - - @property - def bucket_name(self): - return re.sub("[^A-Za-z0-9\.-]", "", re.sub("[@]", ".at.", self.username.lower())) - - async def update_profile(self): - async def get_profile(token: str) -> Tuple[str, str]: - async with httpx.AsyncClient() as client: - response = await client.get( - get_settings().user_info_endpoint, - headers={"Authorization": f"Bearer {token}"}, - ) - return response.json() - - profile = await get_profile(self.oauth_accounts[0].access_token) - self.name = profile['name'] - self.username = profile['preferred_username'] - self.given_name = profile['given_name'] - self.family_name = profile['family_name'] - await self.save() - - def get_submission(self, workflow_id: str) -> Submission: - try: - return next(submission for submission in self.submissions if submission.workflow_id == workflow_id) - except: - return None - - def running_submissions(self) -> list[Submission]: - return [submission for submission in self.submissions if submission.phase == PhaseEnum.RUNNING] - - async def update_submission(self, submission: Submission) -> None: - if self.get_submission(submission.workflow_id): - self.submissions = [ - submission if submission.workflow_id == ws.workflow_id else ws for ws in self.submissions - ] - else: - self.submissions.append(submission) - await self.save() - - -async def get_user_db(): - yield BeanieUserDatabase(User, OAuthAccount) - - -@lru_cache -def get_hydroshare_access_db(): - return db_hydroshare diff --git a/api/com_res/app/models.py b/api/com_res/app/models.py deleted file mode 100644 index 2d0b4f23..00000000 --- a/api/com_res/app/models.py +++ /dev/null @@ -1,54 +0,0 @@ -from enum import Enum -from typing import Annotated, Any - -from fastapi import Depends, HTTPException, Path, status -from pydantic import BaseModel, Field - -from app.db import Submission, User -from app.users import current_active_user - - -class WorkflowParams(BaseModel): - workflow_id: str = Field(title="Workflow ID", description="The id of the workflow") - submission: Submission - user: User - - -async def workflow_params( - workflow_id: Annotated[str, Path(title="Workflow ID", description="The id of the workflow")], - user: User = Depends(current_active_user), -): - submission = user.get_submission(workflow_id) - if workflow_id not in [submission.workflow_id for submission in user.submissions]: - raise HTTPException(status.HTTP_404_NOT_FOUND) - return WorkflowParams(workflow_id=workflow_id, user=user, submission=submission) - - -WorkflowDep = Annotated[WorkflowParams, Depends(workflow_params)] - - -class LogsResponseModel(BaseModel): - logs: str = Field(description="The logs for a workflow submission") - - -class UrlResponseModel(BaseModel): - url: str = Field(description="The presigned url to download a submission result") - - -class UserSubmissionsResponseModel(BaseModel): - submissions: list[Submission] - - -class SubmissionResponseModel(Submission): - workflow_id: str - - -class NWMVersionEnum(str, Enum): - nwm1 = "nwm1" - nwm2 = "nwm2" - nwm3 = "nwm3" - - -class ExtractMetadataRequestBody(BaseModel): - workflow_id: str - metadata: Any = None diff --git a/api/com_res/app/nwmmap.py b/api/com_res/app/nwmmap.py deleted file mode 100644 index e8bae163..00000000 --- a/api/com_res/app/nwmmap.py +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python3 - -""" -Description: This script contains helper functions for generating - an interactive map containing Water Model features. - -Author(s): Tony Castronova -""" - -import json - -import geopandas as gpd -import ipyleaflet -import requests -import shapely -from ipywidgets import Layout -from sidecar import Sidecar - - -class Map: - def __init__(self, basemap=ipyleaflet.basemaps.OpenStreetMap.Mapnik, gdf=None, plot_gdf=False, name='Map'): - self.selected_id = None - self.selected_layer = None - - self.basemap = basemap - self.name = name - - self.map = self.build_map() - - def build_map(self): - defaultLayout = Layout(width='960px', height='940px') - - m = ipyleaflet.Map( - basemap=ipyleaflet.basemap_to_tiles(ipyleaflet.basemaps.OpenStreetMap.Mapnik, layout=defaultLayout), - center=(45.9163, -94.8593), - zoom=9, - scroll_wheel_zoom=True, - tap=False, - ) - - m.add_layer( - ipyleaflet.WMSLayer( - url='https://maps.water.noaa.gov/server/services/reference/static_nwm_flowlines/MapServer/WMSServer', - layers='0', - transparent=True, - format='image/png', - min_zoom=8, - max_zoom=18, - ) - ) - - # add USGS Gages - m.add_layer( - ipyleaflet.WMSLayer( - url='http://arcgis.cuahsi.org/arcgis/services/NHD/usgs_gages/MapServer/WmsServer', - layers='0', - transparent=True, - format='image/png', - min_zoom=8, - max_zoom=18, - ) - ) - # bind the map handler function - m.on_interaction(self.handle_map_interaction) - - return m - - def asInlineMap(self): - display(self.map) - - def asSideCarMap(self): - - sc = Sidecar(title=self.name) - with sc: - display(self.map) - - def action_after_map_click(self): - # Method can be implemented by Subclasses to provide additional - # additional functionality after a NWM reach has been selected. - pass - - def handle_map_interaction(self, **kwargs): - - if kwargs.get('type') == 'click': - - # remove the previously selected layers - if self.selected_layer is not None: - self.map.remove(self.selected_layer) - self.selected_layer = None - - # get the mouse coordinates, convert to a point, - # buffer it, then find the reach that interects with it. - # - # buffer the selected point by a small degree. This - # is a hack for now and Buffer operations should only - # be applied in a projected coordinate system in the future. - lat, lon = kwargs['coordinates'] - point = shapely.Point(lon, lat) - pt_buf = point.buffer(0.001) - - # Convert Shapely Polygon to ArcGIS JSON format - geometry = { - "rings": [[list(coord) for coord in pt_buf.exterior.coords]], - "spatialReference": {"wkid": 4326}, - } - - url = ( - "https://maps.water.noaa.gov/server/rest/services/reference/static_nwm_flowlines/FeatureServer/0/query" - ) - params = { - "geometry": f"{geometry}", - "geometryType": "esriGeometryPolygon", - "spatialRel": "esriSpatialRelIntersects", - "outFields": "*", - "returnGeometry": "true", - "f": "geojson", - } - response = requests.get(url, params=params) - data = response.json() - - if "features" in data and data["features"]: - - # Convert features to Shapely geometries - flowlines = [ - (shapely.geometry.shape(feature["geometry"]), feature["properties"]) for feature in data["features"] - ] - - # Find the nearest flowline to the buffered polygon - nearest_line, nearest_properties = min(flowlines, key=lambda item: pt_buf.distance(item[0])) - self.set_selected(nearest_properties['feature_id']) - - # Convert nearest flowline to GeoJSON format - self.nearest_geojson = { - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "geometry": json.loads(gpd.GeoSeries([nearest_line]).to_json())["features"][0]["geometry"], - "properties": {"name": "Nearest Flowline"}, - } - ], - } - - self.map.add_layer(ipyleaflet.GeoJSON(data=self.nearest_geojson, style={"color": "red", "weight": 3})) - - # save this layer as the selected_layer - self.selected_layer = self.map.layers[-1] - - else: - self.nearest_geojson = None - self.set_selected(None) - - # call the after_reach_selected function. - # this is intended to enable extra functionality - # that can be implemented in subclasses - self.action_after_map_click() - - # setter for the selected reach - def set_selected(self, value): - self.selected_id = value - - # getter for selected reach - def selected(self): - return self.selected_id diff --git a/api/com_res/app/routers/access_control/__init__.py b/api/com_res/app/routers/access_control/__init__.py deleted file mode 100644 index f1170b21..00000000 --- a/api/com_res/app/routers/access_control/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from app.routers.access_control.router import router diff --git a/api/com_res/app/routers/access_control/policy_generation.py b/api/com_res/app/routers/access_control/policy_generation.py deleted file mode 100644 index 19e6b848..00000000 --- a/api/com_res/app/routers/access_control/policy_generation.py +++ /dev/null @@ -1,109 +0,0 @@ -import copy -import json -import logging as logger -import os -import subprocess -import tempfile -from typing import Dict - - -def admin_policy_create(target, name, file): - arguments = ['mc', '--json', 'admin', 'policy', 'create', target, name, file] - logger.info(arguments) - try: - _output = subprocess.check_output(arguments) - logger.info(_output) - except subprocess.CalledProcessError as e: - logger.exception(e.output) - - -def admin_policy_remove(target, name): - arguments = ['mc', '--json', 'admin', 'policy', 'rm', target, name] - logger.info(arguments) - try: - _output = subprocess.check_output(arguments) - logger.info(_output) - except subprocess.CalledProcessError as e: - logger.exception(e.output) - - -def refresh_minio_policy(user, policy): - logger.info(json.dumps(policy, indent=2)) - with tempfile.TemporaryDirectory() as tmpdirname: - filepath = os.path.join(tmpdirname, "metadata.json") - fp = open(filepath, "w") - fp.write(json.dumps(policy)) - fp.close() - admin_policy_create(target='cuahsi', name=user.username, file=filepath) - return policy - - -def create_view_statements(user, views: Dict[str, list[str]]) -> list: - if not views: - return [] - view_statement_template_get_object = { - "Effect": "Allow", - "Action": ["s3:GetObject"], - "Resource": [], - } - view_statement_template_get_bucket = { - "Effect": "Allow", - "Action": ["s3:GetBucketLocation"], - "Resource": [], - } - view_statement_template_listing = { - "Effect": "Allow", - "Action": ["s3:ListBucket"], - "Resource": [], - "Condition": {"StringLike": {"s3:prefix": []}}, - } - - get_objectt_resources = [] - get_bucket_resources = [] - list_statements = [] - for bucket_owner, resource_paths in views.items(): - get_objectt_resources = get_objectt_resources + [ - f"arn:aws:s3:::{bucket_owner}/{resource_path}/*" for resource_path in resource_paths - ] - get_bucket_resources.append(f"arn:aws:s3:::{bucket_owner}") - view_statement = copy.deepcopy(view_statement_template_listing) - view_statement["Resource"] = [f"arn:aws:s3:::{bucket_owner}"] - view_statement["Condition"]["StringLike"]["s3:prefix"] = [ - f"{resource_path}/*" for resource_path in resource_paths - ] - list_statements.append(view_statement) - view_statement_template_get_object["Resource"] = get_objectt_resources - view_statement_template_get_bucket["Resource"] = get_bucket_resources - return list_statements + [view_statement_template_get_object] - - -def create_edit_statements(user, edits: Dict[str, list[str]]) -> list: - edit_all_statement = { - "Effect": "Allow", - "Action": ["s3:*"], - "Resource": [f"arn:aws:s3:::{user.username}"], - } - edit_paths_resources = [] - for bucket_owner, resource_paths in edits.items(): - edit_paths_resources = edit_paths_resources + [ - f"arn:aws:s3:::{bucket_owner}/{resource_path}/*" for resource_path in resource_paths - ] - - edit_paths_statement = { - "Effect": "Allow", - "Action": ["s3:*Object"], - "Resource": edit_paths_resources, - } - list_bucket_statement = { - "Effect": "Allow", - "Action": ["s3:ListBucket"], - "Resource": [f"arn:aws:s3:::{bucket_owner}" for bucket_owner, _ in edits.items()], - } - statements = [edit_all_statement, edit_paths_statement, list_bucket_statement] - return [statement for statement in statements if statement] - - -def minio_policy(user, owners: Dict[str, list[str]], edits: Dict[str, list[str]], views: Dict[str, list[str]]): - statements = create_view_statements(user, views) - statements = statements + create_edit_statements(user, edits) - return {"Version": "2012-10-17", "Statement": statements} diff --git a/api/com_res/app/routers/access_control/router.py b/api/com_res/app/routers/access_control/router.py deleted file mode 100644 index 870d2efa..00000000 --- a/api/com_res/app/routers/access_control/router.py +++ /dev/null @@ -1,84 +0,0 @@ -import json -import os -import subprocess -import tempfile -from typing import Dict - -from fastapi import APIRouter, Depends -from pydantic import BaseModel - -from app.db import User, get_hydroshare_access_db -from app.routers.access_control.policy_generation import minio_policy -from app.users import current_active_user - -from .policy_generation import refresh_minio_policy - -router = APIRouter() - - -class UserAccess(BaseModel): - owner: list[str] - edit: list[str] - view: list[str] - - -class MinioUserResourceAccess(BaseModel): - owners: list[str] - resource_id: str - minio_resource_url: str - - -class MinioUserAccess(BaseModel): - owner: list[MinioUserResourceAccess] - edit: list[MinioUserResourceAccess] - view: list[MinioUserResourceAccess] - - -class UserPrivilege(BaseModel): - username: str - all: UserAccess - minio: MinioUserAccess - - -def check_owners_in_bucket_path(resource_access: MinioUserResourceAccess): - for owner in resource_access.owners: - if f"/browser/{owner}/" in resource_access.minio_resource_url: - return owner - return None - - -def sort_privileges(user_accesses: list[MinioUserResourceAccess]): - authorized_users = {} - for user_access in user_accesses: - bucket_owner = check_owners_in_bucket_path(user_access) - if bucket_owner: - resource_path = user_access.minio_resource_url.split(f"{bucket_owner}/", 1)[-1] - authorized_users.setdefault(bucket_owner, []).append(resource_path) - return authorized_users - - -@router.get('/policy') -async def generate_user_policy(user: User = Depends(current_active_user)): - hydroshare_access_db = get_hydroshare_access_db() - user_privilege = await hydroshare_access_db.userprivileges.find_one({"username": user.username}) - user_privilege: UserPrivilege = UserPrivilege(**user_privilege) - - # Check Authorization - minio_user_access: MinioUserAccess = user_privilege.minio - authorized_owners: Dict[str, list[str]] = sort_privileges(minio_user_access.owner) - authorized_edits: Dict[str, list[str]] = sort_privileges(minio_user_access.edit) - authorized_views: Dict[str, list[str]] = sort_privileges(minio_user_access.view) - - return minio_policy(user, authorized_owners, authorized_edits, authorized_views) - - -@router.get('/profile') -async def refresh_profile(user: User = Depends(current_active_user)): - await user.update_profile() - return user - - -@router.get('/policy/minio/cuahsi') -async def generate_and_save_user_policy(user: User = Depends(current_active_user)): - user_policy = await generate_user_policy(user) - return refresh_minio_policy(user, user_policy) diff --git a/api/com_res/app/routers/discovery/__init__.py b/api/com_res/app/routers/discovery/__init__.py deleted file mode 100644 index 23780433..00000000 --- a/api/com_res/app/routers/discovery/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .router import router diff --git a/api/com_res/app/routers/discovery/discovery.py b/api/com_res/app/routers/discovery/discovery.py deleted file mode 100644 index 5cda8d69..00000000 --- a/api/com_res/app/routers/discovery/discovery.py +++ /dev/null @@ -1,192 +0,0 @@ -from datetime import datetime - -from fastapi import APIRouter, Depends, Request -from pydantic import BaseModel, validator - -router = APIRouter() - - -class SearchQuery(BaseModel): - term: str = None - sortBy: str = None - reverseSort: bool = True - contentType: str = None - providerName: str = None - creatorName: str = None - dataCoverageStart: int = None - dataCoverageEnd: int = None - publishedStart: int = None - publishedEnd: int = None - hasPartName: str = None - isPartOfName: str = None - associatedMediaName: str = None - fundingGrantName: str = None - fundingFunderName: str = None - creativeWorkStatus: str = None - pageNumber: int = 1 - pageSize: int = 30 - - @validator('*') - def empty_str_to_none(cls, v, field, **kwargs): - if field.name == 'term' and v: - return v.strip() - - if isinstance(v, str) and v.strip() == '': - return None - return v - - @validator('dataCoverageStart', 'dataCoverageEnd', 'publishedStart', 'publishedEnd') - def validate_year(cls, v, values, field, **kwargs): - if v is None: - return v - try: - datetime(v, 1, 1) - except ValueError: - raise ValueError(f'{field.name} is not a valid year') - if field.name == 'dataCoverageEnd': - if 'dataCoverageStart' in values and v < values['dataCoverageStart']: - raise ValueError(f'{field.name} must be greater or equal to dataCoverageStart') - if field.name == 'publishedEnd': - if 'publishedStart' in values and v < values['publishedStart']: - raise ValueError(f'{field.name} must be greater or equal to publishedStart') - return v - - @validator('pageNumber', 'pageSize') - def validate_page(cls, v, field, **kwargs): - if v <= 0: - raise ValueError(f'{field.name} must be greater than 0') - return v - - @property - def _filters(self): - filters = [] - if self.publishedStart: - filters.append( - { - 'range': { - 'path': 'datePublished', - 'gte': datetime(self.publishedStart, 1, 1), - }, - } - ) - if self.publishedEnd: - filters.append( - { - 'range': { - 'path': 'datePublished', - 'lt': datetime(self.publishedEnd + 1, 1, 1), # +1 to include all of the publishedEnd year - }, - } - ) - - if self.dataCoverageStart: - filters.append( - {'range': {'path': 'temporalCoverage.startDate', 'gte': datetime(self.dataCoverageStart, 1, 1)}} - ) - if self.dataCoverageEnd: - filters.append( - {'range': {'path': 'temporalCoverage.endDate', 'lt': datetime(self.dataCoverageEnd + 1, 1, 1)}} - ) - return filters - - @property - def _should(self): - search_paths = ['name', 'description', 'keywords', 'keywords.name'] - should = [{'autocomplete': {'query': self.term, 'path': key, 'fuzzy': {'maxEdits': 1}}} for key in search_paths] - return should - - @property - def _must(self): - must = [] - must.append({'term': {'path': '@type', 'query': "Dataset"}}) - if self.contentType: - must.append({'term': {'path': '@type', 'query': self.contentType}}) - if self.creatorName: - must.append({'text': {'path': 'creator.name', 'query': self.creatorName}}) - if self.providerName: - must.append({'text': {'path': 'provider.name', 'query': self.providerName}}) - if self.hasPartName: - must.append({'text': {'path': 'hasPart.name', 'query': self.hasPartName}}) - if self.isPartOfName: - must.append({'text': {'path': 'isPartOf.name', 'query': self.isPartOfName}}) - if self.associatedMediaName: - must.append({'text': {'path': 'associatedMedia.name', 'query': self.associatedMediaName}}) - if self.fundingGrantName: - must.append({'text': {'path': 'funding.name', 'query': self.fundingGrantName}}) - if self.fundingFunderName: - must.append({'text': {'path': 'funding.funder.name', 'query': self.fundingFunderName}}) - if self.creativeWorkStatus: - must.append( - {'text': {'path': ['creativeWorkStatus', 'creativeWorkStatus.name'], 'query': self.creativeWorkStatus}} - ) - - return must - - @property - def stages(self): - highlightPaths = ['name', 'description', 'keywords', 'keywords.name', 'creator.name'] - stages = [] - compound = {'filter': self._filters, 'must': self._must} - if self.term: - compound['should'] = self._should - search_stage = { - '$search': { - 'index': 'fuzzy_search', - 'compound': compound, - } - } - if self.term: - search_stage["$search"]['highlight'] = {'path': highlightPaths} - - stages.append(search_stage) - - # sorting needs to happen before pagination - if self.sortBy: - if self.sortBy == "name": - self.sortBy = "name_for_sorting" - self.reverseSort = not self.reverseSort - stages.append({'$sort': {self.sortBy: -1 if self.reverseSort else 1}}) - stages.append({'$skip': (self.pageNumber - 1) * self.pageSize}) - stages.append({'$limit': self.pageSize}) - # stages.append({'$unset': ['_id', '_class_id']}) - stages.append( - {'$set': {'score': {'$meta': 'searchScore'}, 'highlights': {'$meta': 'searchHighlights'}}}, - ) - return stages - - -@router.get("/search") -async def search(request: Request, search_query: SearchQuery = Depends()): - stages = search_query.stages - result = await request.app.mongodb["discovery"].aggregate(stages).to_list(search_query.pageSize) - import json - - json_str = json.dumps(result, default=str) - return json.loads(json_str) - - -@router.get("/typeahead") -async def typeahead(request: Request, term: str, pageSize: int = 30): - search_paths = ['name', 'description', 'keywords', 'keywords.name'] - should = [{'autocomplete': {'query': term, 'path': key, 'fuzzy': {'maxEdits': 1}}} for key in search_paths] - - stages = [ - { - '$search': { - 'index': 'fuzzy_search', - 'compound': {'should': should}, - 'highlight': {'path': ['description', 'name', 'keywords', 'keywords.name']}, - } - }, - { - '$project': { - 'name': 1, - 'description': 1, - 'keywords': 1, - 'highlights': {'$meta': 'searchHighlights'}, - '_id': 0, - } - }, - ] - result = await request.app.mongodb["discovery"].aggregate(stages).to_list(pageSize) - return result diff --git a/api/com_res/app/routers/fim/router.py b/api/com_res/app/routers/fim/router.py index 0174bda8..f9a88d3a 100644 --- a/api/com_res/app/routers/fim/router.py +++ b/api/com_res/app/routers/fim/router.py @@ -12,33 +12,43 @@ router = APIRouter() +def _log_bigquery_identity(credentials, project_id: str) -> None: + principal = getattr(credentials, "service_account_email", None) + if not principal: + principal = credentials.__class__.__name__ + logging.error("BigQuery client initialized: project=%s principal=%s", project_id, principal) + + def get_bigquery_client(): """Helper function to create BigQuery client with flexible credential handling""" __settings = get_settings() + default_project = __settings.bigquery_project_id # 1. First try explicit service account path if configured - if hasattr(__settings, 'google_application_credentials_path'): - credentials_path = __settings.google_application_credentials_path - if credentials_path and os.path.exists(credentials_path): - try: - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = credentials_path - credentials, project = default() - return bigquery.Client(credentials=credentials, project=project or "com-res") - except Exception as e: - logging.warning(f"GOOGLE_APPLICATION_CREDENTIALS auth failed: {e}") + credentials_path = __settings.google_application_credentials_path + if credentials_path and os.path.exists(credentials_path): + try: + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = credentials_path + credentials, project = default() + client_project = default_project or project or "com-res" + client = bigquery.Client(credentials=credentials, project=client_project) + _log_bigquery_identity(credentials, client.project) + return client + except Exception as e: + logging.warning(f"GOOGLE_APPLICATION_CREDENTIALS auth failed: {e}") # 2. Try Application Default Credentials try: credentials, project = default() - return bigquery.Client(credentials=credentials, project=project or "com-res") # Fallback project + client_project = default_project or project or "com-res" + client = bigquery.Client(credentials=credentials, project=client_project) + _log_bigquery_identity(credentials, client.project) + return client except exceptions.DefaultCredentialsError as e: logging.error("No valid credentials found") raise HTTPException(status_code=500, detail=f"Could not authenticate with BigQuery credentials: {str(e)}") -router = APIRouter() - - @router.get("/fim") async def get_fim( reach_id: str = Query(..., description="The unique NWM reach identifier.", example="8584970"), @@ -60,11 +70,13 @@ async def get_fim( HTTPException: if the BigQuery operation fails or if the reach ID is not found. """ try: + settings = get_settings() client = get_bigquery_client() + table_ref = f"`{settings.bigquery_project_id}.flood_data.fim_catalog`" - query = """ + query = f""" SELECT * - FROM `com-res.flood_data.fim_catalog` + FROM {table_ref} WHERE reach_id = @reach_id ORDER BY stage ASC """ @@ -82,13 +94,10 @@ async def get_fim( for row in query_job: # TODO fix the "public_url listing in bigQuery" # https://cuahsi.atlassian.net/browse/CAM-797 - results['files'].append(row['asset_url']) + results['files'].append(row['public_url']) results['stages_ft'].append(row['stage']) results['flows_cfs'].append(row['flow']) - # replace the "gs://" prefix with "https://storage.googleapis.com/" - results['files'] = [url.replace("gs://", "https://storage.googleapis.com/") for url in results['files']] - except Exception as e: logging.error(f"Query failed: {str(e)}") raise HTTPException(status_code=500, detail=f"BigQuery operation failed: {str(e)}") diff --git a/api/com_res/app/routers/hydroshare/__init__.py b/api/com_res/app/routers/hydroshare/__init__.py deleted file mode 100644 index 23780433..00000000 --- a/api/com_res/app/routers/hydroshare/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .router import router diff --git a/api/com_res/app/routers/hydroshare/router.py b/api/com_res/app/routers/hydroshare/router.py deleted file mode 100644 index 5cd892b5..00000000 --- a/api/com_res/app/routers/hydroshare/router.py +++ /dev/null @@ -1,50 +0,0 @@ -import json -import tempfile -from typing import Any, Union - -import google.cloud.logging as logging -from fastapi import APIRouter, Depends -from pydantic import BaseModel - -from app.db import User -from app.users import current_active_user -from config import get_minio_client, get_settings - -if get_settings().cloud_run: - logging_client = logging.Client() - logging_client.setup_logging() - -router = APIRouter() - - -class HydroShareMetadata(BaseModel): - title: str - description: str - - -class DatasetMetadataRequestModel(BaseModel): - file_path: str - # bucket_name: str - metadata: Union[HydroShareMetadata, Any] - - -@router.post('/dataset/metadata') -async def create_metadata(metadata_request: DatasetMetadataRequestModel, user: User = Depends(current_active_user)): - with tempfile.NamedTemporaryFile(delete=False) as fp: - metadata_json_str = json.dumps(metadata_request.metadata) - print(metadata_json_str) - fp.write(str.encode(metadata_json_str)) - fp.close() - get_minio_client().fput_object(user.bucket_name, metadata_request.file_path, fp.name) - - -@router.put('/dataset/metadata') -async def update_metadata(metadata_request: DatasetMetadataRequestModel, user: User = Depends(current_active_user)): - get_minio_client().remove_object(user.bucket_name, metadata_request.file_path) - return await create_metadata(metadata_request, user) - - -class DatasetExtractRequestModel(BaseModel): - file_path: str = None - # bucket_name: str - metadata: Union[HydroShareMetadata, Any] = None diff --git a/api/com_res/app/routers/storage/__init__.py b/api/com_res/app/routers/storage/__init__.py deleted file mode 100644 index 23780433..00000000 --- a/api/com_res/app/routers/storage/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .router import router diff --git a/api/com_res/app/routers/storage/router.py b/api/com_res/app/routers/storage/router.py deleted file mode 100644 index d2df109a..00000000 --- a/api/com_res/app/routers/storage/router.py +++ /dev/null @@ -1,28 +0,0 @@ -from fastapi import APIRouter, Depends - -from app.db import User -from app.models import WorkflowDep -from app.users import current_active_user -from config import get_minio_client - -router = APIRouter() - - -@router.get('/presigned/get/{workflow_id}', description="Create a download url") -async def presigned_get_minio(workflow_params: WorkflowDep, user: User = Depends(current_active_user)): - submission = workflow_params.user.get_submission(workflow_params.workflow_id) - url = get_minio_client().presigned_get_object("com_res-outputs", submission.output_path(user.bucket_name)) - return {'url': url} - - -@router.get('/url/{workflow_id}', description="Create a download url") -async def presigned_get_url(workflow_params: WorkflowDep, user: User = Depends(current_active_user)): - submission = workflow_params.user.get_submission(workflow_params.workflow_id) - url = get_minio_client().presigned_get_object("com_res-outputs", submission.output_path(user.bucket_name)) - return {'url': url} - - -@router.get('/presigned/put/{bucket}', description="Create a PUT file presigned url") -async def presigned_put_minio(bucket: str, path: str): - url = get_minio_client().presigned_put_object(bucket, path) - return {'url': url} diff --git a/api/com_res/app/routers/timeseries/router.py b/api/com_res/app/routers/timeseries/router.py index e07a06ac..4e252350 100644 --- a/api/com_res/app/routers/timeseries/router.py +++ b/api/com_res/app/routers/timeseries/router.py @@ -1,5 +1,4 @@ import json -import logging from datetime import date, datetime import pandas @@ -9,13 +8,12 @@ from google.cloud import bigquery from app.routers.fim.router import get_bigquery_client +from config import get_settings from . import unit_conversions as units from .forecast import Forecasts, ForecastTypes from .historical import AnalysisAssim -logger = logging.getLogger(__name__) - router = APIRouter() @@ -264,11 +262,13 @@ async def get_quantiles( if the BigQuery operation fails or if the feature ID is not found. """ try: + settings = get_settings() client = get_bigquery_client() + table_ref = f"`{settings.bigquery_project_id}.flood_data.quantiles_catalog`" - query = """ + query = f""" SELECT * - FROM `com-res.flood_data.quantiles_catalog` + FROM {table_ref} WHERE feature_id = @feature_id ORDER BY doy ASC """ diff --git a/api/com_res/app/routers/timeseries/unit_conversions.py b/api/com_res/app/routers/timeseries/unit_conversions.py index 7f04cb2f..2ccf2cc8 100644 --- a/api/com_res/app/routers/timeseries/unit_conversions.py +++ b/api/com_res/app/routers/timeseries/unit_conversions.py @@ -39,20 +39,3 @@ def m_to_ft(meters): Length in feet. """ return meters * 3.28084 - - -def ft_to_m(feet): - """ - Converts from feet to meters. - - Parameters: - ========== - feet: float - Length in feet. - - Returns: - ======== - float: - Length in meters. - """ - return feet / 3.28084 diff --git a/api/com_res/app/routers/utilities/__init__.py b/api/com_res/app/routers/utilities/__init__.py deleted file mode 100644 index 23780433..00000000 --- a/api/com_res/app/routers/utilities/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .router import router diff --git a/api/com_res/app/routers/utilities/router.py b/api/com_res/app/routers/utilities/router.py deleted file mode 100644 index da25219d..00000000 --- a/api/com_res/app/routers/utilities/router.py +++ /dev/null @@ -1,112 +0,0 @@ -from typing import Any, List - -from fastapi import APIRouter -from pydantic import BaseModel -from pyproj import CRS, Transformer -from shapely.geometry import GeometryCollection, Polygon, shape - -router = APIRouter() - - -class GeoJsonGeometry(BaseModel): - type: str - coordinates: Any - - -class GeoJsonFeature(BaseModel): - type: str - geometry: GeoJsonGeometry - properties: dict - - -class GeoJsonFeatureCollection(BaseModel): - type: str - features: List[GeoJsonFeature] - - -def geojson_to_geometry_collection( - feature_collection: GeoJsonFeatureCollection, -) -> GeometryCollection: - """ - Converts a GeoJSON string to a dictionary containing a Shapely GeometryCollection. - - Arguments: - ========== - geojson: str - a GeoJSON string representing the geometries. - - Returns: - ======== - GeometryCollection: a Shapely GeometryCollection object. - """ - - geometries = [] - - for feature in feature_collection.features: - shapely_geom = shape({"type": feature.geometry.type, "coordinates": feature.geometry.coordinates}) - # NOTE: buffer(0) is a trick for fixing scenarios where polygons have overlapping coordinates - geometries.append(shapely_geom.buffer(0)) - - return GeometryCollection(geometries) - - -def transform_polygon(geom: Polygon, source_crs: str, target_crs: str) -> List: - """ - Transforms a polygon from the source CRS to the target CRS. - - Arguments: - ========== - geom: Polygon - the polygon to transform. - source_crs: str - the source coordinate reference system (CRS). - target_crs: str - the target coordinate reference system (CRS). - - Returns: - ======== - list - a list of transformed coordinates. - """ - - from_crs = CRS(source_crs) - to_crs = CRS(target_crs) - transformer = Transformer.from_crs(from_crs, to_crs, always_xy=True) - pts = [transformer.transform(x, y) for x, y in geom.exterior.coords] - return pts - - -@router.post("/nwm/compute_bbox") -async def compute_nwm_bbox(feature_collection: GeoJsonFeatureCollection): - """ - Computes the bounding box of geometries provided in the WGS 1984 coordinate - reference system (CRS) in the CRS used by the National Water Model (NWM). - - Arguments: - ========== - geojson: str - a GeoJSON string representing the geometries for which to compute the bounding box. - - Returns: - ======== - dict - a dictionary containing the bounding box in the CRS used by the NWM. - """ - - # initialize the bounding box coordinates - minx = 10e9 - miny = 10e9 - maxx = -10e9 - maxy = -10e9 - - # convert the geojson to a geometry collection - geometries = geojson_to_geometry_collection(feature_collection) - - # loop through geometries, transform their coordinates, and update the bounding box extent - for geom in geometries.geoms: - pts = transform_polygon( - geom, - "EPSG:4326", - "+proj=lcc +lat_1=30 +lat_2=60 +lat_0=40 +lon_0=-97 +x_0=0 +y_0=0 +a=6370000 +b=6370000 +units=m +no_defs", - ) - - xs, ys = zip(*pts) - minx = min(minx, min(xs)) - miny = min(miny, min(ys)) - maxx = max(maxx, max(xs)) - maxy = max(maxy, max(ys)) - - return {"minx": minx, "miny": miny, "maxx": maxx, "maxy": maxy} diff --git a/api/com_res/app/schemas.py b/api/com_res/app/schemas.py deleted file mode 100644 index f9b2b9a1..00000000 --- a/api/com_res/app/schemas.py +++ /dev/null @@ -1,14 +0,0 @@ -from beanie import PydanticObjectId -from fastapi_users import schemas - - -class UserRead(schemas.BaseUser[PydanticObjectId]): - pass - - -class UserCreate(schemas.BaseUserCreate): - pass - - -class UserUpdate(schemas.BaseUserUpdate): - pass diff --git a/api/com_res/app/users.py b/api/com_res/app/users.py deleted file mode 100644 index 8687134c..00000000 --- a/api/com_res/app/users.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -from typing import Any, Dict, Optional, Tuple, cast - -import httpx -from beanie import PydanticObjectId -from fastapi import Depends, Request -from fastapi_users import BaseUserManager, FastAPIUsers -from fastapi_users.authentication import AuthenticationBackend, BearerTransport, JWTStrategy -from fastapi_users.db import BeanieUserDatabase, ObjectIDIDMixin -from httpx_oauth.exceptions import GetIdEmailError -from httpx_oauth.oauth2 import OAuth2 - -from app.db import User, get_user_db -from config import get_minio_client, get_settings - -SECRET = "SECRET" - - -class CUAHSIOAuth2(OAuth2): - async def get_id_email(self, token: str) -> Tuple[str, str]: - async with httpx.AsyncClient() as client: - response = await client.get( - get_settings().user_info_endpoint, - headers={"Authorization": f"Bearer {token}"}, - ) - - if response.status_code >= 400: - raise GetIdEmailError(response.json()) - - data = cast(Dict[str, Any], response.json()) - - return data["sub"], data["email"] - - -client_params = dict( - client_id=os.getenv("OAUTH2_CLIENT_ID"), - client_secret=os.getenv("OAUTH2_CLIENT_SECRET"), - authorize_endpoint=get_settings().authorize_endpoint, - access_token_endpoint=get_settings().access_token_endpoint, - refresh_token_endpoint=get_settings().refresh_token_endpoint, - # revoke_token_endpoint=get_settings().revoke_token_endpoint, - base_scopes=["openid", "profile"], -) - -cuahsi_oauth_client = CUAHSIOAuth2(**client_params) - - -class UserManager(ObjectIDIDMixin, BaseUserManager[User, PydanticObjectId]): - reset_password_token_secret = SECRET - verification_token_secret = SECRET - - async def on_after_register(self, user: User, request: Optional[Request] = None): - await user.update_profile() - if not get_minio_client().bucket_exists(user.bucket_name): - get_minio_client().make_bucket(user.bucket_name) - print(f"created bucket: {user.bucket_name}") - print(f"User {user.id} has registered.") - - async def on_after_forgot_password(self, user: User, token: str, request: Optional[Request] = None): - print(f"User {user.id} has forgot their password. Reset token: {token}") - - async def on_after_request_verify(self, user: User, token: str, request: Optional[Request] = None): - print(f"Verification requested for user {user.id}. Verification token: {token}") - - -async def get_user_manager(user_db: BeanieUserDatabase = Depends(get_user_db)): - yield UserManager(user_db) - - -bearer_transport = BearerTransport(tokenUrl="auth/jwt/login") - - -def get_jwt_strategy() -> JWTStrategy: - return JWTStrategy(secret=SECRET, lifetime_seconds=60 * 60 * 24 * 30) # one month - - -auth_backend = AuthenticationBackend( - name="jwt", - transport=bearer_transport, - get_strategy=get_jwt_strategy, -) - -fastapi_users = FastAPIUsers[User, PydanticObjectId](get_user_manager, [auth_backend]) - -current_active_fastapi_user = fastapi_users.current_user(active=True) - - -async def current_active_user(user: User = Depends(current_active_fastapi_user)) -> User: - if user.username is None: - await user.update_profile() - return user diff --git a/api/com_res/config/__init__.py b/api/com_res/config/__init__.py index f6df14b5..519912c1 100644 --- a/api/com_res/config/__init__.py +++ b/api/com_res/config/__init__.py @@ -1,7 +1,6 @@ from functools import lru_cache from dotenv import load_dotenv -from minio import Minio from pydantic_settings import BaseSettings # had to use load_dotenv() to get the env variables to work during testing @@ -9,64 +8,19 @@ class Settings(BaseSettings): - - mongo_url: str - mongo_database: str - - hydroshare_mongo_url: str - hydroshare_mongo_database: str - - oauth2_client_id: str - oauth2_client_secret: str - oauth2_redirect_url: str - vite_oauth2_redirect_url: str vite_app_api_url: str allow_origins: str - minio_access_key: str - minio_secret_key: str - minio_api_url: str - nwm_bigquery_key: str nwm_bigquery_url: str + bigquery_project_id: str = "com-res" + cloud_run_region: str = "us-central1" + cloud_run_job_name: str = "fimserv" + gcs_bucket_name: str = "com_res_fim_output" - google_application_credentials_path: str - - cloud_run: bool = False - - OIDC_BASE_URL: str - - @property - def user_info_endpoint(self): - return self.OIDC_BASE_URL + "userinfo" - - @property - def authorize_endpoint(self): - return self.OIDC_BASE_URL + "auth" - - @property - def access_token_endpoint(self): - return self.OIDC_BASE_URL + "token" - - @property - def refresh_token_endpoint(self): - # TODO look up refresh token endpoint - return self.OIDC_BASE_URL + "token" - - @property - def revoke_token_endpoint(self): - return self.OIDC_BASE_URL + "revoke" + google_application_credentials_path: str = "" @lru_cache() def get_settings() -> Settings: return Settings() - - -@lru_cache() -def get_minio_client() -> Minio: - return Minio( - get_settings().minio_api_url, - access_key=get_settings().minio_access_key, - secret_key=get_settings().minio_secret_key, - ) diff --git a/api/com_res/main.py b/api/com_res/main.py index be7cc3cd..b7c90ea0 100644 --- a/api/com_res/main.py +++ b/api/com_res/main.py @@ -3,25 +3,10 @@ from app.routers.fim import router as fim_router from app.routers.timeseries import router as timeseries_router -from app.users import cuahsi_oauth_client from config import get_settings -# TODO: get oauth working with swagger/redoc -# Setting the base url for swagger docs -# https://github.com/tiangolo/fastapi/pull/1547 -# https://swagger.io/docs/specification/api-host-and-base-path/ -# https://fastapi.tiangolo.com/how-to/configure-swagger-ui/ -# https://github.com/tiangolo/fastapi/pull/499 -swagger_params = { - "withCredentials": True, - "oauth2RedirectUrl": cuahsi_oauth_client.authorize_endpoint, - "swagger_ui_client_id": cuahsi_oauth_client.client_id, -} - - app = FastAPI( servers=[{"url": get_settings().vite_app_api_url}], - swagger_ui_parameters=swagger_params, ) origins_from_settings = get_settings().allow_origins diff --git a/api/requirements-dev.txt b/api/requirements-dev.txt index 849f2820..f986d572 100644 --- a/api/requirements-dev.txt +++ b/api/requirements-dev.txt @@ -1,4 +1,3 @@ isort==5.13.2 black==24.4.2 -debugpy==1.8.2 epdb diff --git a/api/requirements.txt b/api/requirements.txt index 763dedbc..da862ec2 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,16 +1,6 @@ -pyyaml requests fastapi -fastapi-users[beanie] -motor google-cloud-bigquery uvicorn[standard] -httpx_oauth==0.15.1 -minio pydantic-settings -google-cloud-logging -pyproj -wget -shapely -pyproj pandas diff --git a/docker-compose.yml b/docker-compose.yml index 018c65b5..de8b4a2e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,36 +1,34 @@ -volumes: - mongo_data: - driver: local - services: api: platform: linux/amd64 image: api ports: - 8000:8000 - - 5678:5678 - - 8181:8181 restart: unless-stopped volumes: - ./api/com_res:/com_res # https://cloud.google.com/docs/authentication/external/set-up-adc - - $HOME/.config/gcloud/application_default_credentials.json:/app/key.json + #- $HOME/.config/gcloud/application_default_credentials.json:/app/key.json + - ./api/key.json:/app/key.json build: context: ./api/ dockerfile: Dockerfile working_dir: /com_res - command: python -m debugpy --listen 0.0.0.0:5678 -m uvicorn --host 0.0.0.0 --port 8000 --proxy-headers main:app --reload + command: uvicorn --host 0.0.0.0 --port 8000 --proxy-headers main:app env_file: - .env - depends_on: - - mongodb - mongodb: - image: mongo:5.0 + frontend: + platform: linux/amd64 + image: frontend ports: - - '27017:27017' + - 8080:8080 + restart: unless-stopped volumes: - - mongo_data:/data/db - environment: - - MONGO_INITDB_ROOT_USERNAME=root - - MONGO_INITDB_ROOT_PASSWORD=password + - ./frontend:/app + build: + context: ./frontend/ + dockerfile: Dockerfile + command: caddy run --config /etc/caddy/Caddyfile + env_file: + - .env diff --git a/env.template b/env.template index 5516e0a5..2dd982c1 100644 --- a/env.template +++ b/env.template @@ -1,18 +1,5 @@ -MONGO_URL=mongodb://root:password@mongodb -MONGO_DATABASE=com_res - -HYDROSHARE_MONGO_URL=empty -HYDROSHARE_MONGO_DATABASE=empty - -OAUTH2_CLIENT_ID=com_res -OAUTH2_CLIENT_SECRET= - -MINIO_ACCESS_KEY= -MINIO_SECRET_KEY= -MINIO_API_URL=api.minio.cuahsi.io - NWM_BIGQUERY_KEY=key -NWM_BIGQUERY_URL='https://nwm-api.ciroh.org/' +NWM_BIGQUERY_URL=https://nwm-api.ciroh.org VITE_APP_NAME=com_res @@ -34,16 +21,9 @@ VITE_APP_API_URL=http://localhost:8000 # ALLOW_ORIGINS=${VITE_APP_ORIGIN} ALLOW_ORIGINS=.* -# OAUTH2_REDIRECT_URL=${VITE_APP_API_URL}/auth/cuahsi/callback -OAUTH2_REDIRECT_URL=http://localhost:8000/auth/cuahsi/callback - -# VITE_OAUTH2_REDIRECT_URL="${VITE_APP_FULL_URL}#/auth-redirect" -VITE_OAUTH2_REDIRECT_URL="http://localhost:5173/#/auth-redirect" - -OIDC_BASE_URL=https://auth.cuahsi.org/realms/CUAHSI/protocol/openid-connect/ - -NWM_BIGQUERY_KEY=hello -NWM_BIGQUERY_URL=https://nwm-api.ciroh.org/ - # https://cloud.google.com/docs/authentication/set-up-adc-local-dev-environment +BIGQUERY_PROJECT_ID=floodsavvy +CLOUD_RUN_REGION=us-central1 +CLOUD_RUN_JOB_NAME=fimserv +GCS_BUCKET_NAME=floodsavvy_fim_output GOOGLE_APPLICATION_CREDENTIALS_PATH=/app/key.json \ No newline at end of file diff --git a/fim/build_fim_catalog/build_catalog.py b/fim/build_fim_catalog/build_catalog.py index 2fc2c21a..a2dd79d7 100644 --- a/fim/build_fim_catalog/build_catalog.py +++ b/fim/build_fim_catalog/build_catalog.py @@ -65,7 +65,7 @@ def create_bigquery_fim_records(fim_files, extension=".cog"): "stage": stage, "flow": flow, "asset_url": url, - "public_url": f'https://storage.googleapis.com/{fim_files[0].replace("gs://", "")}', + "public_url": fim_files[0], } items.append(dat) diff --git a/fim/submit_cloudrun.py b/fim/submit_cloudrun.py index ad6a861b..053a7bae 100644 --- a/fim/submit_cloudrun.py +++ b/fim/submit_cloudrun.py @@ -14,6 +14,7 @@ import re import time import json +from api.com_res.config import get_settings import typer from typing import List from pathlib import Path @@ -31,11 +32,11 @@ app = typer.Typer() console = Console() - -PROJECT_ID = "com-res" -REGION = "us-central1" -JOB_NAME = "fimserv" -GCS_BUCKET_NAME = "com_res_fim_output" +settings = get_settings() +PROJECT_ID = settings.bigquery_project_id +REGION = settings.cloud_run_region +JOB_NAME = settings.cloud_run_job_name +GCS_BUCKET_NAME = settings.gcs_bucket_name def execute_job(run_client, args: List[str]):