From 8dfe2a6e3b3bf7e9aadcbe7ed621bcb8bd03f99b Mon Sep 17 00:00:00 2001 From: riniangreani2 Date: Tue, 23 Jun 2026 13:01:29 +0800 Subject: [PATCH 1/7] add possum REST api client --- automation/README | 8 +- automation/config.env.example | 9 +-- automation/database_queries.py | 1 + automation/possum_api_client.py | 135 ++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 10 deletions(-) create mode 100644 automation/possum_api_client.py diff --git a/automation/README b/automation/README index f05a1a8..fc351f5 100644 --- a/automation/README +++ b/automation/README @@ -15,11 +15,9 @@ PREFECT_API_URL : IP address of the Prefect server to bypass the OAuth Proxy (Gi 2. Prefect secrets (under Blocks): ---------------------------------- cadc-proxy-pem | Generate cadcproxy.pem with CANFAR interactive session, and copy the content here. -database-host | POSSUM database IP address -database-name | POSSUM database name -database-password | POSSUM database password -database-port | POSSUM database port -database-user | POSSUM database username +possum-api-url | POSSUM REST API URL +possum-api-username | POSSUM portal username with update privileges +possum-api-password | POSSUM portal password possum-status-sheet | link to POSSUM status Google spreadsheet possum-status-token | Copy the content of the json token to access the possum-status-sheet above diff --git a/automation/config.env.example b/automation/config.env.example index 85e8a1c..8734fd0 100644 --- a/automation/config.env.example +++ b/automation/config.env.example @@ -3,10 +3,9 @@ PREFECT_API_URL= PREFECT_API_AUTH_STRING= # Optional unless testing with custom database and Google spreadsheet -DATABASE_HOST= -DATABASE_PORT= -DATABASE_NAME= -DATABASE_USER= -DATABASE_PASSWORD= +# TODO update the prefect secrets +POSSUM_API_URL= +POSSUM_API_USERNAME= +POSSUM_API_PASSWORD= POSSUM_STATUS_TOKEN=/arc/home/ErikOsinga/.ssh/psm_gspread_token.json POSSUM_STATUS_SHEET=https://docs.google.com/spreadsheets/d/1sWCtxSSzTwjYjhxr1_KVLWG2AnrHwSJf_RWQow7wbH0 \ No newline at end of file diff --git a/automation/database_queries.py b/automation/database_queries.py index 921c087..597d136 100644 --- a/automation/database_queries.py +++ b/automation/database_queries.py @@ -1,5 +1,6 @@ """ Database query functions for interacting with the ausSRC database. +Jun 23 2026 - This has been replaced with api_client.py. We can no longer directly query the database. """ import os diff --git a/automation/possum_api_client.py b/automation/possum_api_client.py new file mode 100644 index 0000000..71c1378 --- /dev/null +++ b/automation/possum_api_client.py @@ -0,0 +1,135 @@ +""" +This class replaces database_queries.py to act as a proxy between the POSSUM database and the code. +We can no longer directly query the database, so we use the REST API instead. +""" +import os +from threading import Lock + +import requests +from dotenv import load_dotenv + +load_dotenv() + + +class PossumApiClient: + _instance = None + _lock = Lock() + + def __new__(cls): + if cls._instance is None: + # ensure there is only 1 instance of client + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._initialized = False + + return cls._instance + + def __init__(self): + if self._initialized: + return + + self.base_url = os.environ["POSSUM_API_URL"].rstrip("/") + self.username = os.environ["POSSUM_API_USERNAME"] + self.password = os.environ["POSSUM_API_PASSWORD"] + + self.session = requests.Session() + + self.access_token = None + self.refresh_token = None + + self._initialized = True + + def login(self): + response = self.session.post( + f"{self.base_url}/api/token/", + json={ + "username": self.username, + "password": self.password, + }, + timeout=30, + ) + + response.raise_for_status() + + data = response.json() + + self.access_token = data["access"] + self.refresh_token = data["refresh"] + + def refresh_access_token(self): + if not self.refresh_token: + self.login() + return + + response = self.session.post( + f"{self.base_url}/api/token/refresh/", + json={"refresh": self.refresh_token}, + timeout=30, + ) + + if response.status_code == 401: + self.login() + return + + response.raise_for_status() + + self.access_token = response.json()["access"] + + def _ensure_authenticated(self): + if self.access_token is None: + self.login() + + def _headers(self): + return { + "Authorization": f"Bearer {self.access_token}", + } + + def _request(self, method, endpoint, **kwargs): + self._ensure_authenticated() + + response = self.session.request( + method=method, + url=f"{self.base_url}{endpoint}", + headers=self._headers(), + timeout=30, + **kwargs, + ) + + if response.status_code == 401: + self.refresh_access_token() + + response = self.session.request( + method=method, + url=f"{self.base_url}{endpoint}", + headers=self._headers(), + timeout=30, + **kwargs, + ) + + response.raise_for_status() + + return response + + def get(self, endpoint, **kwargs): + return self._request("GET", endpoint, **kwargs).json() + + def post(self, endpoint, **kwargs): + return self._request("POST", endpoint, **kwargs).json() + + def put(self, endpoint, **kwargs): + return self._request("PUT", endpoint, **kwargs).json() + + def patch(self, endpoint, **kwargs): + return self._request("PATCH", endpoint, **kwargs).json() + + def delete(self, endpoint, **kwargs): + return self._request("DELETE", endpoint, **kwargs) + +# singleton instance +client = PossumApiClient() + +# usage +# from api_client import client + +# data = client.get("/api/observations/") \ No newline at end of file From f8839fb65997f2d0e69c6dd0162b9facdbd5b54f Mon Sep 17 00:00:00 2001 From: riniangreani2 Date: Tue, 23 Jun 2026 19:53:20 +0800 Subject: [PATCH 2/7] Remove tests from github action as they do not work without a database --- .github/workflows/tests.yml | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 965ab5c..0000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Run Python Unit Tests - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - test: - runs-on: ubuntu-latest - - env: - TEST_DATABASE_NAME: ${{ secrets.DATABASE_NAME }} - TEST_DATABASE_USER: ${{ secrets.DATABASE_USER }} - TEST_DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }} - TEST_DATABASE_HOST: ${{ secrets.DATABASE_HOST }} - TEST_DATABASE_PORT: ${{ secrets.DATABASE_PORT }} - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' # or 3.10, 3.9, etc. - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: Run unit tests - run: | - python -m unittest discover -s automation/unit_tests From b915da17d71575d45d28ad2deab14420b3ae7c27 Mon Sep 17 00:00:00 2001 From: riniangreani2 Date: Thu, 2 Jul 2026 14:30:32 +0800 Subject: [PATCH 3/7] Use REST API instead of database connection --- automation/compare_sheet_to_database.py | 53 +++++------------ automation/database_queries.py | 2 +- automation/fix_tile3d_3d_pipeline_status.py | 58 +++++++------------ automation/possum_api_client.py | 55 +++++++++--------- .../log_processing_status_1D_PartialTiles.py | 19 +++--- ...ocessing_status_1D_PartialTiles_summary.py | 38 ++++++------ .../log_processing_status_3Dpipeline.py | 18 +++--- .../check_ingest_3Dpipeline.py | 25 ++++---- ...atus_and_launch_1Dpipeline_PartialTiles.py | 44 +++++++------- .../check_status_and_launch_3Dpipeline_v2.py | 15 ++--- .../control_3D_pipeline.py | 6 +- possum_pipeline_control/ingest3Dpipeline.py | 31 +++++----- .../update_partialtile_google_sheet.py | 20 ++++--- .../update_status_sheet.py | 11 ++-- 14 files changed, 177 insertions(+), 218 deletions(-) diff --git a/automation/compare_sheet_to_database.py b/automation/compare_sheet_to_database.py index 5f85a8a..33b97d2 100644 --- a/automation/compare_sheet_to_database.py +++ b/automation/compare_sheet_to_database.py @@ -13,7 +13,7 @@ import tqdm # assume this script is run as a module from the POSSUMutils package -from automation import database_queries as db # noqa: E402 +from automation import possum_api_client # noqa: E402 GOOGLE_API_TOKEN = "/home/erik/.ssh/neural-networks--1524580309831-c5c723e2468e.json" VALIDATION_SHEET_URL = ( @@ -109,25 +109,14 @@ def get_partial_tiles_database(band_number: int = 1) -> list[tuple]: Returns: list[tuple]: Each tuple is a row from the database with p.* and o.sbid as last field. """ - conn = db.get_database_connection(test=False) - try: - print( - f"Fetching full partial tiles data table for 1D pipeline run for band {band_number} " - "from the database." - ) - query = f""" - SELECT p.*, o.sbid - FROM possum.partial_tile_1d_pipeline_band{band_number} AS p - LEFT JOIN possum.observation AS o ON p.observation = o.name - """ - rows = db.execute_query(query, conn) - finally: - try: - conn.close() - except Exception: - # If the connection object is not closable, ignore. - pass - + conn = possum_api_client.PossumApiClient() + + print( + f"Fetching full partial tiles data table for 1D pipeline run for band {band_number} " + "from the database." + ) + rows = conn.get(f"/api/partial_tiles/sbid/band{band_number}/") + return rows @@ -288,23 +277,13 @@ def get_observation_state_validation(band_number: int = 1) -> dict[str, str]: Returns: dict mapping field_name -> normalised 1d_pipeline_validation state. """ - conn = db.get_database_connection(test=False) - try: - query = f""" - SELECT name, "1d_pipeline_validation" - FROM possum.observation_state_band{band_number} - """ - rows = db.execute_query(query, conn) - finally: - try: - conn.close() - except Exception: - pass - - state_by_field: dict[str, str] = {} - for field_name, validation_state in rows: - state_by_field[str(field_name)] = normalize_value(validation_state) - + conn = possum_api_client.PossumApiClient() + rows = conn.get_json(f"/api/1d-pipeline/observations/single-sb-1d-pipeline/full-table/band{band_number}/") + + state_by_field = {} + for row in rows: + state_by_field[row["name"]] = normalize_value(row["1d_pipeline_validation"]) + return state_by_field diff --git a/automation/database_queries.py b/automation/database_queries.py index 597d136..822ca6f 100644 --- a/automation/database_queries.py +++ b/automation/database_queries.py @@ -1,6 +1,6 @@ """ Database query functions for interacting with the ausSRC database. -Jun 23 2026 - This has been replaced with api_client.py. We can no longer directly query the database. +DEPRECATED as of Jun 23 2026 - This has been replaced with api_client.py. We can no longer directly query the database. """ import os diff --git a/automation/fix_tile3d_3d_pipeline_status.py b/automation/fix_tile3d_3d_pipeline_status.py index a2f2949..315f8bb 100644 --- a/automation/fix_tile3d_3d_pipeline_status.py +++ b/automation/fix_tile3d_3d_pipeline_status.py @@ -16,13 +16,13 @@ from astropy import table as at # assume this script is run as a module from the POSSUMutils package -from automation import database_queries as db +from automation import possum_api_client GOOGLE_API_TOKEN = "/home/erik/.ssh/psm_gspread_token.json" VALIDATION_SHEET_URL = "https://docs.google.com/spreadsheets/d/1sWCtxSSzTwjYjhxr1_KVLWG2AnrHwSJf_RWQow7wbH0/" -def build_update_query( +def run_update_query( tiles: np.ndarray, timestamp: Optional[datetime.datetime] = None, ) -> str: @@ -48,30 +48,28 @@ def build_update_query( if tiles.size == 0: raise ValueError("No tiles provided to update.") - # Ensure all entries are integers and build the IN list - tile_list_str = ", ".join(str(int(t)) for t in tiles) - - if timestamp is None: - # Use DB server time - timestamp_expr = "NOW()" - else: - # Format as 'YYYY-MM-DD HH:MM:SS' and cast explicitly - ts_str = timestamp.replace(microsecond=0).isoformat(sep=" ") - timestamp_expr = f"'{ts_str}'::timestamp" - - query = f""" - UPDATE possum.tile_state_band1 - SET "3d_pipeline" = {timestamp_expr} - WHERE tile IN ({tile_list_str}); - """ + api = possum_api_client.PossumApiClient() + + updated_rows = 0 + for t in tiles: + response = api.patch( + "/api/3d-pipeline/tiles/update/3d_pipeline/", + { + "band_number": 1, + "tile_number":t, + "timestamp": timestamp + }, #avoid encoding problem in url + format="json") + + num_rows = response.data.get('rows_updated') + updated_rows += num_rows - return query.strip() + return updated_rows def update_3d_pipeline_for_tiles( tiles: np.ndarray, timestamp: Optional[datetime.datetime] = None, - test: bool = False, ) -> None: """ Execute the UPDATE to set "3d_pipeline" for the given tiles. @@ -82,22 +80,10 @@ def update_3d_pipeline_for_tiles( 1D array of tile ids (integers). timestamp : datetime.datetime or None If None, uses NOW() in the database. - test : bool - Passed through to db.get_database_connection(test=...). """ - query = build_update_query(tiles, timestamp=timestamp) - - conn = db.get_database_connection(test=test) - try: - rows = db.execute_query(query, conn) - # If the connection is not in autocommit mode, commit explicitly: - if hasattr(conn, "commit"): - conn.commit() - finally: - if hasattr(conn, "close"): - conn.close() + num_rows = run_update_query(tiles, timestamp=timestamp) - print(f"Update executed successfully. Updated {len(rows)} rows") + print(f"Update executed successfully. Updated {num_rows} rows") def load_tile_validation_sheet(band: str = "943MHz") -> at.Table: @@ -141,8 +127,8 @@ def load_tile_validation_sheet(band: str = "943MHz") -> at.Table: print(f"Number of tiles to update: {len(processed_tiles)}") # Option A: use NOW() on the DB side - update_3d_pipeline_for_tiles(processed_tiles, timestamp=None, test=False) + update_3d_pipeline_for_tiles(processed_tiles, timestamp=None) # Option B: use an explicit timestamp # explicit_ts = datetime.datetime(2025, 1, 1, 12, 0, 0) - # update_3d_pipeline_for_tiles(example_tiles, timestamp=explicit_ts, test=False) + # update_3d_pipeline_for_tiles(example_tiles, timestamp=explicit_ts) diff --git a/automation/possum_api_client.py b/automation/possum_api_client.py index 71c1378..257963a 100644 --- a/automation/possum_api_client.py +++ b/automation/possum_api_client.py @@ -3,35 +3,33 @@ We can no longer directly query the database, so we use the REST API instead. """ import os -from threading import Lock import requests -from dotenv import load_dotenv - -load_dotenv() +from dotenv import load_dotenv +from pathlib import Path +from prefect.blocks.system import Secret class PossumApiClient: - _instance = None - _lock = Lock() - - def __new__(cls): - if cls._instance is None: - # ensure there is only 1 instance of client - with cls._lock: - if cls._instance is None: - cls._instance = super().__new__(cls) - cls._instance._initialized = False - - return cls._instance - def __init__(self): + def __init__(self, config_file_path: str = None): if self._initialized: return - - self.base_url = os.environ["POSSUM_API_URL"].rstrip("/") - self.username = os.environ["POSSUM_API_USERNAME"] - self.password = os.environ["POSSUM_API_PASSWORD"] + + config_file = Path(config_file_path) if config_file_path else None + + if config_file is not None and config_file.exists(): + # if config.env is supplied, we'll use the variables from the file + load_dotenv(config_file) + self.base_url = os.environ["POSSUM_API_URL"] + self.username = os.environ["POSSUM_API_USERNAME"] + self.password = os.environ["POSSUM_API_PASSWORD"] + if not self.base_url: + # otherwise load from Prefect secrets + self.base_url = Secret.load("possum-api-url").get() + if not self.username or not self.password: + self.username = Secret.load("possum-api-username").get() + self.password = Secret.load("possum-api-password").get() self.session = requests.Session() @@ -112,7 +110,13 @@ def _request(self, method, endpoint, **kwargs): return response def get(self, endpoint, **kwargs): - return self._request("GET", endpoint, **kwargs).json() + data = self._request("GET", endpoint, **kwargs).json() + # return the data as tuples as it was when we queried the DB directly + return [tuple(row.values()) for row in data] + + def get_json(self, endpoint, **kwargs): + # get json as is + return self._request("GET", endpoint, **kwargs).json() def post(self, endpoint, **kwargs): return self._request("POST", endpoint, **kwargs).json() @@ -126,10 +130,3 @@ def patch(self, endpoint, **kwargs): def delete(self, endpoint, **kwargs): return self._request("DELETE", endpoint, **kwargs) -# singleton instance -client = PossumApiClient() - -# usage -# from api_client import client - -# data = client.get("/api/observations/") \ No newline at end of file diff --git a/cirada_software/log_processing_status_1D_PartialTiles.py b/cirada_software/log_processing_status_1D_PartialTiles.py index 8d6b318..e197154 100644 --- a/cirada_software/log_processing_status_1D_PartialTiles.py +++ b/cirada_software/log_processing_status_1D_PartialTiles.py @@ -2,7 +2,7 @@ import ast import glob -from automation import database_queries as db +from automation import possum_api_client as rest_api from possum_pipeline_control import util """ @@ -50,9 +50,14 @@ def update_partial_tile_1d_pipeline(field_ID, tile_numbers, band, status, conn): """ fieldname = util.get_full_field_name(field_ID, band) band_number = util.get_band_number(band) - db.update_partial_tile_1d_pipeline_status( - fieldname, tile_numbers, band_number, status, conn - ) + conn.patch("/api/1d-pipeline/partial-tiles/update/status/", + { + "band_number": band_number, + "field_name": fieldname, + "tile_numbers": tile_numbers, + "status": status, + }, + format="json") ## TODO: validation in case all tiles have been completed # # Find the validation file path @@ -154,8 +159,6 @@ def tilenumbers_to_tilestr(tilenumbers): print(f"Tilenumbers {tilestr} status: {status}, band: {band}") # Update the POSSUM partial_tile_1d_pipeline database table - conn = db.get_database_connection( - test=False, database_config_path=database_config_path - ) + conn = rest_api.PossumApiClient(database_config_path) update_partial_tile_1d_pipeline(field_ID, tilenumbers, band, status, conn) - conn.close() + \ No newline at end of file diff --git a/cirada_software/log_processing_status_1D_PartialTiles_summary.py b/cirada_software/log_processing_status_1D_PartialTiles_summary.py index 9f6d582..6d0b334 100644 --- a/cirada_software/log_processing_status_1D_PartialTiles_summary.py +++ b/cirada_software/log_processing_status_1D_PartialTiles_summary.py @@ -10,7 +10,7 @@ from dotenv import load_dotenv from prefect import flow, task -from automation import database_queries as db +from automation import possum_api_client as rest_api from possum_pipeline_control import util """ @@ -56,15 +56,22 @@ def update_1d_database(field_ID, SBid, band, status, conn): band (str): The band of the tile. Google_API_token (str): The path to the Google API token JSON file. status (str): The status to set in the 'status_column' column. + conn : REST API session """ print("Updating POSSUM pipeline validation database with summary plot status") band_number = util.get_band_number(band) full_field_name = util.get_full_field_name(field_ID, band) - rows_to_update = db.update_1d_pipeline_table( - full_field_name, band_number, status, "1d_pipeline_validation", conn - ) + + + response = conn.patch("/api/1d-pipeline/observations/update/1d-pipeline-validation/?" + f"band_number={band_number}&" + f"field_name={full_field_name}&" + f"status={status}") + rows_to_update = response.data.get("rows_updated") + + if rows_to_update == 0: print(f"No rows found for field {full_field_name} and SBID {SBid}") return False @@ -75,8 +82,8 @@ def update_1d_database(field_ID, SBid, band, status, conn): else: print("Failed to update the database.") # Check if there are boundary issues for this field and SBID - boundary_issue = db.find_boundary_issues(SBid, full_field_name, band_number, conn) - + boundary_issue = conn.get(f"/api/1d-pipeline/partial-tiles/boundary-issues/band{band_number}/{full_field_name}/") + return boundary_issue @@ -128,16 +135,14 @@ def update_status_spreadsheet( col_letter = gspread.utils.rowcol_to_a1( 1, column_names.index(status_column) + 1 )[0] - conn = db.get_database_connection( - test=False, database_config_path=database_config_path - ) + api = rest_api.PossumApiClient(database_config_path) for row_index in rows_to_update: sleep(2) # 60 writes per minute only tile_sheet.update(range_name=f"{col_letter}{row_index}", values=[[status]]) - db.update_1d_pipeline_table( - full_field_name, band_number, status, "single_sb_1d_pipeline", conn - ) - conn.close() + api.patch(f"/api/1d-pipeline/observations/update/{status_column.lower()}/?" + f"band_number={band_number}&" + f"field_name={full_field_name}&" + f"status={status}") print( f"Updated all {len(rows_to_update)} rows for field {full_field_name} and SBID {SBid} to status '{status}' in '{status_column}' column." ) @@ -233,11 +238,8 @@ def main(args): # Update the POSSUM Validation database table t1 = task(update_1d_database, name="update_1d_database") # execute tasks serially such that logging is preserved (instead of .submit) - conn = db.get_database_connection( - test=False, database_config_path=database_config_path - ) - has_boundary_issue = t1(field_ID, SB_num, band, status, conn) - conn.close() + rest_api = rest_api.PossumApiClient(database_config_path) + has_boundary_issue = t1(field_ID, SB_num, band, status, rest_api) if status == "Completed": # Update the POSSUM Pipeline Status spreadsheet as well. A complete field has been processed! diff --git a/cirada_software/log_processing_status_3Dpipeline.py b/cirada_software/log_processing_status_3Dpipeline.py index 6d7ba70..a740f44 100644 --- a/cirada_software/log_processing_status_3Dpipeline.py +++ b/cirada_software/log_processing_status_3Dpipeline.py @@ -9,7 +9,7 @@ from dotenv import load_dotenv from prefect import flow, task -from automation import database_queries as db +from automation import possum_api_client as rest_api from possum_pipeline_control import util """ @@ -157,14 +157,14 @@ def update_3d_tile_database(tile_number, band, status): validation_link = "HTMLFileNotFound" # execute query - conn = db.get_database_connection(test=False) - rows_updated = db.update_3d_pipeline_table( - tile_number, band_number, status, "3d_pipeline_val", conn - ) - db.update_3d_pipeline_table( - tile_number, band_number, validation_link, "3d_val_link", conn - ) - conn.close() + conn = rest_api.PossumApiClient() + response = conn.patch(f"/api/3d-pipeline/tiles/update/3d_pipeline_val/?band_number={band_number}" + f"&tile_number={tile_number}" + f"&3d_pipeline_val={status}") + rows_updated = response.data.get('rows_updated') + conn.patch(f"/api/3d-pipeline/tiles/update/3d_val_link/?band_number={band_number}" + f"&tile_number={tile_number}" + f"&3d_val_link={validation_link}") # Print results if rows_updated <= 0: diff --git a/possum_pipeline_control/check_ingest_3Dpipeline.py b/possum_pipeline_control/check_ingest_3Dpipeline.py index a3453be..092edd2 100644 --- a/possum_pipeline_control/check_ingest_3Dpipeline.py +++ b/possum_pipeline_control/check_ingest_3Dpipeline.py @@ -4,7 +4,7 @@ from canfar.sessions import Session from vos import Client -from automation import database_queries as db +from automation import possum_api_client as rest_api from possum_pipeline_control import util session = Session() @@ -37,8 +37,7 @@ def get_tiles_for_ingest(band_number, conn): list: A list of tile numbers that satisfy the conditions. """ # Find the tiles that satisfy the conditions - return db.get_tiles_for_ingest(band_number, conn) - + return conn.get_json(f"/api/3d-pipeline/tiles/ready_for_ingest/band{band_number}/") def get_canfar_tiles(band_number): client = Client() @@ -100,7 +99,7 @@ def launch_ingest(tilenumber, band): return session_id_str -def update_status(tile_number, band, status, conn): +def update_status(tile_number, band, status, api): """ Update the status of the specified tile in the database. @@ -110,9 +109,10 @@ def update_status(tile_number, band, status, conn): status (str): The status to set in the '3d_pipeline_ingest' column. """ band_no = util.get_band_number(band) - return db.update_3d_pipeline_table( - tile_number, band_no, status, "3d_pipeline_ingest", conn - ) + return api.patch(f"/api/3d-pipeline/tiles/update/3d_pipeline_ingest/?band_number={band_no}" + f"&tile_number={tile_number}" + f"&3d_pipeline_ingest={status}") + def ingest_3Dpipeline(band_number=1): if band_number == 1: @@ -121,13 +121,10 @@ def ingest_3Dpipeline(band_number=1): band = "1367MHz" # Check database for band 1 tiles that have been processed AND validated - conn = db.get_database_connection(test=False) + conn = rest_api.PossumApiClient() tile_numbers = get_tiles_for_ingest(band_number, conn) - tile_numbers = [ - str(tn) for tn in tile_numbers - ] # make sure they are strings for comparison - conn.close() - + tile_numbers = [str(row["tile"]) for row in tile_numbers] # make sure they are strings for comparison + canfar_tilenumbers = get_canfar_tiles(band_number=band_number) if len(tile_numbers) > 0: @@ -160,9 +157,7 @@ def ingest_3Dpipeline(band_number=1): launch_ingest(tilenumber, band) # Update the status of 3d_pipeline_ingest to "IngestRunning" - conn = db.get_database_connection(test=False) row_count = update_status(tilenumber, band, "IngestRunning", conn) - conn.close() if row_count == 0: print(f"Tile {tilenumber} not found in the sheet.") diff --git a/possum_pipeline_control/check_status_and_launch_1Dpipeline_PartialTiles.py b/possum_pipeline_control/check_status_and_launch_1Dpipeline_PartialTiles.py index db5fd59..366ae30 100644 --- a/possum_pipeline_control/check_status_and_launch_1Dpipeline_PartialTiles.py +++ b/possum_pipeline_control/check_status_and_launch_1Dpipeline_PartialTiles.py @@ -6,7 +6,7 @@ from vos import Client -from automation import database_queries as db +from automation import possum_api_client as rest_api from possum_pipeline_control import util from possum_pipeline_control.control_1D_pipeline_PartialTiles import get_open_sessions @@ -59,7 +59,7 @@ def get_results_per_field_sbid_skip_edges(band_number, conn, verbose=False): dict: A dictionary with keys as (field_name, sbid) tuples and boolean values indicating whether the conditions are met for the non-edge rows. """ - rows = db.get_observations_non_edge_rows(band_number, conn) + rows = conn.get(f"/api/1d-pipeline/observations/non-edge-rows/band{band_number}/") results = {} # Group the table by 'field_name' and 'sbid' @@ -83,7 +83,7 @@ def get_results_per_field_sbid(conn, band_number="1", verbose=False): If all Partial tiles for a fieldname have been completed boolean=True, otherwise false. """ # Group the table by 'field_name' and 'sbid' - results = db.get_observations_with_complete_partial_tiles(band_number, conn) + results = conn.get(f"/api/1d-pipeline/partial-tiles/complete-partial-tiles/band{band_number}/") field_sbid_dict = {} # make dict to get rid of duplicates for row in results: @@ -124,7 +124,7 @@ def get_tiles_for_pipeline_run(db_conn, band_number): """ # Find the tiles that satisfy the conditions # (i.e. has an SBID and not yet a '1d_pipeline' status) - rows = db.get_partial_tiles_for_1d_pipeline_run(band_number, db_conn) + rows = db_conn.get(f"/api/1d-pipeline/partial-tiles/ready-for-pipeline/band{band_number}/") fields_to_run, SBids_to_run = [], [] tile1_to_run, tile2_to_run, tile3_to_run, tile4_to_run = [], [], [], [] if rows: @@ -345,13 +345,12 @@ def update_validation_status( status (str): The status to set in the specified column. """ print("Updating partial tile status in the POSSUM pipeline validation sheet.") - conn = db.get_database_connection( - test=False, database_config_path=database_config_path - ) - row_num = db.update_1d_pipeline_table( - field_name, band_number, "Running", "1d_pipeline_validation", conn - ) - conn.close() + conn = rest_api.PossumApiClient(database_config_path) + response = conn.patch("/api/1d-pipeline/observations/update/1d-pipeline-validation/?" + f"band_number={band_number}&" + f"field_name={field_name}&" + "status=Running") + row_num = response.data.get("rows_updated") if row_num > 0: print( @@ -396,9 +395,7 @@ def launch_band1_1Dpipeline(database_config_path=None): # i.e. 'SBID' column is not empty, 'number_sources' is not empty, and '1d_pipeline' column is empty # connect to the database - conn = db.get_database_connection( - test=False, database_config_path=database_config_path - ) + conn = rest_api.PossumApiClient(database_config_path) ( field_IDs, tile1, @@ -412,8 +409,6 @@ def launch_band1_1Dpipeline(database_config_path=None): assert len(tile1) == len(tile2) == len(tile3) == len(tile4), ( "Need to have 4 tile columns in google sheet. Even if row can be empty." ) - # close the connection - conn.close() # list of full sourcelist filenames canfar_sourcelists = get_canfar_sourcelists(band_number=1) # canfar_sourcelists = ['selavy-image.i.EMU_0314-46.SB59159.cont.taylor.0.restored.conv.components.15sig.xml', @@ -548,14 +543,15 @@ def launch_band1_1Dpipeline(database_config_path=None): launch_pipeline(field_ID_no_prefix, tilenumbers, SBid, band) # Update the status to "Running" - conn = db.get_database_connection( - test=False, database_config_path=database_config_path - ) - db.update_partial_tile_1d_pipeline_status( - field_ID, tilenumbers, band_number, "Running", conn - ) - conn.close() - + conn = rest_api.PossumApiClient(database_config_path) + conn.patch("/api/1d-pipeline/partial-tiles/update/status/", + { + "band_number": band_number, + "field_name": field_ID, + "tile_numbers": tilenumbers, + "status": "Running", + }, + format="json") break else: diff --git a/possum_pipeline_control/check_status_and_launch_3Dpipeline_v2.py b/possum_pipeline_control/check_status_and_launch_3Dpipeline_v2.py index 4558288..01c93b0 100644 --- a/possum_pipeline_control/check_status_and_launch_3Dpipeline_v2.py +++ b/possum_pipeline_control/check_status_and_launch_3Dpipeline_v2.py @@ -36,7 +36,7 @@ from vos import Client from automation import canfar_polling -from automation import database_queries as db +from automation import possum_api_client as rest_api from possum_pipeline_control import util from print_all_open_sessions import get_open_sessions @@ -173,11 +173,9 @@ def update_status(tile_number, band, Google_API_token, status): f"Updated tile {tile_number} status to {status} in '3d_pipeline' column in Google Sheet." ) # Also update the DB - conn = db.get_database_connection(test=False) - db.update_3d_pipeline_table( - tile_number, band_number, status, "3d_pipeline_val", conn - ) - conn.close() + conn = rest_api.PossumApiClient() + conn.patch(f"/api/3d-pipeline/tiles/update/3d_pipeline_val/?band_number={band_number}" + f"&tile_number={tile_number}&3d_pipeline_val={status}") else: print(f"Tile {tile_number} not found in the sheet.") @@ -428,13 +426,12 @@ async def launch_band1_3Dpipeline(database_config_path=None): # Check database for band 1 tiles that have been processed by AUSSRC # but not yet processed with 3D pipeline - conn = db.get_database_connection(test=False, database_config_path=database_config_path) + conn = rest_api.PossumApiClient(database_config_path) # We are getting the tiles from the DB instead of the sheet now - tile_numbers = db.get_tiles_for_pipeline_run(conn, band_number=1) + tile_numbers = conn.get("/api/3d-pipeline/tiles/ready-for-3d/band1/") # tile_numbers is a list of single-element tuples, convert to 1D list tile_numbers = [str(tup[0]) for tup in tile_numbers] - conn.close() # Also check whether the tiles have been downloaded to CANFAR canfar_tilenumbers = get_canfar_tiles(band_number=1) diff --git a/possum_pipeline_control/control_3D_pipeline.py b/possum_pipeline_control/control_3D_pipeline.py index d2f874b..2e4df50 100644 --- a/possum_pipeline_control/control_3D_pipeline.py +++ b/possum_pipeline_control/control_3D_pipeline.py @@ -9,7 +9,7 @@ from dotenv import load_dotenv from prefect import flow -from automation import database_queries as db +from automation import possum_api_client as rest_api, database_queries as db from possum_pipeline_control import util from print_all_open_sessions import get_open_sessions @@ -18,8 +18,8 @@ def create_3d_progress_plot(): """Create a progress plot for the 3D pipeline.""" load_dotenv(dotenv_path="./automation/config.env") - conn = db.get_database_connection(test=False) - rows = db.get_3d_tile_data(tile_id=None, band_number=1, conn=conn) + conn = rest_api.PossumApiClient() + rows = conn.get("/api/3d-pipeline/tiles/plotting/band1/") # tile, "3d_pipeline_val", "3d_val_link", "3d_pipeline_ingest", "3d_pipeline", "cube_state" tile3d_table = db.rows_to_table( rows, diff --git a/possum_pipeline_control/ingest3Dpipeline.py b/possum_pipeline_control/ingest3Dpipeline.py index 840ea89..0b631e3 100644 --- a/possum_pipeline_control/ingest3Dpipeline.py +++ b/possum_pipeline_control/ingest3Dpipeline.py @@ -31,7 +31,7 @@ "Could not import possum_run from possum2caom2.composable. Make sure possum2caom2 is installed." ) possum_run = None -from automation import database_queries as db +from automation import possum_api_client as rest_api from possum_pipeline_control import util # 14 (grouped) products for the 3D pipeline @@ -136,23 +136,26 @@ def update_tile_database(tile_number, band_str, status, test_flag, conn): """ print("Updating POSSUM pipeline validation database") band_number = util.get_band_number(band_str) - row = db.get_3d_tile_data(tile_number, band_number, conn) - if len(row) > 0: + rows = conn.get_json(f"/api/3d-pipeline/tiles/tile-id/band{band_number}/{tile_number}/") + + if len(rows) > 0: + ingest_value = rows[0]["3d_pipeline_ingest"] if not test_flag: # Status should be "IngestRunning" otherwise something went wrong - if row[0][3] != "IngestRunning": + if ingest_value != "IngestRunning": raise ValueError( - f"Found status {row[0][3]} while it should be 'IngestRunning'" + f"Found status {ingest_value} while it should be 'IngestRunning'" ) else: print( - f"Testing enabled. Current status of tile {tile_number} is {row[0][3]}" + f"Testing enabled. Current status of tile {tile_number} is {ingest_value}" ) # Update the status in the '3d_pipeline_ingest' column - row_num = db.update_3d_pipeline_table( - tile_number, band_number, status, "3d_pipeline_ingest", conn - ) + response = conn.patch(f"/api/3d-pipeline/tiles/update/3d_pipeline_ingest/?band_number={band_number}" + f"&tile_number={tile_number}" + f"&3d_pipeline_ingest={status}") + row_num = response.data.get('rows_updated') if row_num > 0: print( f"Updated tile {tile_number} status to {status} in '3d_pipeline_ingest' column." @@ -265,9 +268,10 @@ def update_status_spreadsheet(tile_number, band, Google_API_token, date): tile_sheet.update(range_name=f"{col_letter}{tile_index}", values=[[date]]) print(f"Updated tile {tile_number} status to {date} in '3d_pipeline' column.") # Also update the DB - conn = db.get_database_connection(test=False) - db.update_3d_pipeline_table(tile_number, band_number, date, "3d_pipeline", conn) - conn.close() + conn = rest_api.PossumApiClient() + conn.patch(f"/api/3d-pipeline/tiles/update/3d_pipeline/?band_number={band_number}&" + f"tile_number={tile_number}&" + f"3d_pipeline={date}") else: print(f"Tile {tile_number} not found in the sheet.") @@ -326,9 +330,8 @@ def do_ingest( print("_report.txt reports that ingestion failed") # Record the status in the POSSUM Validation database - conn = db.get_database_connection(test=False) + conn = rest_api.PossumApiClient() update_tile_database(tilenumber, band, status, test_flag=test, conn=conn) - conn.close() if status == "Ingested": # If succesful, also record the date of ingestion in POSSUM status spreadsheet diff --git a/possum_pipeline_control/update_partialtile_google_sheet.py b/possum_pipeline_control/update_partialtile_google_sheet.py index 41ee274..c502f04 100644 --- a/possum_pipeline_control/update_partialtile_google_sheet.py +++ b/possum_pipeline_control/update_partialtile_google_sheet.py @@ -34,7 +34,7 @@ import pandas as pd from dotenv import load_dotenv -from automation import database_queries as db +from automation import possum_api_client as rest_api from possum_pipeline_control import util @@ -98,13 +98,14 @@ def get_ready_fields(band: str) -> tuple[at.Table, at.Table]: else: raise ValueError("Band must be either '943MHz' or '1367MHz'") - conn = db.get_database_connection(test=False) - ready_table = db.get_fields_ready_single_SB_pipeline(band_number, conn) - conn.close() - # get rid of the ASKAP- prefix in sbid for easier matching with google sheet - sbids = [row["sbid"].strip("ASKAP-") for row in ready_table] - ready_table["sbid"] = sbids - + conn = rest_api.PossumApiClient() + ready_table = pd.DataFrame( + conn.get_json( + f"/api/1d-pipeline/observations/single-sb-1d-pipeline/fields-ready/band{band_number}/" + ) + ) + # Get rid of the ASKAP- prefix in sbid for easier matching with Google Sheet + ready_table["sbid"] = ready_table["sbid"].str.removeprefix("ASKAP-") ready_table_sheet, full_table_sheet = get_sheet_table(band) if len(ready_table_sheet) != len(ready_table): @@ -123,7 +124,8 @@ def get_ready_fields(band: str) -> tuple[at.Table, at.Table]: print(f" - {fn}") ## getting full table from the database works also, but its a different table structure than Camerons sheet. - # full_table: at.Table = db.get_full_table_single_SB_pipeline(band_number, conn, as_table=True) + # full_table: at.Table = pd.DataFrame( + # conn.get_json(f"/api/1d-pipeline/observations/single-sb-1d-pipeline/full-table/band{band_number}/")) return ready_table, full_table_sheet diff --git a/possum_pipeline_control/update_status_sheet.py b/possum_pipeline_control/update_status_sheet.py index 1877578..56ca1d6 100644 --- a/possum_pipeline_control/update_status_sheet.py +++ b/possum_pipeline_control/update_status_sheet.py @@ -6,7 +6,7 @@ import numpy as np from dotenv import load_dotenv -from automation import database_queries as db +from automation import possum_api_client as rest_api from possum_pipeline_control import util """ @@ -67,11 +67,10 @@ def update_status(tile_number, band, Google_API_token, status): tile_sheet.update(range_name=f"{col_letter}{tile_index}", values=[[status]]) print(f"Updated tile {tile_number} status to {status} in '3d_pipeline' column.") # Also update the DB - conn = db.get_database_connection(test=False) - db.update_3d_pipeline_table( - tile_number, band_number, status, "3d_pipeline_val", conn - ) - conn.close() + conn = rest_api.PossumApiClient() + conn.patch(f"/api/3d-pipeline/tiles/update/3d_pipeline_val/?band_number={band_number}" + f"&tile_number={tile_number}" + f"&3d_pipeline_val={status}") else: print(f"Tile {tile_number} not found in the sheet.") From 6173480f80f7bdd50cb475c7aee4adb5d997339c Mon Sep 17 00:00:00 2001 From: riniangreani2 Date: Fri, 3 Jul 2026 18:26:03 +0800 Subject: [PATCH 4/7] Update readme --- automation/README | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/automation/README b/automation/README index fc351f5..a81ee06 100644 --- a/automation/README +++ b/automation/README @@ -21,19 +21,10 @@ possum-api-password | POSSUM portal password possum-status-sheet | link to POSSUM status Google spreadsheet possum-status-token | Copy the content of the json token to access the possum-status-sheet above -3. Prefect variables (under Variables): ---------------------------------------- -Optional, only if you'd like to override these values: -canfar-num-retries | 2 | Number of retries when CANFAR session fails. -canfar-polling-interval-seconds | 60 | Polling interval (in seconds) to poll CANFAR sessions. - -We need to poll CANFAR periodically because sometimes the sessions silently terminate, and Prefect does not know about it. -Now we'll retry before giving up, and raising an error so to propagate as Prefect flow failure, and send Slack notification. - You can still use your own config.env as below to override the production secrets for testing purposes. 1. Rename config.env.example to config.env. -2. Enter ausSRC database details in the .env file. -3. Enter Google spreadsheet token -4. Update Google spreadsheet file locations (only if the files are different, e.g. for testing) +2. Enter Google spreadsheet token +3. Update Google spreadsheet file locations (only if the files are different, e.g. for testing) +4. Enter POSSUM REST API URL, username and password for using a test REST API instance. From a6255c9ff318fa0c8056959cfd750f67d1d832e3 Mon Sep 17 00:00:00 2001 From: riniangreani2 Date: Wed, 8 Jul 2026 11:58:59 +0800 Subject: [PATCH 5/7] Fix things up after testing REST API --- automation/fix_tile3d_3d_pipeline_status.py | 4 +-- automation/possum_api_client.py | 25 ++++++++++++++----- .../log_processing_status_1D_PartialTiles.py | 4 +-- ...atus_and_launch_1Dpipeline_PartialTiles.py | 4 +-- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/automation/fix_tile3d_3d_pipeline_status.py b/automation/fix_tile3d_3d_pipeline_status.py index 315f8bb..47e7026 100644 --- a/automation/fix_tile3d_3d_pipeline_status.py +++ b/automation/fix_tile3d_3d_pipeline_status.py @@ -54,12 +54,12 @@ def run_update_query( for t in tiles: response = api.patch( "/api/3d-pipeline/tiles/update/3d_pipeline/", - { + json={ "band_number": 1, "tile_number":t, "timestamp": timestamp }, #avoid encoding problem in url - format="json") + ) num_rows = response.data.get('rows_updated') updated_rows += num_rows diff --git a/automation/possum_api_client.py b/automation/possum_api_client.py index 257963a..667ceb8 100644 --- a/automation/possum_api_client.py +++ b/automation/possum_api_client.py @@ -13,17 +13,22 @@ class PossumApiClient: def __init__(self, config_file_path: str = None): - if self._initialized: + if getattr(self, "_initialized", False): return - + + self.base_url = None + self.username = None + self.password = None + config_file = Path(config_file_path) if config_file_path else None if config_file is not None and config_file.exists(): # if config.env is supplied, we'll use the variables from the file load_dotenv(config_file) - self.base_url = os.environ["POSSUM_API_URL"] - self.username = os.environ["POSSUM_API_USERNAME"] - self.password = os.environ["POSSUM_API_PASSWORD"] + self.base_url = os.environ.get("POSSUM_API_URL") + self.username = os.environ.get("POSSUM_API_USERNAME") + self.password = os.environ.get("POSSUM_API_PASSWORD") + if not self.base_url: # otherwise load from Prefect secrets self.base_url = Secret.load("possum-api-url").get() @@ -105,7 +110,15 @@ def _request(self, method, endpoint, **kwargs): **kwargs, ) - response.raise_for_status() + if not response.ok: + try: + error = response.json().get("error", response.text) + except ValueError: + error = response.text + raise requests.HTTPError( + f"{response.status_code} {response.reason}: {error}", + response=response, + ) return response diff --git a/cirada_software/log_processing_status_1D_PartialTiles.py b/cirada_software/log_processing_status_1D_PartialTiles.py index e197154..a0c1af0 100644 --- a/cirada_software/log_processing_status_1D_PartialTiles.py +++ b/cirada_software/log_processing_status_1D_PartialTiles.py @@ -51,13 +51,13 @@ def update_partial_tile_1d_pipeline(field_ID, tile_numbers, band, status, conn): fieldname = util.get_full_field_name(field_ID, band) band_number = util.get_band_number(band) conn.patch("/api/1d-pipeline/partial-tiles/update/status/", - { + json={ "band_number": band_number, "field_name": fieldname, "tile_numbers": tile_numbers, "status": status, }, - format="json") + ) ## TODO: validation in case all tiles have been completed # # Find the validation file path diff --git a/possum_pipeline_control/check_status_and_launch_1Dpipeline_PartialTiles.py b/possum_pipeline_control/check_status_and_launch_1Dpipeline_PartialTiles.py index 366ae30..e6091f9 100644 --- a/possum_pipeline_control/check_status_and_launch_1Dpipeline_PartialTiles.py +++ b/possum_pipeline_control/check_status_and_launch_1Dpipeline_PartialTiles.py @@ -545,13 +545,13 @@ def launch_band1_1Dpipeline(database_config_path=None): # Update the status to "Running" conn = rest_api.PossumApiClient(database_config_path) conn.patch("/api/1d-pipeline/partial-tiles/update/status/", - { + json={ "band_number": band_number, "field_name": field_ID, "tile_numbers": tilenumbers, "status": "Running", }, - format="json") + ) break else: From e885fc887cbdec7ad5658d044cbc36b2e50f1aa1 Mon Sep 17 00:00:00 2001 From: riniangreani2 Date: Fri, 10 Jul 2026 15:51:15 +0800 Subject: [PATCH 6/7] Fix up issues found during manual testing --- automation/compare_sheet_to_database.py | 4 ++-- automation/fix_tile3d_3d_pipeline_status.py | 6 +++--- automation/possum_api_client.py | 14 +++++++++----- .../log_processing_status_1D_PartialTiles.py | 2 +- ..._processing_status_1D_PartialTiles_summary.py | 13 +++++++------ .../log_processing_status_3Dpipeline.py | 6 +++--- .../check_ingest_3Dpipeline.py | 4 ++-- ..._status_and_launch_1Dpipeline_PartialTiles.py | 12 ++++++------ .../check_status_and_launch_3Dpipeline_v2.py | 4 ++-- possum_pipeline_control/control_3D_pipeline.py | 2 +- possum_pipeline_control/ingest3Dpipeline.py | 16 ++++++++++------ .../update_partialtile_google_sheet.py | 4 ++-- possum_pipeline_control/update_status_sheet.py | 2 +- 13 files changed, 49 insertions(+), 40 deletions(-) diff --git a/automation/compare_sheet_to_database.py b/automation/compare_sheet_to_database.py index 33b97d2..e733b7f 100644 --- a/automation/compare_sheet_to_database.py +++ b/automation/compare_sheet_to_database.py @@ -115,7 +115,7 @@ def get_partial_tiles_database(band_number: int = 1) -> list[tuple]: f"Fetching full partial tiles data table for 1D pipeline run for band {band_number} " "from the database." ) - rows = conn.get(f"/api/partial_tiles/sbid/band{band_number}/") + rows = conn.get(f"/partial_tiles/sbid/band{band_number}/") return rows @@ -278,7 +278,7 @@ def get_observation_state_validation(band_number: int = 1) -> dict[str, str]: dict mapping field_name -> normalised 1d_pipeline_validation state. """ conn = possum_api_client.PossumApiClient() - rows = conn.get_json(f"/api/1d-pipeline/observations/single-sb-1d-pipeline/full-table/band{band_number}/") + rows = conn.get_json(f"/1d-pipeline/observations/single-sb-1d-pipeline/full-table/band{band_number}/") state_by_field = {} for row in rows: diff --git a/automation/fix_tile3d_3d_pipeline_status.py b/automation/fix_tile3d_3d_pipeline_status.py index 47e7026..e52d898 100644 --- a/automation/fix_tile3d_3d_pipeline_status.py +++ b/automation/fix_tile3d_3d_pipeline_status.py @@ -53,15 +53,15 @@ def run_update_query( updated_rows = 0 for t in tiles: response = api.patch( - "/api/3d-pipeline/tiles/update/3d_pipeline/", + "/3d-pipeline/tiles/update/3d_pipeline/", json={ "band_number": 1, - "tile_number":t, + "tile_number": t, "timestamp": timestamp }, #avoid encoding problem in url ) - num_rows = response.data.get('rows_updated') + num_rows = response.get('rows_updated') updated_rows += num_rows return updated_rows diff --git a/automation/possum_api_client.py b/automation/possum_api_client.py index 667ceb8..8251ce4 100644 --- a/automation/possum_api_client.py +++ b/automation/possum_api_client.py @@ -45,7 +45,7 @@ def __init__(self, config_file_path: str = None): def login(self): response = self.session.post( - f"{self.base_url}/api/token/", + url = f"{self.base_url.rstrip('/')}/token/", json={ "username": self.username, "password": self.password, @@ -66,7 +66,7 @@ def refresh_access_token(self): return response = self.session.post( - f"{self.base_url}/api/token/refresh/", + url = f"{self.base_url.rstrip('/')}/token/refresh/", json={"refresh": self.refresh_token}, timeout=30, ) @@ -90,10 +90,11 @@ def _headers(self): def _request(self, method, endpoint, **kwargs): self._ensure_authenticated() + url = f"{self.base_url.rstrip('/')}/{endpoint.lstrip('/')}" response = self.session.request( method=method, - url=f"{self.base_url}{endpoint}", + url=url, headers=self._headers(), timeout=30, **kwargs, @@ -104,7 +105,7 @@ def _request(self, method, endpoint, **kwargs): response = self.session.request( method=method, - url=f"{self.base_url}{endpoint}", + url=url, headers=self._headers(), timeout=30, **kwargs, @@ -125,7 +126,10 @@ def _request(self, method, endpoint, **kwargs): def get(self, endpoint, **kwargs): data = self._request("GET", endpoint, **kwargs).json() # return the data as tuples as it was when we queried the DB directly - return [tuple(row.values()) for row in data] + return [ + tuple(row.values()) if isinstance(row, dict) else (row,) + for row in data + ] def get_json(self, endpoint, **kwargs): # get json as is diff --git a/cirada_software/log_processing_status_1D_PartialTiles.py b/cirada_software/log_processing_status_1D_PartialTiles.py index a0c1af0..10c71e9 100644 --- a/cirada_software/log_processing_status_1D_PartialTiles.py +++ b/cirada_software/log_processing_status_1D_PartialTiles.py @@ -50,7 +50,7 @@ def update_partial_tile_1d_pipeline(field_ID, tile_numbers, band, status, conn): """ fieldname = util.get_full_field_name(field_ID, band) band_number = util.get_band_number(band) - conn.patch("/api/1d-pipeline/partial-tiles/update/status/", + conn.patch("/1d-pipeline/partial-tiles/update/status/", json={ "band_number": band_number, "field_name": fieldname, diff --git a/cirada_software/log_processing_status_1D_PartialTiles_summary.py b/cirada_software/log_processing_status_1D_PartialTiles_summary.py index 6d0b334..be4aabd 100644 --- a/cirada_software/log_processing_status_1D_PartialTiles_summary.py +++ b/cirada_software/log_processing_status_1D_PartialTiles_summary.py @@ -65,11 +65,11 @@ def update_1d_database(field_ID, SBid, band, status, conn): full_field_name = util.get_full_field_name(field_ID, band) - response = conn.patch("/api/1d-pipeline/observations/update/1d-pipeline-validation/?" + response = conn.patch("/1d-pipeline/observations/update/1d_pipeline_validation/?" f"band_number={band_number}&" f"field_name={full_field_name}&" f"status={status}") - rows_to_update = response.data.get("rows_updated") + rows_to_update = response.get("rows_updated") if rows_to_update == 0: @@ -82,7 +82,7 @@ def update_1d_database(field_ID, SBid, band, status, conn): else: print("Failed to update the database.") # Check if there are boundary issues for this field and SBID - boundary_issue = conn.get(f"/api/1d-pipeline/partial-tiles/boundary-issues/band{band_number}/{full_field_name}/") + boundary_issue = conn.get(f"/1d-pipeline/partial-tiles/boundary-issues/band{band_number}/{full_field_name}/")[0] return boundary_issue @@ -139,7 +139,7 @@ def update_status_spreadsheet( for row_index in rows_to_update: sleep(2) # 60 writes per minute only tile_sheet.update(range_name=f"{col_letter}{row_index}", values=[[status]]) - api.patch(f"/api/1d-pipeline/observations/update/{status_column.lower()}/?" + api.patch(f"/1d-pipeline/observations/update/{status_column.lower()}/?" f"band_number={band_number}&" f"field_name={full_field_name}&" f"status={status}") @@ -211,6 +211,7 @@ def main(args): # Load constants for Google spreadsheets load_dotenv(dotenv_path=database_config_path) + log_file_path = None if len(log_files) > 1: log_file_path = log_files[-1] @@ -238,8 +239,8 @@ def main(args): # Update the POSSUM Validation database table t1 = task(update_1d_database, name="update_1d_database") # execute tasks serially such that logging is preserved (instead of .submit) - rest_api = rest_api.PossumApiClient(database_config_path) - has_boundary_issue = t1(field_ID, SB_num, band, status, rest_api) + api_client = rest_api.PossumApiClient(database_config_path) + has_boundary_issue = t1(field_ID, SB_num, band, status, api_client) if status == "Completed": # Update the POSSUM Pipeline Status spreadsheet as well. A complete field has been processed! diff --git a/cirada_software/log_processing_status_3Dpipeline.py b/cirada_software/log_processing_status_3Dpipeline.py index a740f44..9b10553 100644 --- a/cirada_software/log_processing_status_3Dpipeline.py +++ b/cirada_software/log_processing_status_3Dpipeline.py @@ -158,11 +158,11 @@ def update_3d_tile_database(tile_number, band, status): # execute query conn = rest_api.PossumApiClient() - response = conn.patch(f"/api/3d-pipeline/tiles/update/3d_pipeline_val/?band_number={band_number}" + response = conn.patch(f"/3d-pipeline/tiles/update/3d_pipeline_val/?band_number={band_number}" f"&tile_number={tile_number}" f"&3d_pipeline_val={status}") - rows_updated = response.data.get('rows_updated') - conn.patch(f"/api/3d-pipeline/tiles/update/3d_val_link/?band_number={band_number}" + rows_updated = response.get('rows_updated') + conn.patch(f"/3d-pipeline/tiles/update/3d_val_link/?band_number={band_number}" f"&tile_number={tile_number}" f"&3d_val_link={validation_link}") diff --git a/possum_pipeline_control/check_ingest_3Dpipeline.py b/possum_pipeline_control/check_ingest_3Dpipeline.py index 092edd2..25d11d4 100644 --- a/possum_pipeline_control/check_ingest_3Dpipeline.py +++ b/possum_pipeline_control/check_ingest_3Dpipeline.py @@ -37,7 +37,7 @@ def get_tiles_for_ingest(band_number, conn): list: A list of tile numbers that satisfy the conditions. """ # Find the tiles that satisfy the conditions - return conn.get_json(f"/api/3d-pipeline/tiles/ready_for_ingest/band{band_number}/") + return conn.get_json(f"/3d-pipeline/tiles/ready-for-ingest/band{band_number}/") def get_canfar_tiles(band_number): client = Client() @@ -109,7 +109,7 @@ def update_status(tile_number, band, status, api): status (str): The status to set in the '3d_pipeline_ingest' column. """ band_no = util.get_band_number(band) - return api.patch(f"/api/3d-pipeline/tiles/update/3d_pipeline_ingest/?band_number={band_no}" + return api.patch(f"/3d-pipeline/tiles/update/3d_pipeline_ingest/?band_number={band_no}" f"&tile_number={tile_number}" f"&3d_pipeline_ingest={status}") diff --git a/possum_pipeline_control/check_status_and_launch_1Dpipeline_PartialTiles.py b/possum_pipeline_control/check_status_and_launch_1Dpipeline_PartialTiles.py index e6091f9..d88c182 100644 --- a/possum_pipeline_control/check_status_and_launch_1Dpipeline_PartialTiles.py +++ b/possum_pipeline_control/check_status_and_launch_1Dpipeline_PartialTiles.py @@ -59,7 +59,7 @@ def get_results_per_field_sbid_skip_edges(band_number, conn, verbose=False): dict: A dictionary with keys as (field_name, sbid) tuples and boolean values indicating whether the conditions are met for the non-edge rows. """ - rows = conn.get(f"/api/1d-pipeline/observations/non-edge-rows/band{band_number}/") + rows = conn.get(f"/1d-pipeline/observations/non-edge-rows/band{band_number}/") results = {} # Group the table by 'field_name' and 'sbid' @@ -83,7 +83,7 @@ def get_results_per_field_sbid(conn, band_number="1", verbose=False): If all Partial tiles for a fieldname have been completed boolean=True, otherwise false. """ # Group the table by 'field_name' and 'sbid' - results = conn.get(f"/api/1d-pipeline/partial-tiles/complete-partial-tiles/band{band_number}/") + results = conn.get(f"/1d-pipeline/partial-tiles/complete-partial-tiles/band{band_number}/") field_sbid_dict = {} # make dict to get rid of duplicates for row in results: @@ -124,7 +124,7 @@ def get_tiles_for_pipeline_run(db_conn, band_number): """ # Find the tiles that satisfy the conditions # (i.e. has an SBID and not yet a '1d_pipeline' status) - rows = db_conn.get(f"/api/1d-pipeline/partial-tiles/ready-for-pipeline/band{band_number}/") + rows = db_conn.get(f"/1d-pipeline/partial-tiles/ready-for-pipeline/band{band_number}/") fields_to_run, SBids_to_run = [], [] tile1_to_run, tile2_to_run, tile3_to_run, tile4_to_run = [], [], [], [] if rows: @@ -346,11 +346,11 @@ def update_validation_status( """ print("Updating partial tile status in the POSSUM pipeline validation sheet.") conn = rest_api.PossumApiClient(database_config_path) - response = conn.patch("/api/1d-pipeline/observations/update/1d-pipeline-validation/?" + response = conn.patch("/1d-pipeline/observations/update/1d-pipeline-validation/?" f"band_number={band_number}&" f"field_name={field_name}&" "status=Running") - row_num = response.data.get("rows_updated") + row_num = response.get("rows_updated") if row_num > 0: print( @@ -544,7 +544,7 @@ def launch_band1_1Dpipeline(database_config_path=None): # Update the status to "Running" conn = rest_api.PossumApiClient(database_config_path) - conn.patch("/api/1d-pipeline/partial-tiles/update/status/", + conn.patch("/1d-pipeline/partial-tiles/update/status/", json={ "band_number": band_number, "field_name": field_ID, diff --git a/possum_pipeline_control/check_status_and_launch_3Dpipeline_v2.py b/possum_pipeline_control/check_status_and_launch_3Dpipeline_v2.py index 01c93b0..993ed08 100644 --- a/possum_pipeline_control/check_status_and_launch_3Dpipeline_v2.py +++ b/possum_pipeline_control/check_status_and_launch_3Dpipeline_v2.py @@ -174,7 +174,7 @@ def update_status(tile_number, band, Google_API_token, status): ) # Also update the DB conn = rest_api.PossumApiClient() - conn.patch(f"/api/3d-pipeline/tiles/update/3d_pipeline_val/?band_number={band_number}" + conn.patch(f"/3d-pipeline/tiles/update/3d_pipeline_val/?band_number={band_number}" f"&tile_number={tile_number}&3d_pipeline_val={status}") else: print(f"Tile {tile_number} not found in the sheet.") @@ -428,7 +428,7 @@ async def launch_band1_3Dpipeline(database_config_path=None): # but not yet processed with 3D pipeline conn = rest_api.PossumApiClient(database_config_path) # We are getting the tiles from the DB instead of the sheet now - tile_numbers = conn.get("/api/3d-pipeline/tiles/ready-for-3d/band1/") + tile_numbers = conn.get("/3d-pipeline/tiles/ready-for-3d/band1/") # tile_numbers is a list of single-element tuples, convert to 1D list tile_numbers = [str(tup[0]) for tup in tile_numbers] diff --git a/possum_pipeline_control/control_3D_pipeline.py b/possum_pipeline_control/control_3D_pipeline.py index 2e4df50..471d3da 100644 --- a/possum_pipeline_control/control_3D_pipeline.py +++ b/possum_pipeline_control/control_3D_pipeline.py @@ -19,7 +19,7 @@ def create_3d_progress_plot(): load_dotenv(dotenv_path="./automation/config.env") conn = rest_api.PossumApiClient() - rows = conn.get("/api/3d-pipeline/tiles/plotting/band1/") + rows = conn.get("/3d-pipeline/tiles/plotting/band1/") # tile, "3d_pipeline_val", "3d_val_link", "3d_pipeline_ingest", "3d_pipeline", "cube_state" tile3d_table = db.rows_to_table( rows, diff --git a/possum_pipeline_control/ingest3Dpipeline.py b/possum_pipeline_control/ingest3Dpipeline.py index 0b631e3..c1e3d36 100644 --- a/possum_pipeline_control/ingest3Dpipeline.py +++ b/possum_pipeline_control/ingest3Dpipeline.py @@ -136,7 +136,7 @@ def update_tile_database(tile_number, band_str, status, test_flag, conn): """ print("Updating POSSUM pipeline validation database") band_number = util.get_band_number(band_str) - rows = conn.get_json(f"/api/3d-pipeline/tiles/tile-id/band{band_number}/{tile_number}/") + rows = conn.get_json(f"/3d-pipeline/tiles/tile-id/band{band_number}/{tile_number}/") if len(rows) > 0: ingest_value = rows[0]["3d_pipeline_ingest"] @@ -152,10 +152,10 @@ def update_tile_database(tile_number, band_str, status, test_flag, conn): ) # Update the status in the '3d_pipeline_ingest' column - response = conn.patch(f"/api/3d-pipeline/tiles/update/3d_pipeline_ingest/?band_number={band_number}" + response = conn.patch(f"/3d-pipeline/tiles/update/3d_pipeline_ingest/?band_number={band_number}" f"&tile_number={tile_number}" f"&3d_pipeline_ingest={status}") - row_num = response.data.get('rows_updated') + row_num = response.get('rows_updated') if row_num > 0: print( f"Updated tile {tile_number} status to {status} in '3d_pipeline_ingest' column." @@ -269,9 +269,13 @@ def update_status_spreadsheet(tile_number, band, Google_API_token, date): print(f"Updated tile {tile_number} status to {date} in '3d_pipeline' column.") # Also update the DB conn = rest_api.PossumApiClient() - conn.patch(f"/api/3d-pipeline/tiles/update/3d_pipeline/?band_number={band_number}&" - f"tile_number={tile_number}&" - f"3d_pipeline={date}") + conn.patch(f"/3d-pipeline/tiles/update/3d_pipeline/", + json={ + "band_number": band_number, + "tile_number": tile_number, + "timestamp": date + }, #avoid encoding problem in url + ) else: print(f"Tile {tile_number} not found in the sheet.") diff --git a/possum_pipeline_control/update_partialtile_google_sheet.py b/possum_pipeline_control/update_partialtile_google_sheet.py index c502f04..c534742 100644 --- a/possum_pipeline_control/update_partialtile_google_sheet.py +++ b/possum_pipeline_control/update_partialtile_google_sheet.py @@ -101,7 +101,7 @@ def get_ready_fields(band: str) -> tuple[at.Table, at.Table]: conn = rest_api.PossumApiClient() ready_table = pd.DataFrame( conn.get_json( - f"/api/1d-pipeline/observations/single-sb-1d-pipeline/fields-ready/band{band_number}/" + f"/1d-pipeline/observations/single-sb-1d-pipeline/fields-ready/band{band_number}/" ) ) # Get rid of the ASKAP- prefix in sbid for easier matching with Google Sheet @@ -125,7 +125,7 @@ def get_ready_fields(band: str) -> tuple[at.Table, at.Table]: ## getting full table from the database works also, but its a different table structure than Camerons sheet. # full_table: at.Table = pd.DataFrame( - # conn.get_json(f"/api/1d-pipeline/observations/single-sb-1d-pipeline/full-table/band{band_number}/")) + # conn.get_json(f"/1d-pipeline/observations/single-sb-1d-pipeline/full-table/band{band_number}/")) return ready_table, full_table_sheet diff --git a/possum_pipeline_control/update_status_sheet.py b/possum_pipeline_control/update_status_sheet.py index 56ca1d6..2d49332 100644 --- a/possum_pipeline_control/update_status_sheet.py +++ b/possum_pipeline_control/update_status_sheet.py @@ -68,7 +68,7 @@ def update_status(tile_number, band, Google_API_token, status): print(f"Updated tile {tile_number} status to {status} in '3d_pipeline' column.") # Also update the DB conn = rest_api.PossumApiClient() - conn.patch(f"/api/3d-pipeline/tiles/update/3d_pipeline_val/?band_number={band_number}" + conn.patch(f"/3d-pipeline/tiles/update/3d_pipeline_val/?band_number={band_number}" f"&tile_number={tile_number}" f"&3d_pipeline_val={status}") else: From 3c697b864fbcb334cc2973b3b30a2768c03cacbc Mon Sep 17 00:00:00 2001 From: riniangreani2 Date: Mon, 13 Jul 2026 13:01:21 +0800 Subject: [PATCH 7/7] New unit test for REST API. Deprecate existing automation unit tests --- automation/unit_tests/README | 3 + .../unit_tests/_3dpipeline_base_test.py | 2 + .../unit_tests/partial_tile_1d_base_test.py | 2 + .../test_check_ingest_3dpipeline.py | 2 + ...atus_and_launch_1dpipeline_partialtiles.py | 2 + ...t_check_status_and_launch_3dpipeline_v2.py | 2 + .../unit_tests/test_ingest3dpipeline.py | 2 + ...t_log_processing_status_1D_PartialTiles.py | 2 + ...ocessing_status_1D_PartialTiles_summary.py | 2 + .../test_log_processing_status_3Dpipeline.py | 2 + possum_pipeline_control/test_rest_api.py | 109 ++++++++++++++++++ 11 files changed, 130 insertions(+) create mode 100644 possum_pipeline_control/test_rest_api.py diff --git a/automation/unit_tests/README b/automation/unit_tests/README index 17d5448..c650dab 100644 --- a/automation/unit_tests/README +++ b/automation/unit_tests/README @@ -1,3 +1,6 @@ +!!DEPRECATED since July 13, 2026!! +!!Direct database access has been replaced by REST API. The tests are now covered in possum_pipeline_control/test_rest_api.py!! + The tests use the database connection details used in test.env file. Please create the test.env file with the following keys: TEST_DATABASE_HOST= diff --git a/automation/unit_tests/_3dpipeline_base_test.py b/automation/unit_tests/_3dpipeline_base_test.py index fe864f1..fe4a726 100644 --- a/automation/unit_tests/_3dpipeline_base_test.py +++ b/automation/unit_tests/_3dpipeline_base_test.py @@ -1,4 +1,6 @@ """ +DEPRECATED since July 13, 2026: Direct database access has been replaced by REST API. The tests are now covered in possum_pipeline_control/test_rest_api.py + Base class for setting up 3D Pipeline test cases. This is to reduce repetitions across test cases because the setup is all the same. """ diff --git a/automation/unit_tests/partial_tile_1d_base_test.py b/automation/unit_tests/partial_tile_1d_base_test.py index dfc14d5..ca20291 100644 --- a/automation/unit_tests/partial_tile_1d_base_test.py +++ b/automation/unit_tests/partial_tile_1d_base_test.py @@ -1,4 +1,6 @@ """ +DEPRECATED since July 13, 2026: Direct database access has been replaced by REST API. The tests are now covered in possum_pipeline_control/test_rest_api.py + Base class for setting up Partial Tile 1D test cases. This is to reduce repetitions across test cases because the setup is all the same. """ diff --git a/automation/unit_tests/test_check_ingest_3dpipeline.py b/automation/unit_tests/test_check_ingest_3dpipeline.py index 231c63d..e246721 100644 --- a/automation/unit_tests/test_check_ingest_3dpipeline.py +++ b/automation/unit_tests/test_check_ingest_3dpipeline.py @@ -1,4 +1,6 @@ """ +DEPRECATED since July 13, 2026: Direct database access has been replaced by REST API. The tests are now covered in possum_pipeline_control/test_rest_api.py + Test possum_pipeline_control: check_ingest_3Dpipeline.py """ diff --git a/automation/unit_tests/test_check_status_and_launch_1dpipeline_partialtiles.py b/automation/unit_tests/test_check_status_and_launch_1dpipeline_partialtiles.py index 75f6a36..3847ab6 100644 --- a/automation/unit_tests/test_check_status_and_launch_1dpipeline_partialtiles.py +++ b/automation/unit_tests/test_check_status_and_launch_1dpipeline_partialtiles.py @@ -1,4 +1,6 @@ """ +DEPRECATED since July 13, 2026: Direct database access has been replaced by REST API. The tests are now covered in possum_pipeline_control/test_rest_api.py + Test possum_pipeline_control: check_status_and_launch_1Dpipeline_PartialTiles.py """ diff --git a/automation/unit_tests/test_check_status_and_launch_3dpipeline_v2.py b/automation/unit_tests/test_check_status_and_launch_3dpipeline_v2.py index 3fa38ca..db5d8f5 100644 --- a/automation/unit_tests/test_check_status_and_launch_3dpipeline_v2.py +++ b/automation/unit_tests/test_check_status_and_launch_3dpipeline_v2.py @@ -1,4 +1,6 @@ """ +DEPRECATED since July 13, 2026: Direct database access has been replaced by REST API. The tests are now covered in possum_pipeline_control/test_rest_api.py + Test possum_pipeline_control: check_status_and_launch_3Dpipeline_v2.py """ diff --git a/automation/unit_tests/test_ingest3dpipeline.py b/automation/unit_tests/test_ingest3dpipeline.py index 8ce063a..f53f5db 100644 --- a/automation/unit_tests/test_ingest3dpipeline.py +++ b/automation/unit_tests/test_ingest3dpipeline.py @@ -1,4 +1,6 @@ """ +DEPRECATED since July 13, 2026: Direct database access has been replaced by REST API. The tests are now covered in possum_pipeline_control/test_rest_api.py + Test possum_pipeline_control: Ingest3Dpipeline.py """ diff --git a/automation/unit_tests/test_log_processing_status_1D_PartialTiles.py b/automation/unit_tests/test_log_processing_status_1D_PartialTiles.py index 1dc5ce5..ce98909 100644 --- a/automation/unit_tests/test_log_processing_status_1D_PartialTiles.py +++ b/automation/unit_tests/test_log_processing_status_1D_PartialTiles.py @@ -1,4 +1,6 @@ """ +DEPRECATED since July 13, 2026: Direct database access has been replaced by REST API. The tests are now covered in possum_pipeline_control/test_rest_api.py + Test cirada_software: log_processing_status_1D_PartialTiles.py """ diff --git a/automation/unit_tests/test_log_processing_status_1D_PartialTiles_summary.py b/automation/unit_tests/test_log_processing_status_1D_PartialTiles_summary.py index 42ec2df..d1d3e57 100644 --- a/automation/unit_tests/test_log_processing_status_1D_PartialTiles_summary.py +++ b/automation/unit_tests/test_log_processing_status_1D_PartialTiles_summary.py @@ -1,4 +1,6 @@ """ +DEPRECATED since July 13, 2026: Direct database access has been replaced by REST API. The tests are now covered in possum_pipeline_control/test_rest_api.py + Test cirada_software: log_processing_status_1D_PartialTiles_summary.py """ diff --git a/automation/unit_tests/test_log_processing_status_3Dpipeline.py b/automation/unit_tests/test_log_processing_status_3Dpipeline.py index 5b9ca12..e36caee 100644 --- a/automation/unit_tests/test_log_processing_status_3Dpipeline.py +++ b/automation/unit_tests/test_log_processing_status_3Dpipeline.py @@ -1,4 +1,6 @@ """ +DEPRECATED since July 13, 2026: Direct database access has been replaced by REST API. The tests are now covered in possum_pipeline_control/test_rest_api.py + Test cirada_software: log_processing_status_3Dpipeline.py """ diff --git a/possum_pipeline_control/test_rest_api.py b/possum_pipeline_control/test_rest_api.py new file mode 100644 index 0000000..ceac02d --- /dev/null +++ b/possum_pipeline_control/test_rest_api.py @@ -0,0 +1,109 @@ +""" +These tests require a local Possum REST API server running connected with a copy of database set up locally. +You also need to run a local Prefect server with the following secrets: +- possum-api-url +- possum-api-username +- possum-api-password +Alteratively you can create a config.env in automation folder with the following keys (and values): +POSSUM_API_USERNAME= +POSSUM_API_PASSWORD= +POSSUM_API_URL= +There are no assertions because it will depend on the database copy. +You will need to observe the output and compare against the database. +""" +from datetime import date +from automation import possum_api_client as rest_api + +band_number = 1 +tile_number = 11315 +status = "COMPLETED" + +def test_update_1d_pipeline_validation(): + full_field_name = "EMU_1058-60" + status = "test" + conn = rest_api.PossumApiClient() # pass in config.env if using config.env instead of Prefect secrets + response = conn.patch("/1d-pipeline/observations/update/1d_pipeline_validation/?" + f"band_number={band_number}&" + f"field_name={full_field_name}&" + f"status={status}") + rows_to_update = response.get("rows_updated") + print('Rows updated:', rows_to_update) + +def test_update_single_sb_1d_pipeline(): + full_field_name = "EMU_1058-60" + status_column = "single_SB_1D_pipeline" + conn = rest_api.PossumApiClient() # pass in config.env if using config.env instead of Prefect secrets + response = conn.patch(f"/1d-pipeline/observations/update/{status_column.lower()}/?" + f"band_number={band_number}&" + f"field_name={full_field_name}&" + f"status={status}") + rows_to_update = response.get("rows_updated") + print('Rows updated:', rows_to_update) + +def test_update_partial_tiles(): + full_field_name = "EMU_1058-60" + conn = rest_api.PossumApiClient() # pass in config.env if using config.env instead of Prefect secrets + response = conn.patch("/1d-pipeline/partial-tiles/update/status/", + json={ + "band_number": band_number, + "field_name": full_field_name, + "tile_numbers": [tile_number, '', '', ''], + "status": status, + }, + ) + rows_to_update = response.get("rows_updated") + print('Rows updated:', rows_to_update) + +def test_update_3d_pipeline_val(): + conn = rest_api.PossumApiClient() # pass in config.env if using config.env instead of Prefect secrets + response = conn.patch(f"/3d-pipeline/tiles/update/3d_pipeline_val/?band_number={band_number}" + f"&tile_number={tile_number}" + f"&3d_pipeline_val={status}") + rows_updated = response.get('rows_updated') + print('Rows updated:', rows_updated) + +def test_update_3d_val_link(): + conn = rest_api.PossumApiClient() # pass in config.env if using config.env instead of Prefect secrets + response = conn.patch(f"/3d-pipeline/tiles/update/3d_val_link/?band_number={band_number}" + f"&tile_number={tile_number}" + f"&3d_val_link='test'") + rows_updated = response.get('rows_updated') + print('Rows updated:', rows_updated) + +def test_update_3d_pipeline(): + conn = rest_api.PossumApiClient() # pass in config.env if using config.env instead of Prefect secrets + date = date.today().isoformat() + response = conn.patch(f"/3d-pipeline/tiles/update/3d_pipeline/?band_number={band_number}", + json={ + "band_number": 1, + "tile_number": tile_number, + "timestamp": date + }, #avoid encoding problem in url + ) + print('Rows updated:', response.get("rows_updated")) + +def test_tiles_ready_for_3d_pipeline(): + conn = rest_api.PossumApiClient() # pass in config.env if using config.env instead of Prefect secrets + tile_numbers = conn.get("/3d-pipeline/tiles/ready-for-3d/band1/") + # tile_numbers is a list of single-element tuples, convert to 1D list + tile_numbers = [str(tup[0]) for tup in tile_numbers] + print(tile_numbers) + +def test_tiles_ready_for_ingest(): + conn = rest_api.PossumApiClient() # pass in config.env if using config.env instead of Prefect secrets + tile_numbers = conn.get_json(f"/3d-pipeline/tiles/ready-for-ingest/band{band_number}/") + print(tile_numbers) + +def test_3d_plotting(): + conn = rest_api.PossumApiClient() # pass in config.env if using config.env instead of Prefect secrets + rows = conn.get("/3d-pipeline/tiles/plotting/band1/") + # tile, "3d_pipeline_val", "3d_val_link", "3d_pipeline_ingest", "3d_pipeline", "cube_state" + print(rows) + +def test_tiles_by_tile_id(): + conn = rest_api.PossumApiClient() # pass in config.env if using config.env instead of Prefect secrets + rows = conn.get_json(f"/3d-pipeline/tiles/tile-id/band{band_number}/{tile_number}/") + print(rows) + if len(rows) > 0: + ingest_value = rows[0]["3d_pipeline_ingest"] + print('ingest value ', ingest_value) \ No newline at end of file