Use REST API instead of database connection

automation/README

@@ -15,27 +15,16 @@ PREFECT_API_URL : IP address of the Prefect server to bypass the OAuth Proxy (Gi
 2. Prefect secrets (under Blocks):
 ----------------------------------
 cadc-proxy-pem | Generate cadcproxy.pem with CANFAR interactive session, and copy the content here. 
-database-host | POSSUM database IP address
-database-name | POSSUM database name
-database-password |  POSSUM database password
-database-port |  POSSUM database port
-database-user |  POSSUM database username
+possum-api-url | POSSUM REST API URL
+possum-api-username | POSSUM portal username with update privileges
+possum-api-password | POSSUM portal password
 possum-status-sheet |  link to POSSUM status Google spreadsheet
 possum-status-token |  Copy the content of the json token to access the possum-status-sheet above
 
-3. Prefect variables (under Variables):
----------------------------------------
-Optional, only if you'd like to override these values:
-canfar-num-retries | 2 | Number of retries when CANFAR session fails.
-canfar-polling-interval-seconds | 60 | Polling interval (in seconds) to poll CANFAR sessions. 
-
-We need to poll CANFAR periodically because sometimes the sessions silently terminate, and Prefect does not know about it. 
-Now we'll retry before giving up, and raising an error so to propagate as Prefect flow failure, and send Slack notification.
-
 You can still use your own config.env as below to override the production secrets for testing purposes.
 1. Rename config.env.example to config.env.
-2. Enter ausSRC database details in the .env file. 
-3. Enter Google spreadsheet token
-4. Update Google spreadsheet file locations (only if the files are different, e.g. for testing)
+2. Enter Google spreadsheet token
+3. Update Google spreadsheet file locations (only if the files are different, e.g. for testing)
+4. Enter POSSUM REST API URL, username and password for using a test REST API instance.
 
 

automation/compare_sheet_to_database.py

@@ -13,7 +13,7 @@
 import tqdm
 
 # assume this script is run as a module from the POSSUMutils package
-from automation import database_queries as db  # noqa: E402
+from automation import possum_api_client # noqa: E402
 
 GOOGLE_API_TOKEN = "/home/erik/.ssh/neural-networks--1524580309831-c5c723e2468e.json"
 VALIDATION_SHEET_URL = (
@@ -109,25 +109,14 @@ def get_partial_tiles_database(band_number: int = 1) -> list[tuple]:
     Returns:
         list[tuple]: Each tuple is a row from the database with p.* and o.sbid as last field.
     """
-    conn = db.get_database_connection(test=False)
-    try:
-        print(
-            f"Fetching full partial tiles data table for 1D pipeline run for band {band_number} "
-            "from the database."
-        )
-        query = f"""
-            SELECT p.*, o.sbid
-            FROM possum.partial_tile_1d_pipeline_band{band_number} AS p
-            LEFT JOIN possum.observation AS o ON p.observation = o.name
-        """
-        rows = db.execute_query(query, conn)
-    finally:
-        try:
-            conn.close()
-        except Exception:
-            # If the connection object is not closable, ignore.
-            pass
-
+    conn = possum_api_client.PossumApiClient()
+
+    print(
+        f"Fetching full partial tiles data table for 1D pipeline run for band {band_number} "
+        "from the database."
+    )
+    rows = conn.get(f"/partial_tiles/sbid/band{band_number}/")
+
     return rows
 
 
@@ -288,23 +277,13 @@ def get_observation_state_validation(band_number: int = 1) -> dict[str, str]:
     Returns:
         dict mapping field_name -> normalised 1d_pipeline_validation state.
     """
-    conn = db.get_database_connection(test=False)
-    try:
-        query = f"""
-            SELECT name, "1d_pipeline_validation"
-            FROM possum.observation_state_band{band_number}
-        """
-        rows = db.execute_query(query, conn)
-    finally:
-        try:
-            conn.close()
-        except Exception:
-            pass
-
-    state_by_field: dict[str, str] = {}
-    for field_name, validation_state in rows:
-        state_by_field[str(field_name)] = normalize_value(validation_state)
-
+    conn = possum_api_client.PossumApiClient()
+    rows = conn.get_json(f"/1d-pipeline/observations/single-sb-1d-pipeline/full-table/band{band_number}/")
+
+    state_by_field = {}
+    for row in rows:
+        state_by_field[row["name"]] = normalize_value(row["1d_pipeline_validation"])
+
     return state_by_field
 
 

automation/config.env.example

@@ -3,10 +3,9 @@ PREFECT_API_URL=
 PREFECT_API_AUTH_STRING=
 
 # Optional unless testing with custom database and Google spreadsheet
-DATABASE_HOST=
-DATABASE_PORT=
-DATABASE_NAME=
-DATABASE_USER=
-DATABASE_PASSWORD=
+# TODO update the prefect secrets
+POSSUM_API_URL=
+POSSUM_API_USERNAME=
+POSSUM_API_PASSWORD=
 POSSUM_STATUS_TOKEN=/arc/home/ErikOsinga/.ssh/psm_gspread_token.json
 POSSUM_STATUS_SHEET=https://docs.google.com/spreadsheets/d/1sWCtxSSzTwjYjhxr1_KVLWG2AnrHwSJf_RWQow7wbH0

automation/database_queries.py

@@ -1,5 +1,6 @@
 """
 Database query functions for interacting with the ausSRC database.
+DEPRECATED as of Jun 23 2026 - This has been replaced with api_client.py. We can no longer directly query the database.
 """
 
 import os

automation/fix_tile3d_3d_pipeline_status.py

@@ -16,13 +16,13 @@
 from astropy import table as at
 
 # assume this script is run as a module from the POSSUMutils package
-from automation import database_queries as db
+from automation import possum_api_client
 
 GOOGLE_API_TOKEN = "/home/erik/.ssh/psm_gspread_token.json"
 VALIDATION_SHEET_URL = "https://docs.google.com/spreadsheets/d/1sWCtxSSzTwjYjhxr1_KVLWG2AnrHwSJf_RWQow7wbH0/"
 
 
-def build_update_query(
+def run_update_query(
     tiles: np.ndarray,
     timestamp: Optional[datetime.datetime] = None,
 ) -> str:
@@ -48,30 +48,28 @@ def build_update_query(
     if tiles.size == 0:
         raise ValueError("No tiles provided to update.")
 
-    # Ensure all entries are integers and build the IN list
-    tile_list_str = ", ".join(str(int(t)) for t in tiles)
-
-    if timestamp is None:
-        # Use DB server time
-        timestamp_expr = "NOW()"
-    else:
-        # Format as 'YYYY-MM-DD HH:MM:SS' and cast explicitly
-        ts_str = timestamp.replace(microsecond=0).isoformat(sep=" ")
-        timestamp_expr = f"'{ts_str}'::timestamp"
-
-    query = f"""
-        UPDATE possum.tile_state_band1
-        SET "3d_pipeline" = {timestamp_expr}
-        WHERE tile IN ({tile_list_str});
-    """
+    api = possum_api_client.PossumApiClient()
+
+    updated_rows = 0
+    for t in tiles:
+        response = api.patch(
+            "/3d-pipeline/tiles/update/3d_pipeline/",
+            json={
+             "band_number": 1,
+             "tile_number": t,
+             "timestamp": timestamp
+            }, #avoid encoding problem in url
+        )
+
+        num_rows = response.get('rows_updated')
+        updated_rows += num_rows
 
-    return query.strip()
+    return updated_rows
 
 
 def update_3d_pipeline_for_tiles(
     tiles: np.ndarray,
     timestamp: Optional[datetime.datetime] = None,
-    test: bool = False,
 ) -> None:
     """
     Execute the UPDATE to set "3d_pipeline" for the given tiles.
@@ -82,22 +80,10 @@ def update_3d_pipeline_for_tiles(
         1D array of tile ids (integers).
     timestamp : datetime.datetime or None
         If None, uses NOW() in the database.
-    test : bool
-        Passed through to db.get_database_connection(test=...).
     """
-    query = build_update_query(tiles, timestamp=timestamp)
-
-    conn = db.get_database_connection(test=test)
-    try:
-        rows = db.execute_query(query, conn)
-        # If the connection is not in autocommit mode, commit explicitly:
-        if hasattr(conn, "commit"):
-            conn.commit()
-    finally:
-        if hasattr(conn, "close"):
-            conn.close()
+    num_rows = run_update_query(tiles, timestamp=timestamp)
 
-    print(f"Update executed successfully. Updated {len(rows)} rows")
+    print(f"Update executed successfully. Updated {num_rows} rows")
 
 
 def load_tile_validation_sheet(band: str = "943MHz") -> at.Table:
@@ -141,8 +127,8 @@ def load_tile_validation_sheet(band: str = "943MHz") -> at.Table:
     print(f"Number of tiles to update: {len(processed_tiles)}")
 
     # Option A: use NOW() on the DB side
-    update_3d_pipeline_for_tiles(processed_tiles, timestamp=None, test=False)
+    update_3d_pipeline_for_tiles(processed_tiles, timestamp=None)
 
     # Option B: use an explicit timestamp
     # explicit_ts = datetime.datetime(2025, 1, 1, 12, 0, 0)
-    # update_3d_pipeline_for_tiles(example_tiles, timestamp=explicit_ts, test=False)
+    # update_3d_pipeline_for_tiles(example_tiles, timestamp=explicit_ts)