UI Modernisation

.github/workflows/dev.yml

@@ -29,7 +29,6 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: '3.12'
-          cache: 'pip'
 
       - run: pip install -r requirements.txt
 
@@ -59,7 +58,6 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: '3.12'
-          cache: 'pip'
 
       - name: Setup API environment
         run: make backend/setup
@@ -92,7 +90,6 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: '3.12'
-          cache: 'pip'
 
       - name: Setup Python Environment
         run: |

.github/workflows/main.yml

@@ -28,7 +28,6 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: '3.12'
-          cache: 'pip'
 
       - run: pip install -r requirements.txt
 
@@ -58,7 +57,6 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: '3.12'
-          cache: 'pip'
 
       - name: Setup API environment
         run: make backend/setup

.github/workflows/release_sdk.yml

@@ -33,7 +33,6 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: '3.12'
-          cache: 'pip'
 
       - name: Setup Python Environment
         run: |

.github/workflows/ui-tests.yml

@@ -41,3 +41,13 @@ jobs:
           E2E_DOMAIN_NAME: ${{ secrets.E2E_DOMAIN_NAME }}
           E2E_RESOURCE_PREFIX: ${{ secrets.E2E_RESOURCE_PREFIX }}
           AWS_REGION: ${{ secrets.AWS_REGION }}
+
+      - name: Upload Playwright results
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-results
+          path: |
+            frontend/test-results/
+            frontend/playwright-report/
+          retention-days: 7

Makefile

@@ -191,7 +191,7 @@ infra/backend:			## Create terraform backend for infrastructure
 	@cd infrastructure/; ./scripts/infra_make_helper.sh create_backend
 
 infra/init:			## Terraform init: make infra/init block=<infra/block>
-	@cd infrastructure/; ./scripts/infra_make_helper.sh run_init "${block}"
+	@cd infrastructure/; ./scripts/infra_make_helper.sh run_init "${block}" "${args}"
 
 infra/plan:			## Terraform view infrastructure changes: make infra/plan block=<infra/block>
 	@cd infrastructure/; ./scripts/infra_make_helper.sh run_tf plan "${block}" "${env}"

backend/api/application/services/authorisation/authorisation_service.py

@@ -164,6 +164,17 @@ def get_subject_id(request: Request):
     return parsed_token.subject if token else None
 
 
+def get_username(request: Request) -> Optional[str]:
+    token = get_token(request)
+    if not token:
+        return None
+    try:
+        parsed_token = parse_token(token)
+    except DecodeError:
+        return None
+    return parsed_token.username
+
+
 def check_credentials_availability(
     browser_request: bool,
     client_token: Optional[str],

backend/api/application/services/data_service.py

@@ -1,4 +1,5 @@
 import uuid
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from threading import Thread
 from typing import List, Tuple
@@ -191,6 +192,22 @@ def get_last_updated_time(self, metadata: DatasetMetadata) -> str:
         )
         return last_updated or "Never updated"
 
+    def enrich_datasets_for_ui(self, datasets: List[DatasetMetadata]) -> List[dict]:
+        def enrich(dataset: DatasetMetadata) -> dict:
+            d = dataset.to_dict()
+            try:
+                d["last_updated"] = self.get_last_updated_time(dataset)
+            except Exception:
+                d["last_updated"] = None
+            try:
+                d["last_uploaded_by"] = self.get_last_uploader(dataset)
+            except Exception:
+                d["last_uploaded_by"] = None
+            return d
+
+        with ThreadPoolExecutor(max_workers=10) as pool:
+            return list(pool.map(enrich, datasets))
+
     def get_last_uploader(self, metadata: DatasetMetadata) -> str:
         """
         Get the name of the user who last successfully uploaded to this dataset.

backend/api/controller/datasets.py

@@ -18,13 +18,13 @@
     DatasetAccessEvaluator,
 )
 from api.application.services.data_service import DataService
-
+from api.application.services.job_service import JobService
 from api.application.services.delete_service import DeleteService
 from api.application.services.format_service import FormatService
 from api.application.services.schema_service import SchemaService
 from api.application.services.search_service import SearchService
 from api.common.data_handlers import store_file_to_disk
-from api.common.utilities import strtobool
+from api.common.utilities import build_error_message_list, construct_dataset_metadata, strtobool
 from api.common.config.auth import Action
 from api.common.config.constants import (
     BASE_API_PATH,
@@ -40,7 +40,6 @@
     InvalidFileUploadError,
 )
 from api.common.logger import AppLogger
-from api.common.utilities import construct_dataset_metadata
 from api.domain.dataset_filters import DatasetFilters
 from api.domain.dataset_metadata import DatasetMetadata
 from api.domain.schema_metadata import SchemaMetadata
@@ -53,6 +52,7 @@
 
 athena_adapter = AthenaAdapter()
 data_service = DataService()
+job_service = JobService()
 delete_service = DeleteService()
 schema_service = SchemaService()
 data_access_evaluator = DatasetAccessEvaluator()
@@ -567,9 +567,15 @@ async def query_dataset(
     ### Click  `Try it out` to use the endpoint
 
     """
-    df = data_service.query_data(
-        construct_dataset_metadata(layer, domain, dataset, version), query
-    )
+    subject_id = get_subject_id(request)
+    dataset_metadata = construct_dataset_metadata(layer, domain, dataset, version)
+    query_job = job_service.create_query_job(subject_id, dataset_metadata)
+    try:
+        df = data_service.query_data(dataset_metadata, query)
+    except Exception as error:
+        job_service.fail(query_job, build_error_message_list(error))
+        raise
+    job_service.succeed_query(query_job, url=None)
     if df.shape[0] == 0:
         # Return 204 if dataframe is empty
         return PlainTextResponse(

backend/api/domain/schema_metadata.py

@@ -40,11 +40,7 @@ def get_is_latest_version(self) -> bool:
         return self.is_latest_version
 
     def remove_duplicates(self):
-        updated_key_only_list = []
-
-        if len(self.key_only_tags) != 0 and self.key_value_tags:
-            for key in self.key_only_tags:
-                if key not in self.key_value_tags.keys():
-                    updated_key_only_list.append(key)
-
-        self.key_only_tags = updated_key_only_list
+        if self.key_value_tags:
+            self.key_only_tags = [
+                key for key in self.key_only_tags if key not in self.key_value_tags
+            ]

backend/api/domain/token.py

@@ -4,6 +4,7 @@
 class Token:
     def __init__(self, payload: dict):
         self.subject: str = self._extract_subject(payload)
+        self.username: str | None = payload.get("cognito:username") or payload.get("username")
 
     def _extract_subject(self, payload: dict) -> str:
         try:

backend/api/entry.py

@@ -11,11 +11,13 @@
 from api.application.services.authorisation.authorisation_service import (
     secure_endpoint,
     get_subject_id,
+    get_username,
 )
 from api.application.services.permissions_service import PermissionsService
 from api.application.services.authorisation.dataset_access_evaluator import (
     DatasetAccessEvaluator,
 )
+from api.application.services.data_service import DataService
 from api.common.config.auth import IDENTITY_PROVIDER_BASE_URL, Action
 from api.common.config.docs import (
     custom_openapi_docs_generator,
@@ -57,6 +59,7 @@
 
 permissions_service = PermissionsService()
 upload_service = DatasetAccessEvaluator()
+data_service = DataService()
 
 app = FastAPI(
     openapi_url=f"{BASE_API_PATH}/openapi.json", docs_url=None
@@ -172,7 +175,7 @@ async def methods(request: Request):
     except AWSServiceError as error:
         error_message = error.message
 
-    return {"error_message": error_message, **allowed_actions}
+    return {"error_message": error_message, "username": get_username(request), **allowed_actions}
 
 
 @app.get(
@@ -194,7 +197,7 @@ async def get_permissions_ui():
 async def get_datasets_ui(action: Action, request: Request):
     subject_id = get_subject_id(request)
     datasets = upload_service.get_authorised_datasets(subject_id, action)
-    return [dataset.to_dict() for dataset in datasets]
+    return data_service.enrich_datasets_for_ui(datasets)
 
 
 @app.get("/favicon.ico", include_in_schema=False)

backend/test/api/controller/test_datasets.py

@@ -10,6 +10,7 @@
 )
 from api.application.services.data_service import DataService
 from api.application.services.delete_service import DeleteService
+from api.application.services.job_service import JobService
 from api.application.services.search_service import SearchService
 from api.common.custom_exceptions import (
     UserError,
@@ -819,9 +820,12 @@ def test_returns_error_response_when_domain_uppercase(self):
             "details": ["domain -> was required to be lowercase only."]
         }
 
+    @patch.object(JobService, "succeed_query")
+    @patch.object(JobService, "create_query_job")
+    @patch("api.controller.datasets.get_subject_id")
     @patch.object(DataService, "query_data")
     def test_call_service_with_only_domain_dataset_when_no_json_provided(
-        self, mock_query_method
+        self, mock_query_method, mock_get_subject_id, mock_create_query_job, mock_succeed_query
     ):
         query_url = f"{BASE_API_PATH}/datasets/raw/mydomain/mydataset/query?version=1"
 
@@ -833,8 +837,13 @@ def test_call_service_with_only_domain_dataset_when_no_json_provided(
             DatasetMetadata("raw", "mydomain", "mydataset", 1), Query()
         )
 
+    @patch.object(JobService, "succeed_query")
+    @patch.object(JobService, "create_query_job")
+    @patch("api.controller.datasets.get_subject_id")
     @patch.object(DataService, "query_data")
-    def test_call_service_with_sql_query_when_json_provided(self, mock_query_method):
+    def test_call_service_with_sql_query_when_json_provided(
+        self, mock_query_method, mock_get_subject_id, mock_create_query_job, mock_succeed_query
+    ):
         request_json = {"select_columns": ["column1"], "limit": "10"}
 
         query_url = f"{BASE_API_PATH}/datasets/raw/mydomain/mydataset/query?version=1"
@@ -848,10 +857,13 @@ def test_call_service_with_sql_query_when_json_provided(self, mock_query_method)
             Query(select_columns=["column1"], limit="10"),
         )
 
+    @patch.object(JobService, "succeed_query")
+    @patch.object(JobService, "create_query_job")
+    @patch("api.controller.datasets.get_subject_id")
     @patch("api.controller.datasets.construct_dataset_metadata")
     @patch.object(DataService, "query_data")
     def test_call_service_with_latest_version_when_none_provided(
-        self, mock_query_method, mock_construct_metadata
+        self, mock_query_method, mock_construct_metadata, mock_get_subject_id, mock_create_query_job, mock_succeed_query
     ):
         mock_construct_metadata.return_value = DatasetMetadata(
             "raw", "mydomain", "mydataset", 32
@@ -864,9 +876,12 @@ def test_call_service_with_latest_version_when_none_provided(
             DatasetMetadata("raw", "mydomain", "mydataset", 32), Query()
         )
 
+    @patch.object(JobService, "succeed_query")
+    @patch.object(JobService, "create_query_job")
+    @patch("api.controller.datasets.get_subject_id")
     @patch.object(DataService, "query_data")
     def test_calls_service_with_sql_query_when_empty_json_values_provided(
-        self, mock_query_method
+        self, mock_query_method, mock_get_subject_id, mock_create_query_job, mock_succeed_query
     ):
         request_json = {
             "select_columns": ["column1"],
@@ -891,8 +906,13 @@ def test_calls_service_with_sql_query_when_empty_json_values_provided(
             ),
         )
 
+    @patch.object(JobService, "succeed_query")
+    @patch.object(JobService, "create_query_job")
+    @patch("api.controller.datasets.get_subject_id")
     @patch.object(DataService, "query_data")
-    def test_returns_formatted_json_from_query_result(self, mock_query_method):
+    def test_returns_formatted_json_from_query_result(
+        self, mock_query_method, mock_get_subject_id, mock_create_query_job, mock_succeed_query
+    ):
         mock_query_method.return_value = pd.DataFrame(
             {
                 "column1": [1, 2],
@@ -918,8 +938,13 @@ def test_returns_formatted_json_from_query_result(self, mock_query_method):
             "1": {"column1": "2", "column2": "item2", "area": "area_2"},
         }
 
+    @patch.object(JobService, "succeed_query")
+    @patch.object(JobService, "create_query_job")
+    @patch("api.controller.datasets.get_subject_id")
     @patch.object(DataService, "query_data")
-    def test_request_query_in_csv_is_successful(self, mock_query_method):
+    def test_request_query_in_csv_is_successful(
+        self, mock_query_method, mock_get_subject_id, mock_create_query_job, mock_succeed_query
+    ):
         mock_query_method.return_value = pd.DataFrame(
             {
                 "column1": [1, 2],
@@ -937,9 +962,12 @@ def test_request_query_in_csv_is_successful(self, mock_query_method):
 
         assert response.status_code == 200
 
+    @patch.object(JobService, "succeed_query")
+    @patch.object(JobService, "create_query_job")
+    @patch("api.controller.datasets.get_subject_id")
     @patch.object(DataService, "query_data")
     def test_returns_formatted_json_from_query_if_format_is_not_provided(
-        self, mock_query_method
+        self, mock_query_method, mock_get_subject_id, mock_create_query_job, mock_succeed_query
     ):
         mock_query_method.return_value = pd.DataFrame(
             {
@@ -961,8 +989,13 @@ def test_returns_formatted_json_from_query_if_format_is_not_provided(
             "1": {"column1": "2", "column2": "item2", "area": "area_2"},
         }
 
+    @patch.object(JobService, "succeed_query")
+    @patch.object(JobService, "create_query_job")
+    @patch("api.controller.datasets.get_subject_id")
     @patch.object(DataService, "query_data")
-    def test_returns_204_if_dataframe_is_empty(self, mock_query_method):
+    def test_returns_204_if_dataframe_is_empty(
+        self, mock_query_method, mock_get_subject_id, mock_create_query_job, mock_succeed_query
+    ):
         mock_query_method.return_value = pd.DataFrame(
             {
                 "column1": [],
@@ -983,9 +1016,12 @@ def test_returns_204_if_dataframe_is_empty(self, mock_query_method):
             == "No rows were returned. Either there is no data or the query is too limiting."
         )
 
+    @patch.object(JobService, "succeed_query")
+    @patch.object(JobService, "create_query_job")
+    @patch("api.controller.datasets.get_subject_id")
     @patch.object(DataService, "query_data")
     def test_returns_error_from_query_request_when_format_is_unsupported(
-        self, mock_query_method
+        self, mock_query_method, mock_get_subject_id, mock_create_query_job, mock_succeed_query
     ):
         mock_query_method.return_value = pd.DataFrame(
             {
@@ -1007,6 +1043,25 @@ def test_returns_error_from_query_request_when_format_is_unsupported(
             "details": "Provided value for Accept header parameter [text/plain] is not supported. Supported formats: application/json, text/csv, application/octet-stream"
         }
 
+    @patch.object(JobService, "succeed_query")
+    @patch.object(JobService, "create_query_job")
+    @patch("api.controller.datasets.get_subject_id")
+    @patch.object(DataService, "query_data")
+    def test_creates_audit_job_on_successful_query(
+        self, mock_query_method, mock_get_subject_id, mock_create_query_job, mock_succeed_query
+    ):
+        mock_query_method.return_value = pd.DataFrame({"col": [1]})
+        mock_get_subject_id.return_value = "subject_id"
+        fake_job = mock_create_query_job.return_value
+
+        query_url = f"{BASE_API_PATH}/datasets/raw/mydomain/mydataset/query?version=1"
+        self.client.post(query_url, headers={"Authorization": "Bearer test-token"})
+
+        mock_create_query_job.assert_called_once_with(
+            "subject_id", DatasetMetadata("raw", "mydomain", "mydataset", 1)
+        )
+        mock_succeed_query.assert_called_once_with(fake_job, url=None)
+
     @pytest.mark.parametrize(
         "input_key", ["select_column", "invalid_key", "another_invalid_key"]
     )