From 53e750c7afb256159d4d120dd33052f62154ca77 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Mon, 15 Jun 2026 14:12:37 +0200 Subject: [PATCH 01/14] Add integrated analyze context router --- .../api/routers/integrated_context_router.py | 208 ++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 app/backend/api/routers/integrated_context_router.py diff --git a/app/backend/api/routers/integrated_context_router.py b/app/backend/api/routers/integrated_context_router.py new file mode 100644 index 00000000..e8f51a93 --- /dev/null +++ b/app/backend/api/routers/integrated_context_router.py @@ -0,0 +1,208 @@ +import logging +from collections import deque +from typing import Any, Callable, Dict, Iterable, List, Optional, Set + +from fastapi import APIRouter, Depends, HTTPException, status +from fastapi.responses import JSONResponse + +from app.backend.api.dependencies import getCurrentUser +from app.backend.api.services.project_service import ProjectService +from app.backend.database import getMapper +from app.backend.mapper.postgresql import PostgresqlFlatMapper + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/projects", tags=["projects"]) + + +def getProjectService() -> ProjectService: + return ProjectService() + + +def _normalizedKind(value: Optional[str]) -> str: + return "".join(str(value or "").split()).lower() + + +def _isVolumeKind(value: Optional[str]) -> bool: + kind = _normalizedKind(value) + return kind in {"volume", "volumemask", "setofvolumes", "setoftomograms"} + + +def _isCoords3dKind(value: Optional[str]) -> bool: + return "setofcoordinates3d" in _normalizedKind(value) + + +def _isTiltSeriesKind(value: Optional[str]) -> bool: + kind = _normalizedKind(value) + return "setoftiltseries" in kind and kind != "setoftiltseriesm" + + +def _isCTFTomoKind(value: Optional[str]) -> bool: + return "setofctftomoseries" in _normalizedKind(value) + + +def _safeStr(value: Any, default: str = "") -> str: + if value is None: + return default + try: + return str(value) + except Exception: + return default + + +def _toInt(value: Any) -> Any: + try: + return int(value) + except Exception: + return value + + +def _nodeOutputs(node: Dict[str, Any]) -> List[Dict[str, Any]]: + outputs = node.get("outputs") or [] + return [item for item in outputs if isinstance(item, dict)] + + +def _outputClass(output: Dict[str, Any]) -> str: + return _safeStr( + output.get("pointerClass") + or output.get("outputClassName") + or output.get("className") + or output.get("type") + ) + + +def _outputName(output: Dict[str, Any]) -> str: + return _safeStr(output.get("name") or output.get("outputName")) + + +def _findOutput(node: Dict[str, Any], outputName: str) -> Optional[Dict[str, Any]]: + wanted = _safeStr(outputName).strip() + for output in _nodeOutputs(node): + if _outputName(output) == wanted: + return output + return None + + +def _ancestorProtocolIds(protocols: Dict[str, Any], protocolId: int) -> List[str]: + ordered: List[str] = [] + seen: Set[str] = set() + queue = deque([str(protocolId)]) + + while queue: + currentId = queue.popleft() + if currentId in seen: + continue + + seen.add(currentId) + ordered.append(currentId) + + node = protocols.get(currentId) or {} + for parentId in node.get("parents") or []: + parentText = _safeStr(parentId).strip() + if parentText and parentText != "PROJECT" and parentText not in seen: + queue.append(parentText) + + return ordered + + +def _buildLink( + protocols: Dict[str, Any], + protocolIds: Iterable[str], + matcher: Callable[[str], bool], +) -> Optional[Dict[str, Any]]: + for nodeId in protocolIds: + node = protocols.get(str(nodeId)) or {} + for output in _nodeOutputs(node): + outputClass = _outputClass(output) + outputName = _outputName(output) + if not outputName or not matcher(outputClass): + continue + + return { + "protocolId": _toInt(nodeId), + "outputName": outputName, + "label": output.get("info") or outputName, + "status": "inferred", + } + + return None + + +def _missingLink() -> Dict[str, Any]: + return {"status": "missing"} + + +def _buildSummary(link: Optional[Dict[str, Any]], protocols: Dict[str, Any]) -> Optional[Dict[str, Any]]: + if not link or link.get("status") == "missing": + return None + + node = protocols.get(str(link.get("protocolId"))) or {} + return { + "protocolId": link.get("protocolId"), + "outputName": link.get("outputName"), + "label": node.get("label") or node.get("runName"), + "status": node.get("status"), + } + + +@router.get( + "/{projectId}/protocols/{protocolId}/outputs/{outputName}/integrated-context", + response_model=Any, + status_code=status.HTTP_200_OK, +) +def getIntegratedAnalyzeContext( + projectId: int, + protocolId: int, + outputName: str, + currentUser=Depends(getCurrentUser), + mapper: PostgresqlFlatMapper = Depends(getMapper), + service: ProjectService = Depends(getProjectService), +): + project = service.getProjectById( + mapper, + projectId, + currentUser, + refresh=False, + checkPid=False, + ) + if not project: + raise HTTPException(status_code=404, detail="Project not found") + + protocols = project.get("protocols") or {} + if not isinstance(protocols, dict): + protocols = {} + + node = protocols.get(str(protocolId)) + if not isinstance(node, dict): + raise HTTPException(status_code=404, detail="Protocol not found") + + rootOutput = _findOutput(node, outputName) or {} + rootClass = _outputClass(rootOutput) + protocolIds = _ancestorProtocolIds(protocols, protocolId) + + links = { + "tiltSeries": _buildLink(protocols, protocolIds, _isTiltSeriesKind) or _missingLink(), + "ctf": _buildLink(protocols, protocolIds, _isCTFTomoKind) or _missingLink(), + "tomogram": _buildLink(protocols, protocolIds, _isVolumeKind) or _missingLink(), + "coordinates3d": _buildLink(protocols, protocolIds, _isCoords3dKind) or _missingLink(), + } + + payload = { + "root": { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "outputClass": rootClass or None, + }, + "links": links, + "summaries": { + key: _buildSummary(link, protocols) + for key, link in links.items() + }, + } + + response = JSONResponse(payload) + response.headers["X-Debug-Auth"] = "ok" + response.headers["X-Debug-UserId"] = _safeStr(getattr(currentUser, "id", currentUser.get("id", ""))) + response.headers["Vary"] = "Authorization" + return response From ccb9ef9e6f19a6f1c23bf0928803a340bcf5123c Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Mon, 15 Jun 2026 14:13:20 +0200 Subject: [PATCH 02/14] Register integrated context router --- app/backend/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/backend/main.py b/app/backend/main.py index 7c62184e..f9ef4adb 100644 --- a/app/backend/main.py +++ b/app/backend/main.py @@ -45,6 +45,7 @@ from app.backend.api.routers.user_router import router as users from app.backend.api.routers.settings_router import router as settingsRouter from app.backend.api.routers.coords2d_router import router as coords2dRouter +from app.backend.api.routers.integrated_context_router import router as integratedContextRouter from app.backend.api.routers.system_router import router as systemRouter from app.backend.utils.error_handlers import registerAllErrorHandlers from starlette.staticfiles import StaticFiles @@ -136,6 +137,7 @@ def _buildApiApp() -> FastAPI: apiApp.include_router(users) apiApp.include_router(settingsRouter) apiApp.include_router(coords2dRouter) + apiApp.include_router(integratedContextRouter) apiApp.include_router(systemRouter) @apiApp.get("/health") @@ -214,4 +216,4 @@ def openapi_redirect(): if serveWeb and webDistPath and (webDistPath / "index.html").exists(): # mountSpaStaticRootLast - app.mount("/", SpaStaticFiles(directory=str(webDistPath), html=True), name="web") + app.mount("/", SpaStaticFiles(directory=str(webDistPath), html=True), name="web") \ No newline at end of file From f1c4967aafd408d01c2efd37b7ce2c86299c1e20 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Mon, 15 Jun 2026 14:20:19 +0200 Subject: [PATCH 03/14] Remove integrated context router shell --- .../api/routers/integrated_context_router.py | 208 ------------------ 1 file changed, 208 deletions(-) delete mode 100644 app/backend/api/routers/integrated_context_router.py diff --git a/app/backend/api/routers/integrated_context_router.py b/app/backend/api/routers/integrated_context_router.py deleted file mode 100644 index e8f51a93..00000000 --- a/app/backend/api/routers/integrated_context_router.py +++ /dev/null @@ -1,208 +0,0 @@ -import logging -from collections import deque -from typing import Any, Callable, Dict, Iterable, List, Optional, Set - -from fastapi import APIRouter, Depends, HTTPException, status -from fastapi.responses import JSONResponse - -from app.backend.api.dependencies import getCurrentUser -from app.backend.api.services.project_service import ProjectService -from app.backend.database import getMapper -from app.backend.mapper.postgresql import PostgresqlFlatMapper - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/projects", tags=["projects"]) - - -def getProjectService() -> ProjectService: - return ProjectService() - - -def _normalizedKind(value: Optional[str]) -> str: - return "".join(str(value or "").split()).lower() - - -def _isVolumeKind(value: Optional[str]) -> bool: - kind = _normalizedKind(value) - return kind in {"volume", "volumemask", "setofvolumes", "setoftomograms"} - - -def _isCoords3dKind(value: Optional[str]) -> bool: - return "setofcoordinates3d" in _normalizedKind(value) - - -def _isTiltSeriesKind(value: Optional[str]) -> bool: - kind = _normalizedKind(value) - return "setoftiltseries" in kind and kind != "setoftiltseriesm" - - -def _isCTFTomoKind(value: Optional[str]) -> bool: - return "setofctftomoseries" in _normalizedKind(value) - - -def _safeStr(value: Any, default: str = "") -> str: - if value is None: - return default - try: - return str(value) - except Exception: - return default - - -def _toInt(value: Any) -> Any: - try: - return int(value) - except Exception: - return value - - -def _nodeOutputs(node: Dict[str, Any]) -> List[Dict[str, Any]]: - outputs = node.get("outputs") or [] - return [item for item in outputs if isinstance(item, dict)] - - -def _outputClass(output: Dict[str, Any]) -> str: - return _safeStr( - output.get("pointerClass") - or output.get("outputClassName") - or output.get("className") - or output.get("type") - ) - - -def _outputName(output: Dict[str, Any]) -> str: - return _safeStr(output.get("name") or output.get("outputName")) - - -def _findOutput(node: Dict[str, Any], outputName: str) -> Optional[Dict[str, Any]]: - wanted = _safeStr(outputName).strip() - for output in _nodeOutputs(node): - if _outputName(output) == wanted: - return output - return None - - -def _ancestorProtocolIds(protocols: Dict[str, Any], protocolId: int) -> List[str]: - ordered: List[str] = [] - seen: Set[str] = set() - queue = deque([str(protocolId)]) - - while queue: - currentId = queue.popleft() - if currentId in seen: - continue - - seen.add(currentId) - ordered.append(currentId) - - node = protocols.get(currentId) or {} - for parentId in node.get("parents") or []: - parentText = _safeStr(parentId).strip() - if parentText and parentText != "PROJECT" and parentText not in seen: - queue.append(parentText) - - return ordered - - -def _buildLink( - protocols: Dict[str, Any], - protocolIds: Iterable[str], - matcher: Callable[[str], bool], -) -> Optional[Dict[str, Any]]: - for nodeId in protocolIds: - node = protocols.get(str(nodeId)) or {} - for output in _nodeOutputs(node): - outputClass = _outputClass(output) - outputName = _outputName(output) - if not outputName or not matcher(outputClass): - continue - - return { - "protocolId": _toInt(nodeId), - "outputName": outputName, - "label": output.get("info") or outputName, - "status": "inferred", - } - - return None - - -def _missingLink() -> Dict[str, Any]: - return {"status": "missing"} - - -def _buildSummary(link: Optional[Dict[str, Any]], protocols: Dict[str, Any]) -> Optional[Dict[str, Any]]: - if not link or link.get("status") == "missing": - return None - - node = protocols.get(str(link.get("protocolId"))) or {} - return { - "protocolId": link.get("protocolId"), - "outputName": link.get("outputName"), - "label": node.get("label") or node.get("runName"), - "status": node.get("status"), - } - - -@router.get( - "/{projectId}/protocols/{protocolId}/outputs/{outputName}/integrated-context", - response_model=Any, - status_code=status.HTTP_200_OK, -) -def getIntegratedAnalyzeContext( - projectId: int, - protocolId: int, - outputName: str, - currentUser=Depends(getCurrentUser), - mapper: PostgresqlFlatMapper = Depends(getMapper), - service: ProjectService = Depends(getProjectService), -): - project = service.getProjectById( - mapper, - projectId, - currentUser, - refresh=False, - checkPid=False, - ) - if not project: - raise HTTPException(status_code=404, detail="Project not found") - - protocols = project.get("protocols") or {} - if not isinstance(protocols, dict): - protocols = {} - - node = protocols.get(str(protocolId)) - if not isinstance(node, dict): - raise HTTPException(status_code=404, detail="Protocol not found") - - rootOutput = _findOutput(node, outputName) or {} - rootClass = _outputClass(rootOutput) - protocolIds = _ancestorProtocolIds(protocols, protocolId) - - links = { - "tiltSeries": _buildLink(protocols, protocolIds, _isTiltSeriesKind) or _missingLink(), - "ctf": _buildLink(protocols, protocolIds, _isCTFTomoKind) or _missingLink(), - "tomogram": _buildLink(protocols, protocolIds, _isVolumeKind) or _missingLink(), - "coordinates3d": _buildLink(protocols, protocolIds, _isCoords3dKind) or _missingLink(), - } - - payload = { - "root": { - "projectId": projectId, - "protocolId": protocolId, - "outputName": outputName, - "outputClass": rootClass or None, - }, - "links": links, - "summaries": { - key: _buildSummary(link, protocols) - for key, link in links.items() - }, - } - - response = JSONResponse(payload) - response.headers["X-Debug-Auth"] = "ok" - response.headers["X-Debug-UserId"] = _safeStr(getattr(currentUser, "id", currentUser.get("id", ""))) - response.headers["Vary"] = "Authorization" - return response From 2f802c67db0db08092b7e8aba8d9a9da338c1433 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Mon, 15 Jun 2026 14:20:58 +0200 Subject: [PATCH 04/14] Unregister integrated context router --- app/backend/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/backend/main.py b/app/backend/main.py index f9ef4adb..7ad0aa20 100644 --- a/app/backend/main.py +++ b/app/backend/main.py @@ -45,7 +45,6 @@ from app.backend.api.routers.user_router import router as users from app.backend.api.routers.settings_router import router as settingsRouter from app.backend.api.routers.coords2d_router import router as coords2dRouter -from app.backend.api.routers.integrated_context_router import router as integratedContextRouter from app.backend.api.routers.system_router import router as systemRouter from app.backend.utils.error_handlers import registerAllErrorHandlers from starlette.staticfiles import StaticFiles @@ -137,7 +136,6 @@ def _buildApiApp() -> FastAPI: apiApp.include_router(users) apiApp.include_router(settingsRouter) apiApp.include_router(coords2dRouter) - apiApp.include_router(integratedContextRouter) apiApp.include_router(systemRouter) @apiApp.get("/health") From a6d04ee7968892b52900f764c3344c3dda690e3d Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 15 Jun 2026 15:18:56 +0200 Subject: [PATCH 05/14] Add integrated tomography context service --- app/backend/api/routers/project_router.py | 29 +++ app/backend/api/services/project_service.py | 232 ++++++++++++++++++++ 2 files changed, 261 insertions(+) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index ceb20da2..12030e2b 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2348,6 +2348,35 @@ def createCoords3dOutputFromPoints(projectId: int, raise HTTPException( status_code=500, detail=f"Failed to create coords3d output from points: {e}", ) +@router.get( + "/{projectId}/protocols/{protocolId}/outputs/{outputName}/integrated-context", + response_model=Any, + status_code=status.HTTP_200_OK, +) +def getIntegratedAnalyzeContext( + projectId: int, + protocolId: int, + outputName: str, + currentUser=Depends(getCurrentUser), + mapper: PostgresqlFlatMapper = Depends(getMapper), + service: ProjectService = Depends(getProjectService), +): + project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + if not project: + raise HTTPException(status_code=404, detail="Project not found") + + payload = service.getIntegratedAnalyzeContextService( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + resp = JSONResponse(payload) + resp.headers["X-Debug-Auth"] = "ok" + resp.headers["X-Debug-UserId"] = str(getattr(currentUser, "id", currentUser.get("id", ""))) + resp.headers["Vary"] = "Authorization" + return resp + # ============================================================================== # ANALYZE RESULTS: FSC (SetOfFSCs) # ============================================================================== diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 00f5837b..535d5c7a 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2115,6 +2115,238 @@ def buildProtocolOutputThumbnailUrl(projectId: int, protocolId: int, outputName: # buildProtocolOutputThumbnailUrl return f"/projects/{projectId}/protocols/{protocolId}/outputs/{outputName}/thumbnail" + def getIntegratedAnalyzeContextService( + self, + projectId: int, + protocolId: int, + outputName: str, + ) -> Dict[str, Any]: + if self.currentProject is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="No current project loaded", + ) + + try: + protocol = self.currentProject.getProtocol(int(protocolId)) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol {protocolId} not found: {e}", + ) + + outputObj = getattr(protocol, outputName, None) + if outputObj is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Output '{outputName}' not found in protocol {protocolId}", + ) + + def className(obj: Any) -> str: + try: + return obj.getClassName() + except Exception: + return obj.__class__.__name__ if obj is not None else "" + + def normalizedClassName(obj: Any) -> str: + return className(obj).replace(" ", "").lower() + + def safeCall(obj: Any, methodName: str, default: Any = None) -> Any: + try: + method = getattr(obj, methodName, None) + if method is None: + return default + return method() + except Exception: + return default + + def safeList(value: Any) -> List[Any]: + if value is None: + return [] + if isinstance(value, (list, tuple, set)): + return list(value) + return [value] + + def getTsIds(obj: Any) -> Set[str]: + values = safeCall(obj, "getTSIds", []) + return {str(v) for v in safeList(values) if v is not None and str(v)} + + def getObjId(obj: Any) -> Optional[Any]: + return safeCall(obj, "getObjId", None) + + def isTiltSeriesSet(obj: Any) -> bool: + name = normalizedClassName(obj) + return "setoftiltseries" in name and "setoftiltseriesm" not in name + + def isTomogramSet(obj: Any) -> bool: + return "setoftomograms" in normalizedClassName(obj) + + def isCoordinates3dSet(obj: Any) -> bool: + return "setofcoordinates3d" in normalizedClassName(obj) + + def isCtfTomoSeriesSet(obj: Any) -> bool: + return "setofctftomoseries" in normalizedClassName(obj) + + def buildLink( + obj: Any, + source: Optional[Dict[str, Any]] = None, + statusValue: str = "available", + label: Optional[str] = None, + ) -> Dict[str, Any]: + source = source or {} + return { + "protocolId": source.get("protocolId"), + "outputName": source.get("outputName"), + "itemId": getObjId(obj), + "label": label or source.get("label") or className(obj), + "status": statusValue, + } + + def buildSummary(obj: Any, tsIds: Optional[Set[str]] = None) -> Dict[str, Any]: + summary = { + "objectClass": className(obj), + "objectId": getObjId(obj), + "size": safeCall(obj, "getSize", None), + "tsIds": sorted(tsIds if tsIds is not None else getTsIds(obj)), + "samplingRate": safeCall(obj, "getSamplingRate", None), + "dimensions": safeCall(obj, "getDimensions", safeCall(obj, "getDim", None)), + "fileName": safeCall(obj, "getFileName", None), + } + + boxSize = safeCall(obj, "getBoxSize", None) + if boxSize is not None: + summary["boxSize"] = boxSize + + ctfCorrected = safeCall(obj, "ctfCorrected", None) + if ctfCorrected is not None: + summary["ctfCorrected"] = ctfCorrected + + return self._safeScipionValue(summary) + + inputRefs: List[Dict[str, Any]] = [] + + for inputName, pointer in protocol.iterInputAttributes(): + try: + inputObj = pointer.get() if pointer else None + except Exception: + inputObj = None + + if inputObj is None: + continue + + try: + inputProtocolId = pointer.getObjValue().getObjId() + except Exception: + inputProtocolId = None + + try: + inputOutputName = pointer.getExtended() + except Exception: + inputOutputName = None + + inputRefs.append({ + "name": inputName, + "object": inputObj, + "protocolId": inputProtocolId, + "outputName": inputOutputName, + "label": inputName, + }) + + def findInputRef(predicate, tsIds: Optional[Set[str]] = None) -> Optional[Dict[str, Any]]: + for ref in inputRefs: + obj = ref["object"] + if not predicate(obj): + continue + + if tsIds: + candidateTsIds = getTsIds(obj) + if candidateTsIds and not candidateTsIds.intersection(tsIds): + continue + + return ref + + return None + + links = { + "tiltSeries": None, + "ctf": None, + "tomogram": None, + "coordinates3d": None, + } + summaries = { + "tiltSeries": None, + "ctf": None, + "tomogram": None, + "coordinates3d": None, + } + + rootSource = { + "protocolId": protocolId, + "outputName": outputName, + "label": outputName, + } + + outputTsIds = getTsIds(outputObj) + + if isCoordinates3dSet(outputObj): + links["coordinates3d"] = buildLink(outputObj, rootSource) + summaries["coordinates3d"] = buildSummary(outputObj, outputTsIds) + + tomograms = safeCall(outputObj, "getPrecedents", None) + if tomograms is not None: + tomoTsIds = outputTsIds or getTsIds(tomograms) + links["tomogram"] = buildLink(tomograms, statusValue="inferred") + summaries["tomogram"] = buildSummary(tomograms, tomoTsIds) + outputTsIds = tomoTsIds + + elif isTomogramSet(outputObj): + links["tomogram"] = buildLink(outputObj, rootSource) + summaries["tomogram"] = buildSummary(outputObj, outputTsIds) + + elif isCtfTomoSeriesSet(outputObj): + links["ctf"] = buildLink(outputObj, rootSource) + summaries["ctf"] = buildSummary(outputObj, outputTsIds) + + tiltSeries = safeCall(outputObj, "getSetOfTiltSeries", None) + if tiltSeries is not None: + links["tiltSeries"] = buildLink(tiltSeries, statusValue="inferred") + summaries["tiltSeries"] = buildSummary(tiltSeries, outputTsIds) + + elif isTiltSeriesSet(outputObj): + links["tiltSeries"] = buildLink(outputObj, rootSource) + summaries["tiltSeries"] = buildSummary(outputObj, outputTsIds) + + if outputTsIds and links["ctf"] is None: + ctfRef = findInputRef(isCtfTomoSeriesSet, outputTsIds) + if ctfRef is not None: + ctfSet = ctfRef["object"] + links["ctf"] = buildLink(ctfSet, ctfRef) + summaries["ctf"] = buildSummary(ctfSet, outputTsIds) + + tiltSeries = safeCall(ctfSet, "getSetOfTiltSeries", None) + if tiltSeries is not None and links["tiltSeries"] is None: + links["tiltSeries"] = buildLink(tiltSeries, statusValue="inferred") + summaries["tiltSeries"] = buildSummary(tiltSeries, outputTsIds) + + if outputTsIds and links["tiltSeries"] is None: + tiltRef = findInputRef(isTiltSeriesSet, outputTsIds) + if tiltRef is not None: + tiltSeries = tiltRef["object"] + links["tiltSeries"] = buildLink(tiltSeries, tiltRef) + summaries["tiltSeries"] = buildSummary(tiltSeries, outputTsIds) + + return { + "root": { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "outputClass": className(outputObj), + }, + "links": links, + "summaries": summaries, + } + + def buildProtocolOutputThumbnail( self, protocolId: int, From eac35071e72efd5257dbe7f78fbec2401f3a833c Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 15 Jun 2026 15:34:15 +0200 Subject: [PATCH 06/14] Resolve integrated tomography links from Scipion inputs --- app/backend/api/services/project_service.py | 106 +++++++++++++++----- 1 file changed, 80 insertions(+), 26 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 535d5c7a..22436b65 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2223,34 +2223,50 @@ def buildSummary(obj: Any, tsIds: Optional[Set[str]] = None) -> Dict[str, Any]: return self._safeScipionValue(summary) - inputRefs: List[Dict[str, Any]] = [] + def getProtocolInputRefs(protocolObj: Any) -> List[Dict[str, Any]]: + refs: List[Dict[str, Any]] = [] - for inputName, pointer in protocol.iterInputAttributes(): - try: - inputObj = pointer.get() if pointer else None - except Exception: - inputObj = None + for inputName, pointer in protocolObj.iterInputAttributes(): + try: + inputObj = pointer.get() if pointer else None + except Exception: + inputObj = None - if inputObj is None: - continue + if inputObj is None: + continue - try: - inputProtocolId = pointer.getObjValue().getObjId() - except Exception: - inputProtocolId = None + try: + inputProtocolId = pointer.getObjValue().getObjId() + except Exception: + inputProtocolId = None + + try: + inputOutputName = pointer.getExtended() + except Exception: + inputOutputName = None + + refs.append({ + "name": inputName, + "object": inputObj, + "protocolId": inputProtocolId, + "outputName": inputOutputName, + "label": inputName, + }) + + return refs + + def getProtocolInputRefsById(sourceProtocolId: Any) -> List[Dict[str, Any]]: + if sourceProtocolId is None: + return [] try: - inputOutputName = pointer.getExtended() + sourceProtocol = self.currentProject.getProtocol(int(sourceProtocolId)) except Exception: - inputOutputName = None - - inputRefs.append({ - "name": inputName, - "object": inputObj, - "protocolId": inputProtocolId, - "outputName": inputOutputName, - "label": inputName, - }) + return [] + + return getProtocolInputRefs(sourceProtocol) + + inputRefs = getProtocolInputRefs(protocol) def findInputRef(predicate, tsIds: Optional[Set[str]] = None) -> Optional[Dict[str, Any]]: for ref in inputRefs: @@ -2267,6 +2283,38 @@ def findInputRef(predicate, tsIds: Optional[Set[str]] = None) -> Optional[Dict[s return None + def findInputRefForObject(targetObj: Any, refs: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + if targetObj is None: + return None + + targetClass = className(targetObj) + targetObjId = getObjId(targetObj) + targetFileName = safeCall(targetObj, "getFileName", None) + targetTsIds = getTsIds(targetObj) + + for ref in refs: + obj = ref["object"] + + if obj is targetObj: + return ref + + if className(obj) != targetClass: + continue + + objId = getObjId(obj) + if targetObjId is not None and objId is not None and str(objId) == str(targetObjId): + return ref + + fileName = safeCall(obj, "getFileName", None) + if targetFileName and fileName and str(fileName) == str(targetFileName): + return ref + + objTsIds = getTsIds(obj) + if targetTsIds and objTsIds and targetTsIds == objTsIds: + return ref + + return None + links = { "tiltSeries": None, "ctf": None, @@ -2295,7 +2343,8 @@ def findInputRef(predicate, tsIds: Optional[Set[str]] = None) -> Optional[Dict[s tomograms = safeCall(outputObj, "getPrecedents", None) if tomograms is not None: tomoTsIds = outputTsIds or getTsIds(tomograms) - links["tomogram"] = buildLink(tomograms, statusValue="inferred") + tomogramRef = findInputRefForObject(tomograms, inputRefs) + links["tomogram"] = buildLink(tomograms, tomogramRef, statusValue="inferred") summaries["tomogram"] = buildSummary(tomograms, tomoTsIds) outputTsIds = tomoTsIds @@ -2309,7 +2358,8 @@ def findInputRef(predicate, tsIds: Optional[Set[str]] = None) -> Optional[Dict[s tiltSeries = safeCall(outputObj, "getSetOfTiltSeries", None) if tiltSeries is not None: - links["tiltSeries"] = buildLink(tiltSeries, statusValue="inferred") + tiltRef = findInputRefForObject(tiltSeries, inputRefs) + links["tiltSeries"] = buildLink(tiltSeries, tiltRef, statusValue="inferred") summaries["tiltSeries"] = buildSummary(tiltSeries, outputTsIds) elif isTiltSeriesSet(outputObj): @@ -2325,8 +2375,12 @@ def findInputRef(predicate, tsIds: Optional[Set[str]] = None) -> Optional[Dict[s tiltSeries = safeCall(ctfSet, "getSetOfTiltSeries", None) if tiltSeries is not None and links["tiltSeries"] is None: - links["tiltSeries"] = buildLink(tiltSeries, statusValue="inferred") - summaries["tiltSeries"] = buildSummary(tiltSeries, outputTsIds) + ctfInputRefs = getProtocolInputRefsById(ctfRef.get("protocolId")) + tiltRef = ( + findInputRefForObject(tiltSeries, ctfInputRefs) + or findInputRefForObject(tiltSeries, inputRefs) + ) + links["tiltSeries"] = buildLink(tiltSeries, tiltRef, statusValue="inferred") if outputTsIds and links["tiltSeries"] is None: tiltRef = findInputRef(isTiltSeriesSet, outputTsIds) From 4ba902c405fe94e13eb2e8f8dc403e24fdc1ee70 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 15 Jun 2026 16:08:42 +0200 Subject: [PATCH 07/14] Resolve integrated tilt series links --- app/backend/api/services/project_service.py | 46 +++++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 22436b65..cdc399c8 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2266,7 +2266,26 @@ def getProtocolInputRefsById(sourceProtocolId: Any) -> List[Dict[str, Any]]: return getProtocolInputRefs(sourceProtocol) + def getProtocolOutputRefs(protocolObj: Any) -> List[Dict[str, Any]]: + refs: List[Dict[str, Any]] = [] + + for outputAttrName, outputObjRef in protocolObj.iterOutputAttributes(): + if outputObjRef is None: + continue + + refs.append({ + "name": outputAttrName, + "object": outputObjRef, + "protocolId": safeCall(protocolObj, "getObjId", None), + "outputName": outputAttrName, + "label": outputAttrName, + }) + + return refs + inputRefs = getProtocolInputRefs(protocol) + outputRefs = getProtocolOutputRefs(protocol) + localRefs = inputRefs + outputRefs def findInputRef(predicate, tsIds: Optional[Set[str]] = None) -> Optional[Dict[str, Any]]: for ref in inputRefs: @@ -2283,7 +2302,11 @@ def findInputRef(predicate, tsIds: Optional[Set[str]] = None) -> Optional[Dict[s return None - def findInputRefForObject(targetObj: Any, refs: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + def findInputRefForObject( + targetObj: Any, + refs: List[Dict[str, Any]], + predicate=None, + ) -> Optional[Dict[str, Any]]: if targetObj is None: return None @@ -2295,12 +2318,12 @@ def findInputRefForObject(targetObj: Any, refs: List[Dict[str, Any]]) -> Optiona for ref in refs: obj = ref["object"] + if predicate is not None and not predicate(obj): + continue + if obj is targetObj: return ref - if className(obj) != targetClass: - continue - objId = getObjId(obj) if targetObjId is not None and objId is not None and str(objId) == str(targetObjId): return ref @@ -2313,6 +2336,9 @@ def findInputRefForObject(targetObj: Any, refs: List[Dict[str, Any]]) -> Optiona if targetTsIds and objTsIds and targetTsIds == objTsIds: return ref + if className(obj) != targetClass: + continue + return None links = { @@ -2352,13 +2378,14 @@ def findInputRefForObject(targetObj: Any, refs: List[Dict[str, Any]]) -> Optiona links["tomogram"] = buildLink(outputObj, rootSource) summaries["tomogram"] = buildSummary(outputObj, outputTsIds) + elif isCtfTomoSeriesSet(outputObj): links["ctf"] = buildLink(outputObj, rootSource) summaries["ctf"] = buildSummary(outputObj, outputTsIds) tiltSeries = safeCall(outputObj, "getSetOfTiltSeries", None) - if tiltSeries is not None: - tiltRef = findInputRefForObject(tiltSeries, inputRefs) + if tiltSeries is not None and isTiltSeriesSet(tiltSeries): + tiltRef = findInputRefForObject(tiltSeries, localRefs, isTiltSeriesSet) links["tiltSeries"] = buildLink(tiltSeries, tiltRef, statusValue="inferred") summaries["tiltSeries"] = buildSummary(tiltSeries, outputTsIds) @@ -2374,13 +2401,14 @@ def findInputRefForObject(targetObj: Any, refs: List[Dict[str, Any]]) -> Optiona summaries["ctf"] = buildSummary(ctfSet, outputTsIds) tiltSeries = safeCall(ctfSet, "getSetOfTiltSeries", None) - if tiltSeries is not None and links["tiltSeries"] is None: + if tiltSeries is not None and links["tiltSeries"] is None and isTiltSeriesSet(tiltSeries): ctfInputRefs = getProtocolInputRefsById(ctfRef.get("protocolId")) tiltRef = ( - findInputRefForObject(tiltSeries, ctfInputRefs) - or findInputRefForObject(tiltSeries, inputRefs) + findInputRefForObject(tiltSeries, ctfInputRefs, isTiltSeriesSet) + or findInputRefForObject(tiltSeries, inputRefs, isTiltSeriesSet) ) links["tiltSeries"] = buildLink(tiltSeries, tiltRef, statusValue="inferred") + summaries["tiltSeries"] = buildSummary(tiltSeries, outputTsIds) if outputTsIds and links["tiltSeries"] is None: tiltRef = findInputRef(isTiltSeriesSet, outputTsIds) From eec2b0e5260ca773e57c7d82946396a2618c7bab Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 16 Jun 2026 00:01:37 +0200 Subject: [PATCH 08/14] Add integrated tomography item relations --- app/backend/api/services/project_service.py | 134 +++++++++++++++++++- 1 file changed, 132 insertions(+), 2 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index cdc399c8..19e5739e 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2167,6 +2167,15 @@ def safeList(value: Any) -> List[Any]: return list(value) return [value] + def firstNonEmpty(*values: Any) -> Optional[Any]: + for value in values: + if value is None: + continue + text = str(value) + if text: + return value + return None + def getTsIds(obj: Any) -> Set[str]: values = safeCall(obj, "getTSIds", []) return {str(v) for v in safeList(values) if v is not None and str(v)} @@ -2174,6 +2183,37 @@ def getTsIds(obj: Any) -> Set[str]: def getObjId(obj: Any) -> Optional[Any]: return safeCall(obj, "getObjId", None) + def iterItems(obj: Any) -> List[Any]: + if obj is None: + return [] + + try: + return list(obj.iterItems()) + except Exception: + pass + + try: + return list(obj) + except Exception: + return [] + + def getItemTsId(item: Any) -> Optional[Any]: + return firstNonEmpty( + safeCall(item, "getTsId", None), + safeCall(item, "getTSId", None), + safeCall(item, "getTomoId", None), + safeCall(item, "getTomogramId", None), + ) + + def getItemLabel(item: Any, fallback: Any = None) -> Optional[Any]: + return firstNonEmpty( + safeCall(item, "getTsId", None), + safeCall(item, "getTSId", None), + safeCall(item, "getObjLabel", None), + safeCall(item, "getFileName", None), + fallback, + ) + def isTiltSeriesSet(obj: Any) -> bool: name = normalizedClassName(obj) return "setoftiltseries" in name and "setoftiltseriesm" not in name @@ -2353,6 +2393,75 @@ def findInputRefForObject( "tomogram": None, "coordinates3d": None, } + relationObjects = { + "tiltSeries": None, + "ctf": None, + "tomogram": None, + "coordinates3d": None, + } + relationsByKey: Dict[str, Dict[str, Any]] = {} + + def upsertRelation(keyValue: Any, **values: Any) -> None: + key = str(keyValue) if keyValue is not None else "" + if not key: + return + + relation = relationsByKey.setdefault(key, { + "key": key, + "label": key, + }) + + for name, value in values.items(): + if value is not None: + relation[name] = value + + def addSetRelations(kind: str, obj: Any) -> None: + items = iterItems(obj) + + if not items: + for tsId in sorted(getTsIds(obj)): + if kind == "tiltSeries": + upsertRelation(tsId, tiltSeriesId=tsId, label=tsId) + elif kind == "ctf": + upsertRelation(tsId, ctfSeriesId=tsId, tiltSeriesId=tsId, label=tsId) + elif kind == "tomogram": + upsertRelation(tsId, tomogramId=tsId, label=tsId) + elif kind == "coordinates3d": + upsertRelation(tsId, coordinatesTomogramId=tsId, label=tsId) + return + + for index, item in enumerate(items): + tsId = getItemTsId(item) + objId = getObjId(item) + key = firstNonEmpty(tsId, objId, index) + label = getItemLabel(item, key) + + if kind == "tiltSeries": + upsertRelation( + key, + tiltSeriesId=firstNonEmpty(tsId, objId, index), + label=label, + ) + elif kind == "ctf": + upsertRelation( + key, + ctfSeriesId=firstNonEmpty(tsId, objId, index), + tiltSeriesId=tsId, + label=label, + ) + elif kind == "tomogram": + upsertRelation( + key, + tomogramId=firstNonEmpty(tsId, objId, index), + tomogramVolumeId=index, + label=label, + ) + elif kind == "coordinates3d": + upsertRelation( + key, + coordinatesTomogramId=firstNonEmpty(tsId, objId, index), + label=label, + ) rootSource = { "protocolId": protocolId, @@ -2365,6 +2474,7 @@ def findInputRefForObject( if isCoordinates3dSet(outputObj): links["coordinates3d"] = buildLink(outputObj, rootSource) summaries["coordinates3d"] = buildSummary(outputObj, outputTsIds) + relationObjects["coordinates3d"] = outputObj tomograms = safeCall(outputObj, "getPrecedents", None) if tomograms is not None: @@ -2372,26 +2482,30 @@ def findInputRefForObject( tomogramRef = findInputRefForObject(tomograms, inputRefs) links["tomogram"] = buildLink(tomograms, tomogramRef, statusValue="inferred") summaries["tomogram"] = buildSummary(tomograms, tomoTsIds) + relationObjects["tomogram"] = tomograms outputTsIds = tomoTsIds elif isTomogramSet(outputObj): links["tomogram"] = buildLink(outputObj, rootSource) summaries["tomogram"] = buildSummary(outputObj, outputTsIds) - + relationObjects["tomogram"] = outputObj elif isCtfTomoSeriesSet(outputObj): links["ctf"] = buildLink(outputObj, rootSource) summaries["ctf"] = buildSummary(outputObj, outputTsIds) + relationObjects["ctf"] = outputObj tiltSeries = safeCall(outputObj, "getSetOfTiltSeries", None) if tiltSeries is not None and isTiltSeriesSet(tiltSeries): tiltRef = findInputRefForObject(tiltSeries, localRefs, isTiltSeriesSet) links["tiltSeries"] = buildLink(tiltSeries, tiltRef, statusValue="inferred") summaries["tiltSeries"] = buildSummary(tiltSeries, outputTsIds) + relationObjects["tiltSeries"] = tiltSeries elif isTiltSeriesSet(outputObj): links["tiltSeries"] = buildLink(outputObj, rootSource) summaries["tiltSeries"] = buildSummary(outputObj, outputTsIds) + relationObjects["tiltSeries"] = outputObj if outputTsIds and links["ctf"] is None: ctfRef = findInputRef(isCtfTomoSeriesSet, outputTsIds) @@ -2399,6 +2513,7 @@ def findInputRefForObject( ctfSet = ctfRef["object"] links["ctf"] = buildLink(ctfSet, ctfRef) summaries["ctf"] = buildSummary(ctfSet, outputTsIds) + relationObjects["ctf"] = ctfSet tiltSeries = safeCall(ctfSet, "getSetOfTiltSeries", None) if tiltSeries is not None and links["tiltSeries"] is None and isTiltSeriesSet(tiltSeries): @@ -2409,6 +2524,7 @@ def findInputRefForObject( ) links["tiltSeries"] = buildLink(tiltSeries, tiltRef, statusValue="inferred") summaries["tiltSeries"] = buildSummary(tiltSeries, outputTsIds) + relationObjects["tiltSeries"] = tiltSeries if outputTsIds and links["tiltSeries"] is None: tiltRef = findInputRef(isTiltSeriesSet, outputTsIds) @@ -2416,6 +2532,20 @@ def findInputRefForObject( tiltSeries = tiltRef["object"] links["tiltSeries"] = buildLink(tiltSeries, tiltRef) summaries["tiltSeries"] = buildSummary(tiltSeries, outputTsIds) + relationObjects["tiltSeries"] = tiltSeries + + addSetRelations("tiltSeries", relationObjects["tiltSeries"]) + addSetRelations("ctf", relationObjects["ctf"]) + addSetRelations("tomogram", relationObjects["tomogram"]) + + if relationObjects["tomogram"] is not None: + addSetRelations("coordinates3d", relationObjects["tomogram"]) + else: + addSetRelations("coordinates3d", relationObjects["coordinates3d"]) + + relations = self._safeScipionValue({ + "items": list(relationsByKey.values()), + }) return { "root": { @@ -2426,9 +2556,9 @@ def findInputRefForObject( }, "links": links, "summaries": summaries, + "relations": relations, } - def buildProtocolOutputThumbnail( self, protocolId: int, From 0e8a6efd61b35ae5d916f38bbd4f5989e520ca49 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 17 Jun 2026 12:57:12 +0200 Subject: [PATCH 09/14] Improve integrated context resolution for Coordinates3D tomograms --- app/backend/api/services/project_service.py | 35 +++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 19e5739e..cb788771 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2227,6 +2227,37 @@ def isCoordinates3dSet(obj: Any) -> bool: def isCtfTomoSeriesSet(obj: Any) -> bool: return "setofctftomoseries" in normalizedClassName(obj) + def getFirstIteratorItem(value: Any) -> Any: + if value is None: + return None + + try: + iterator = value.iterItems() if hasattr(value, "iterItems") else iter(value) + return next(iterator, None) + except Exception: + return None + + def getCoordinates3dTomograms(coordsSet: Any) -> Any: + for methodName in ("getTomograms", "getVolumes", "getPrecedents"): + tomograms = safeCall(coordsSet, methodName, None) + if tomograms is not None and isTomogramSet(tomograms): + return tomograms + + for methodName in ("iterTomograms", "iterVolumes"): + tomogramsIter = safeCall(coordsSet, methodName, None) + firstTomogram = getFirstIteratorItem(tomogramsIter) + if firstTomogram is None: + continue + + parent = safeCall(firstTomogram, "getObjParent", None) + if parent is None: + parent = getattr(firstTomogram, "_objParent", None) + + if parent is not None and isTomogramSet(parent): + return parent + + return None + def buildLink( obj: Any, source: Optional[Dict[str, Any]] = None, @@ -2476,10 +2507,10 @@ def addSetRelations(kind: str, obj: Any) -> None: summaries["coordinates3d"] = buildSummary(outputObj, outputTsIds) relationObjects["coordinates3d"] = outputObj - tomograms = safeCall(outputObj, "getPrecedents", None) + tomograms = getCoordinates3dTomograms(outputObj) if tomograms is not None: tomoTsIds = outputTsIds or getTsIds(tomograms) - tomogramRef = findInputRefForObject(tomograms, inputRefs) + tomogramRef = findInputRefForObject(tomograms, localRefs, isTomogramSet) links["tomogram"] = buildLink(tomograms, tomogramRef, statusValue="inferred") summaries["tomogram"] = buildSummary(tomograms, tomoTsIds) relationObjects["tomogram"] = tomograms From 810b7fb89a53c28e2f02644abefa534584746b89 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 17 Jun 2026 17:48:27 +0200 Subject: [PATCH 10/14] feat(protocols): sync protocol steps to PostgreSQL --- alembic/env.py | 9 +- app/backend/api/routers/project_router.py | 15 ++ app/backend/api/schemas/protocols_schema.py | 16 +++ app/backend/api/services/environment.py | 4 + app/backend/api/services/project_service.py | 10 ++ .../api/services/protocol_steps_sync.py | 132 ++++++++++++++++++ app/backend/mapper/postgresql.py | 98 +++++++++++++ app/backend/models/__init__.py | 1 + app/backend/models/protocol_step_model.py | 64 +++++++++ 9 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 app/backend/api/services/protocol_steps_sync.py create mode 100644 app/backend/models/protocol_step_model.py diff --git a/alembic/env.py b/alembic/env.py index 0d9f9590..b8910b44 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -29,7 +29,14 @@ # importModelsForMetadata from app.backend.database import Base # noqa: E402 -from app.backend.models import user_model, project_model, protocol_model, tag_model, protocol_tag_assignment_model # noqa: F401,E402 +from app.backend.models import ( # noqa: F401,E402 + user_model, + project_model, + protocol_model, + tag_model, + protocol_tag_assignment_model, + protocol_step_model, +) target_metadata = Base.metadata diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 12030e2b..68e0f52e 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -348,6 +348,21 @@ async def loadProtocol( return service.getProtocolParams(projectId, protocolId) +@router.get("/{projectId}/protocols/{protocolId}/steps", response_model=Any) +def listProtocolSteps( + projectId: int, + protocolId: int, + currentUser=Depends(getCurrentUser), + mapper: PostgresqlFlatMapper = Depends(getMapper), + service: ProjectService = Depends(getProjectService), +): + project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + if not project: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") + + return service.listProtocolStepsService(mapper, projectId, protocolId) + + @router.get("/{projectId}/protclass/{protClassName}", response_model=Any) async def loadNewProtocol( projectId: int, diff --git a/app/backend/api/schemas/protocols_schema.py b/app/backend/api/schemas/protocols_schema.py index fe1c5b65..89126ae5 100644 --- a/app/backend/api/schemas/protocols_schema.py +++ b/app/backend/api/schemas/protocols_schema.py @@ -99,3 +99,19 @@ class WorkflowImportRequest(BaseModel): sourceProjectId: Optional[Union[int, str]] = None sourceProjectName: Optional[str] = None + +class ProtocolStepOut(BaseModel): + index: int + name: str + status: str + prerequisites: List[int] = [] + args: Optional[Any] = None + initTime: Optional[datetime] = None + endTime: Optional[datetime] = None + elapsedSeconds: Optional[float] = None + error: Optional[str] = None + interactive: bool = False + needsGpu: bool = True + event: Optional[str] = None + updatedAt: Optional[datetime] = None + diff --git a/app/backend/api/services/environment.py b/app/backend/api/services/environment.py index b524bfaf..ecbdd563 100644 --- a/app/backend/api/services/environment.py +++ b/app/backend/api/services/environment.py @@ -52,3 +52,7 @@ def prepareEnvironment(): scipionHome = variables.get(pyworkflow.SCIPION_HOME_VAR) or os.environ.get(pyworkflow.SCIPION_HOME_VAR) if scipionHome: os.chdir(scipionHome) + os.environ.setdefault( + "SCIPION_PROTOCOL_STEPS_NOTIFIER", + "app.backend.api.services.protocol_steps_sync:syncProtocolStepsEvent", + ) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index cb788771..d9faec2c 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -3331,6 +3331,9 @@ def saveProtocol(self, mapper, projectId, protocolId, protocolClassName, params, return protocol, errorList + def listProtocolStepsService(self, mapper, projectId: int, protocolId: int): + return mapper.listProtocolSteps(projectId, protocolId) + def launchProtocol(self, mapper, projectId, protocolId, protocolClassName, params, executeMode): """ Save, validate, and execute a protocol action. @@ -3406,6 +3409,13 @@ def launchProtocol(self, mapper, projectId, protocolId, protocolClassName, param detail=errors, ) + self.syncProjectProtocolsAndDependencies( + mapper, + projectId, + refresh=True, + checkPid=False, + ) + try: if executeMode == "schedule": self.currentProject.scheduleProtocol(protocol) diff --git a/app/backend/api/services/protocol_steps_sync.py b/app/backend/api/services/protocol_steps_sync.py new file mode 100644 index 00000000..cdebccf5 --- /dev/null +++ b/app/backend/api/services/protocol_steps_sync.py @@ -0,0 +1,132 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +import json +import logging +import os +from typing import Any, Dict, List, Optional + +from app.backend.mapper.postgresql import PostgresqlDb, PostgresqlFlatMapper + +logger = logging.getLogger(__name__) + + +def _value(obj: Any, default=None): + try: + if hasattr(obj, "hasValue") and not obj.hasValue(): + return default + if hasattr(obj, "get"): + return obj.get(default) + except TypeError: + try: + return obj.get() + except Exception: + return default + except Exception: + return default + return obj if obj is not None else default + + +def _jsonValue(text: Any): + if text in (None, ""): + return None + try: + return json.loads(str(text)) + except Exception: + return str(text) + + +def _projectPath(protocol) -> Optional[str]: + project = protocol.getProject() + for attr in ("path", "_path"): + value = getattr(project, attr, None) + if value: + return os.path.abspath(str(value)) + if hasattr(project, "getPath"): + return os.path.abspath(str(project.getPath())) + return None + + +def _serializeStep(step, event: str) -> Dict[str, Any]: + elapsed = None + try: + elapsed = step.getElapsedTime().total_seconds() + except Exception: + pass + + return { + "index": int(step.getIndex() or 0), + "name": _value(getattr(step, "funcName", None), step.getClassName()), + "status": step.getStatus(), + "prerequisites": [int(p) for p in step.getPrerequisites()], + "args": _jsonValue(_value(getattr(step, "argsStr", None))), + "initTime": _value(getattr(step, "initTime", None)), + "endTime": _value(getattr(step, "endTime", None)), + "elapsedSeconds": elapsed, + "error": step.getErrorMessage(), + "interactive": step.isInteractive(), + "needsGpu": step.needsGPU(), + "event": event, + } + + +def _buildMapper() -> PostgresqlFlatMapper: + db = PostgresqlDb( + dbName=os.environ["DATABASE_NAME"], + user=os.environ["DATABASE_USER"], + password=os.environ["DATABASE_PASS"], + ) + return PostgresqlFlatMapper(db) + + +def syncProtocolStepsEvent(protocol, event: str, steps: List[Any], step: Any = None) -> None: + mapper = _buildMapper() + try: + projectPath = _projectPath(protocol) + target = mapper.resolveProtocolStepTarget(projectPath, protocol.getObjId()) + if not target: + logger.warning("Protocol steps target not found. projectPath=%s protocolId=%s", projectPath, protocol.getObjId()) + return + + projectId = int(target["projectId"]) + protocolDbId = int(target["protocolDbId"]) + protocolId = int(protocol.getObjId()) + + if event == "steps-stored": + mapper.replaceProtocolSteps( + projectId, + protocolDbId, + protocolId, + [_serializeStep(s, event) for s in steps], + ) + elif step is not None: + mapper.upsertProtocolStep( + projectId, + protocolDbId, + protocolId, + _serializeStep(step, event), + ) + finally: + mapper.db.close() \ No newline at end of file diff --git a/app/backend/mapper/postgresql.py b/app/backend/mapper/postgresql.py index 4e113a40..4ed440bf 100644 --- a/app/backend/mapper/postgresql.py +++ b/app/backend/mapper/postgresql.py @@ -164,6 +164,33 @@ def initTables(self): """ ) + self.db.execute(""" + CREATE TABLE IF NOT EXISTS protocol_steps ( + id SERIAL PRIMARY KEY, + "projectId" INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + "protocolDbId" INTEGER NOT NULL REFERENCES protocols(id) ON DELETE CASCADE, + "protocolId" TEXT NOT NULL, + "stepIndex" INTEGER NOT NULL, + name TEXT NOT NULL, + status TEXT NOT NULL, + prerequisites JSONB NOT NULL DEFAULT '[]'::jsonb, + args JSONB, + "initTime" TIMESTAMPTZ, + "endTime" TIMESTAMPTZ, + "elapsedSeconds" DOUBLE PRECISION, + error TEXT, + interactive BOOLEAN NOT NULL DEFAULT FALSE, + "needsGpu" BOOLEAN NOT NULL DEFAULT TRUE, + event TEXT, + "createdAt" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "updatedAt" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE ("projectId", "protocolDbId", "stepIndex") + ); + + CREATE INDEX IF NOT EXISTS protocol_steps_by_protocol + ON protocol_steps("projectId", "protocolDbId", "stepIndex"); + """) + # CreateProjectSharesTable (requires users and projects) self.db.execute( """ @@ -1152,6 +1179,77 @@ def deleteProjectProtocolsNotInProtocolIds( return len(staleDbIds) + def resolveProtocolStepTarget(self, projectPath: str, protocolId: int) -> Optional[Dict[str, Any]]: + return self.db.fetchOne( + """ + SELECT p."projectId", p.id AS "protocolDbId", p."protocolId" + FROM protocols p + JOIN projects pr ON pr.id = p."projectId" + WHERE p."protocolId" = %s + AND pr.name = %s + LIMIT 1 + """, + (str(protocolId), str(projectPath)), + ) + + def replaceProtocolSteps(self, projectId: int, protocolDbId: int, protocolId: int, steps: List[Dict[str, Any]]) -> None: + self.db.execute( + 'DELETE FROM protocol_steps WHERE "projectId" = %s AND "protocolDbId" = %s', + (projectId, protocolDbId), + ) + for step in steps or []: + self.upsertProtocolStep(projectId, protocolDbId, protocolId, step) + + def upsertProtocolStep(self, projectId: int, protocolDbId: int, protocolId: int, step: Dict[str, Any]) -> None: + self.db.execute( + """ + INSERT INTO protocol_steps ( + "projectId", "protocolDbId", "protocolId", "stepIndex", + name, status, prerequisites, args, "initTime", "endTime", + "elapsedSeconds", error, interactive, "needsGpu", event + ) + VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb, %s::jsonb, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT ("projectId", "protocolDbId", "stepIndex") + DO UPDATE SET + name = EXCLUDED.name, + status = EXCLUDED.status, + prerequisites = EXCLUDED.prerequisites, + args = EXCLUDED.args, + "initTime" = EXCLUDED."initTime", + "endTime" = EXCLUDED."endTime", + "elapsedSeconds" = EXCLUDED."elapsedSeconds", + error = EXCLUDED.error, + interactive = EXCLUDED.interactive, + "needsGpu" = EXCLUDED."needsGpu", + event = EXCLUDED.event, + "updatedAt" = NOW() + """, + ( + projectId, protocolDbId, str(protocolId), step["index"], + step["name"], step["status"], + json.dumps(step.get("prerequisites") or []), + json.dumps(step.get("args")), + step.get("initTime"), step.get("endTime"), + step.get("elapsedSeconds"), step.get("error"), + bool(step.get("interactive")), bool(step.get("needsGpu", True)), + step.get("event"), + ), + ) + + def listProtocolSteps(self, projectId: int, protocolId: int) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT "stepIndex" AS index, name, status, prerequisites, args, + "initTime", "endTime", "elapsedSeconds", error, + interactive, "needsGpu", event, "updatedAt" + FROM protocol_steps + WHERE "projectId" = %s + AND "protocolId" = %s + ORDER BY "stepIndex" ASC + """, + (projectId, str(protocolId)), + ) + # ----------------------------- # Settings Methods # ----------------------------- diff --git a/app/backend/models/__init__.py b/app/backend/models/__init__.py index faa6bc33..4abd50b4 100644 --- a/app/backend/models/__init__.py +++ b/app/backend/models/__init__.py @@ -27,3 +27,4 @@ from app.backend.models.project_model import Project # noqa: F401 from app.backend.models.protocol_model import Protocol # noqa: F401 from app.backend.models.settings_models import * # noqa: F401 +from app.backend.models.protocol_step_model import ProtocolStep # noqa: F401 diff --git a/app/backend/models/protocol_step_model.py b/app/backend/models/protocol_step_model.py new file mode 100644 index 00000000..d03bd258 --- /dev/null +++ b/app/backend/models/protocol_step_model.py @@ -0,0 +1,64 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from sqlalchemy import Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.sql import func + +from app.backend.database import Base + + +class ProtocolStep(Base): + __tablename__ = "protocol_steps" + + id = Column(Integer, primary_key=True, index=True) + projectId = Column(Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + protocolDbId = Column(Integer, ForeignKey("protocols.id", ondelete="CASCADE"), nullable=False) + protocolId = Column(String, nullable=False) + stepIndex = Column(Integer, nullable=False) + + name = Column(Text, nullable=False) + status = Column(Text, nullable=False) + + prerequisites = Column(JSONB, nullable=False, server_default="[]") + args = Column(JSONB, nullable=True) + + initTime = Column(DateTime(timezone=True), nullable=True) + endTime = Column(DateTime(timezone=True), nullable=True) + elapsedSeconds = Column(Float, nullable=True) + + error = Column(Text, nullable=True) + interactive = Column(Boolean, nullable=False, server_default="false") + needsGpu = Column(Boolean, nullable=False, server_default="true") + event = Column(Text, nullable=True) + + createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updatedAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + + __table_args__ = ( + UniqueConstraint("projectId", "protocolDbId", "stepIndex", name="ux_protocol_steps_project_protocol_step"), + Index("idx_protocol_steps_protocol", "projectId", "protocolDbId", "stepIndex"), + Index("idx_protocol_steps_protocol_id", "projectId", "protocolId"), + ) \ No newline at end of file From 92c08d41fe39f119227ac71b4de355266cc6cf50 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 17 Jun 2026 17:49:29 +0200 Subject: [PATCH 11/14] Adding the migration --- .../6806b9a72a6e_add_protocol_steps.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 alembic/versions/6806b9a72a6e_add_protocol_steps.py diff --git a/alembic/versions/6806b9a72a6e_add_protocol_steps.py b/alembic/versions/6806b9a72a6e_add_protocol_steps.py new file mode 100644 index 00000000..2fd558f8 --- /dev/null +++ b/alembic/versions/6806b9a72a6e_add_protocol_steps.py @@ -0,0 +1,54 @@ +"""add protocol steps + +Revision ID: 6806b9a72a6e +Revises: 9b7b3c4d8e21 +Create Date: 2026-06-17 17:40:02.063195 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '6806b9a72a6e' +down_revision: Union[str, None] = '9b7b3c4d8e21' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "protocol_steps", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("projectId", sa.Integer(), nullable=False), + sa.Column("protocolDbId", sa.Integer(), nullable=False), + sa.Column("protocolId", sa.String(), nullable=False), + sa.Column("stepIndex", sa.Integer(), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("prerequisites", sa.dialects.postgresql.JSONB(), server_default="[]", nullable=False), + sa.Column("args", sa.dialects.postgresql.JSONB(), nullable=True), + sa.Column("initTime", sa.DateTime(timezone=True), nullable=True), + sa.Column("endTime", sa.DateTime(timezone=True), nullable=True), + sa.Column("elapsedSeconds", sa.Float(), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("interactive", sa.Boolean(), server_default="false", nullable=False), + sa.Column("needsGpu", sa.Boolean(), server_default="true", nullable=False), + sa.Column("event", sa.Text(), nullable=True), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.ForeignKeyConstraint(["projectId"], ["projects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["protocolDbId"], ["protocols.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("projectId", "protocolDbId", "stepIndex", name="ux_protocol_steps_project_protocol_step"), + ) + op.create_index("idx_protocol_steps_protocol", "protocol_steps", ["projectId", "protocolDbId", "stepIndex"]) + op.create_index("idx_protocol_steps_protocol_id", "protocol_steps", ["projectId", "protocolId"]) + + +def downgrade() -> None: + op.drop_index("idx_protocol_steps_protocol_id", table_name="protocol_steps") + op.drop_index("idx_protocol_steps_protocol", table_name="protocol_steps") + op.drop_table("protocol_steps") From 752ea06e3a0d467d880bb8c56ede5f073dc50794 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 17 Jun 2026 18:07:37 +0200 Subject: [PATCH 12/14] feat(protocols): expose protocol steps on demand --- app/backend/mapper/postgresql.py | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/app/backend/mapper/postgresql.py b/app/backend/mapper/postgresql.py index 4ed440bf..01f5c3be 100644 --- a/app/backend/mapper/postgresql.py +++ b/app/backend/mapper/postgresql.py @@ -1250,6 +1250,40 @@ def listProtocolSteps(self, projectId: int, protocolId: int) -> List[Dict[str, A (projectId, str(protocolId)), ) + def getProjectProtocolStepsByProtocolId(self, projectId: int) -> Dict[str, List[Dict[str, Any]]]: + rows = self.db.fetchAll( + """ + SELECT + "protocolId", + "stepIndex" AS "index", + name, + status, + prerequisites, + args, + "initTime", + "endTime", + "elapsedSeconds", + error, + interactive, + "needsGpu", + event, + "updatedAt" + FROM protocol_steps + WHERE "projectId" = %s + ORDER BY "protocolId", "stepIndex" ASC + """, + (projectId,), + ) + + result: Dict[str, List[Dict[str, Any]]] = {} + for row in rows: + protocolId = str(row["protocolId"]) + step = dict(row) + step.pop("protocolId", None) + result.setdefault(protocolId, []).append(step) + + return result + # ----------------------------- # Settings Methods # ----------------------------- From 85b43987647bafe5f9e5dd45fded39b6f9fc9e49 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 18 Jun 2026 10:02:29 +0200 Subject: [PATCH 13/14] feat(protocols): add protocol step status update endpoint --- app/backend/api/routers/project_router.py | 27 +++++ app/backend/api/services/project_service.py | 119 +++++++++++++++++++- app/backend/mapper/postgresql.py | 33 ++++++ 3 files changed, 178 insertions(+), 1 deletion(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 68e0f52e..31cccff0 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -348,6 +348,10 @@ async def loadProtocol( return service.getProtocolParams(projectId, protocolId) +class ProtocolStepStatusUpdate(BaseModel): + status: Literal["new", "finished"] = Field(..., description="New status for the selected protocol step") + + @router.get("/{projectId}/protocols/{protocolId}/steps", response_model=Any) def listProtocolSteps( projectId: int, @@ -363,6 +367,29 @@ def listProtocolSteps( return service.listProtocolStepsService(mapper, projectId, protocolId) +@router.patch("/{projectId}/protocols/{protocolId}/steps/{stepIndex}/status", response_model=Any) +def updateProtocolStepStatus( + projectId: int, + protocolId: int, + stepIndex: int, + payload: ProtocolStepStatusUpdate, + currentUser=Depends(getCurrentUser), + mapper: PostgresqlFlatMapper = Depends(getMapper), + service: ProjectService = Depends(getProjectService), +): + project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + if not project: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") + + return service.updateProtocolStepStatusService( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + stepIndex=stepIndex, + stepStatus=payload.status, + ) + + @router.get("/{projectId}/protclass/{protClassName}", response_model=Any) async def loadNewProtocol( projectId: int, diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index d9faec2c..a9f634aa 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -62,7 +62,15 @@ from pwem.viewers.mdviewer.sqlite_dao import ScipionSetsDAO, OBJECT_TABLE from pwem.viewers.mdviewer.star_dao import StarFile from pyworkflow.object import PointerList, Pointer, CsvList -from pyworkflow.protocol import MODE_RESUME, MODE_RESTART, STATUS_LAUNCHED, STATUS_RUNNING, STATUS_SCHEDULED +from pyworkflow.protocol import ( + MODE_RESUME, + MODE_RESTART, + STATUS_FINISHED, + STATUS_LAUNCHED, + STATUS_NEW, + STATUS_RUNNING, + STATUS_SCHEDULED, +) from pyworkflow.template import TemplateList try: @@ -3334,6 +3342,115 @@ def saveProtocol(self, mapper, projectId, protocolId, protocolClassName, params, def listProtocolStepsService(self, mapper, projectId: int, protocolId: int): return mapper.listProtocolSteps(projectId, protocolId) + def updateProtocolStepStatusService( + self, + mapper, + projectId: int, + protocolId: int, + stepIndex: int, + stepStatus: str, + ): + statusMap = { + "new": STATUS_NEW, + "finished": STATUS_FINISHED, + } + + normalizedStatus = str(stepStatus or "").strip().lower() + if normalizedStatus not in statusMap: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Invalid step status. Allowed values: new, finished", + ) + + targetStatus = statusMap[normalizedStatus] + + if self.currentProject is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="No current project loaded", + ) + + try: + protocol = self.currentProject.getProtocol(int(protocolId)) + except Exception: + protocol = None + + if protocol is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol not found: {protocolId}", + ) + + try: + steps = protocol.loadSteps() or [] + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load protocol steps: {e}", + ) + + targetStep = None + for fallbackIndex, step in enumerate(steps, start=1): + rawIndex = getattr(step, "_index", None) or fallbackIndex + try: + if int(rawIndex) == int(stepIndex): + targetStep = step + break + except Exception: + continue + + if targetStep is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Step not found: {stepIndex}", + ) + + stepObjId = None + try: + stepObjId = targetStep.getObjId() + except Exception: + stepObjId = None + + if stepObjId is None: + stepObjId = getattr(targetStep, "_objId", None) + try: + if hasattr(stepObjId, "get"): + stepObjId = stepObjId.get() + except Exception: + pass + + if stepObjId is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Could not resolve object id for step {stepIndex}", + ) + + try: + protocol._updateSteps( + lambda step: step.setStatus(targetStatus), + where="id='%s'" % stepObjId, + ) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to update Scipion step status: {e}", + ) + + row = mapper.updateProtocolStepStatus( + projectId=projectId, + protocolId=protocolId, + stepIndex=stepIndex, + stepStatus=targetStatus, + ) + + if not row: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol step not found in PostgreSQL: {stepIndex}", + ) + + return row + def launchProtocol(self, mapper, projectId, protocolId, protocolClassName, params, executeMode): """ Save, validate, and execute a protocol action. diff --git a/app/backend/mapper/postgresql.py b/app/backend/mapper/postgresql.py index 01f5c3be..f7236bf7 100644 --- a/app/backend/mapper/postgresql.py +++ b/app/backend/mapper/postgresql.py @@ -1250,6 +1250,39 @@ def listProtocolSteps(self, projectId: int, protocolId: int) -> List[Dict[str, A (projectId, str(protocolId)), ) + def updateProtocolStepStatus( + self, + projectId: int, + protocolId: int, + stepIndex: int, + stepStatus: str, + ) -> Optional[Dict[str, Any]]: + return self.db.fetchOne( + """ + UPDATE protocol_steps + SET status = %s, + "updatedAt" = NOW() + WHERE "projectId" = %s + AND "protocolId" = %s + AND "stepIndex" = %s + RETURNING + "stepIndex" AS index, + name, + status, + prerequisites, + args, + "initTime", + "endTime", + "elapsedSeconds", + error, + interactive, + "needsGpu", + event, + "updatedAt" + """, + (stepStatus, projectId, str(protocolId), stepIndex), + ) + def getProjectProtocolStepsByProtocolId(self, projectId: int) -> Dict[str, List[Dict[str, Any]]]: rows = self.db.fetchAll( """ From 535dcb0be71dcfc39c58d9b925abd9f2a78a5034 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 18 Jun 2026 13:44:43 +0200 Subject: [PATCH 14/14] test(protocols): cover protocol steps status updates --- .../api/test_project_router_protocol_steps.py | 229 ++++++++++++++ .../api/routers/test_settings_router.py | 2 + .../test_project_service_protocol_steps.py | 281 ++++++++++++++++++ .../mapper/test_postgresql_protocol_steps.py | 74 +++++ 4 files changed, 586 insertions(+) create mode 100644 tests/integration/api/test_project_router_protocol_steps.py create mode 100644 tests/unit/backend/api/services/test_project_service_protocol_steps.py create mode 100644 tests/unit/backend/mapper/test_postgresql_protocol_steps.py diff --git a/tests/integration/api/test_project_router_protocol_steps.py b/tests/integration/api/test_project_router_protocol_steps.py new file mode 100644 index 00000000..8e5597f6 --- /dev/null +++ b/tests/integration/api/test_project_router_protocol_steps.py @@ -0,0 +1,229 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from fastapi import HTTPException, FastAPI +from fastapi.testclient import TestClient +import pytest + + +class FakeProtocolStepsProjectService: + def __init__(self): + self.projectByIdResult = {"id": 1, "name": "Demo project"} + self.lastGetProjectByIdCall = None + + self.protocolStepsResult = [ + { + "index": 1, + "name": "resumeStep", + "status": "finished", + "prerequisites": [], + "args": [], + "initTime": None, + "endTime": None, + "elapsedSeconds": 0, + "error": None, + "interactive": False, + "needsGpu": False, + "event": None, + "updatedAt": None, + }, + ] + self.lastListProtocolStepsCall = None + + self.updateProtocolStepStatusResult = { + "index": 2, + "name": "processStep", + "status": "finished", + "prerequisites": [1], + "args": ["TS_1"], + "initTime": None, + "endTime": None, + "elapsedSeconds": 0, + "error": None, + "interactive": False, + "needsGpu": True, + "event": None, + "updatedAt": None, + } + self.updateProtocolStepStatusError = None + self.lastUpdateProtocolStepStatusCall = None + + def getProjectById(self, mapper, projectId, currentUser, refresh=False, checkPid=False): + self.lastGetProjectByIdCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + "refresh": refresh, + "checkPid": checkPid, + } + return self.projectByIdResult + + def listProtocolStepsService(self, mapper, projectId, protocolId): + self.lastListProtocolStepsCall = { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + } + return self.protocolStepsResult + + def updateProtocolStepStatusService( + self, + mapper, + projectId, + protocolId, + stepIndex, + stepStatus, + ): + self.lastUpdateProtocolStepStatusCall = { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + "stepIndex": stepIndex, + "stepStatus": stepStatus, + } + if self.updateProtocolStepStatusError is not None: + raise self.updateProtocolStepStatusError + return self.updateProtocolStepStatusResult + + +@pytest.fixture +def fakeProtocolStepsProjectService(): + return FakeProtocolStepsProjectService() + + +@pytest.fixture +def protocolStepsClient( + projectRouterModule, + fakeProjectMapper, + fakeProtocolStepsProjectService, +): + app = FastAPI() + app.include_router(projectRouterModule.router) + + app.dependency_overrides[projectRouterModule.getMapper] = lambda: fakeProjectMapper + app.dependency_overrides[projectRouterModule.getCurrentUser] = lambda: { + "id": 1, + "email": "user@example.com", + "role": "user", + } + app.dependency_overrides[ + projectRouterModule.getProjectService + ] = lambda: fakeProtocolStepsProjectService + + with TestClient(app) as client: + yield client + + app.dependency_overrides.clear() + + +def test_ListProtocolStepsReturns404WhenProjectMissing( + protocolStepsClient, + fakeProtocolStepsProjectService, +): + fakeProtocolStepsProjectService.projectByIdResult = None + + response = protocolStepsClient.get("/projects/1/protocols/10/steps") + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + + +def test_ListProtocolStepsDelegatesToService( + protocolStepsClient, + fakeProjectMapper, + fakeProtocolStepsProjectService, +): + response = protocolStepsClient.get("/projects/1/protocols/10/steps") + + assert response.status_code == 200 + assert response.json() == fakeProtocolStepsProjectService.protocolStepsResult + assert fakeProtocolStepsProjectService.lastListProtocolStepsCall == { + "mapper": fakeProjectMapper, + "projectId": 1, + "protocolId": 10, + } + + +def test_UpdateProtocolStepStatusReturns404WhenProjectMissing( + protocolStepsClient, + fakeProtocolStepsProjectService, +): + fakeProtocolStepsProjectService.projectByIdResult = None + + response = protocolStepsClient.patch( + "/projects/1/protocols/10/steps/2/status", + json={"status": "finished"}, + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + + +def test_UpdateProtocolStepStatusRejectsInvalidStatus(protocolStepsClient): + response = protocolStepsClient.patch( + "/projects/1/protocols/10/steps/2/status", + json={"status": "running"}, + ) + + assert response.status_code == 422 + + +def test_UpdateProtocolStepStatusDelegatesToService( + protocolStepsClient, + fakeProjectMapper, + fakeProtocolStepsProjectService, +): + response = protocolStepsClient.patch( + "/projects/1/protocols/10/steps/2/status", + json={"status": "finished"}, + ) + + assert response.status_code == 200 + assert response.json() == fakeProtocolStepsProjectService.updateProtocolStepStatusResult + assert fakeProtocolStepsProjectService.lastUpdateProtocolStepStatusCall == { + "mapper": fakeProjectMapper, + "projectId": 1, + "protocolId": 10, + "stepIndex": 2, + "stepStatus": "finished", + } + + +def test_UpdateProtocolStepStatusPropagatesHttpException( + protocolStepsClient, + fakeProtocolStepsProjectService, +): + fakeProtocolStepsProjectService.updateProtocolStepStatusError = HTTPException( + status_code=404, + detail="Step not found: 99", + ) + + response = protocolStepsClient.patch( + "/projects/1/protocols/10/steps/99/status", + json={"status": "new"}, + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "Step not found: 99" \ No newline at end of file diff --git a/tests/unit/backend/api/routers/test_settings_router.py b/tests/unit/backend/api/routers/test_settings_router.py index dfd77400..901a9d72 100644 --- a/tests/unit/backend/api/routers/test_settings_router.py +++ b/tests/unit/backend/api/routers/test_settings_router.py @@ -45,6 +45,7 @@ def __init__(self): "graphFocusModeEnabled": False, "protocolOutputThumbnailsEnabled": True, "workflowsAutoRefreshSec": 5, + "workflowViewMode": "treeTb" } self.instanceSettingsResult = { "enableCelery": False, @@ -312,6 +313,7 @@ def test_PutUserSettingsDelegatesToService(settingsClient, fakeSettingsService): "graphFocusModeEnabled": True, "workflowsAutoRefreshSec": 10, "protocolOutputThumbnailsEnabled": True, + "workflowViewMode": "treeTb" } diff --git a/tests/unit/backend/api/services/test_project_service_protocol_steps.py b/tests/unit/backend/api/services/test_project_service_protocol_steps.py new file mode 100644 index 00000000..b6fcc2c5 --- /dev/null +++ b/tests/unit/backend/api/services/test_project_service_protocol_steps.py @@ -0,0 +1,281 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +import importlib + +import pytest +from fastapi import HTTPException + + +class FakeCurrentProject: + def __init__(self): + self.protocols = {} + + def getProtocol(self, protocolId): + return self.protocols[int(protocolId)] + + +class FakeMapper: + def __init__(self): + self.listProtocolStepsResult = [] + self.updateProtocolStepStatusCalls = [] + self.updateProtocolStepStatusResult = None + + def listProtocolSteps(self, projectId, protocolId): + return self.listProtocolStepsResult + + def updateProtocolStepStatus(self, **kwargs): + self.updateProtocolStepStatusCalls.append(kwargs) + return self.updateProtocolStepStatusResult + + +class FakeValueHolder: + def __init__(self, value): + self.value = value + + def get(self): + return self.value + + +class FakeStep: + def __init__(self, index, objId, raiseGetObjId=False): + self._index = index + self._objId = objId + self.raiseGetObjId = raiseGetObjId + self.status = None + + def getObjId(self): + if self.raiseGetObjId: + raise RuntimeError("getObjId failed") + return self._objId + + def setStatus(self, status): + self.status = status + + +class FakeProtocolWithSteps: + def __init__(self, steps): + self.steps = steps + self.updateStepsCalls = [] + self.failLoadSteps = None + self.failUpdateSteps = None + + @staticmethod + def _resolveStepObjId(step): + objId = getattr(step, "_objId", None) + if hasattr(objId, "get"): + return objId.get() + return objId + + def loadSteps(self): + if self.failLoadSteps is not None: + raise self.failLoadSteps + return self.steps + + def _updateSteps(self, callback, where=None): + if self.failUpdateSteps is not None: + raise self.failUpdateSteps + + self.updateStepsCalls.append({"where": where}) + + for step in self.steps: + if where == "id='%s'" % self._resolveStepObjId(step): + callback(step) + + +@pytest.fixture +def projectServiceModule(authTestEnv): + return importlib.import_module("app.backend.api.services.project_service") + + +@pytest.fixture +def service(projectServiceModule): + instance = object.__new__(projectServiceModule.ProjectService) + instance.currentProject = FakeCurrentProject() + instance.tomoList = {} + return instance + + +@pytest.fixture +def mapper(): + return FakeMapper() + + +def test_ListProtocolStepsDelegatesToMapper(service, mapper): + mapper.listProtocolStepsResult = [ + {"index": 1, "name": "resumeStep", "status": "finished"}, + ] + + result = service.listProtocolStepsService(mapper, projectId=1, protocolId=10) + + assert result == [{"index": 1, "name": "resumeStep", "status": "finished"}] + + +def test_UpdateProtocolStepStatusUpdatesScipionAndPostgres( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "STATUS_FINISHED", "finished") + + stepA = FakeStep(index=1, objId=101) + stepB = FakeStep(index=2, objId=102) + protocol = FakeProtocolWithSteps([stepA, stepB]) + service.currentProject.protocols[10] = protocol + mapper.updateProtocolStepStatusResult = { + "index": 2, + "name": "processStep", + "status": "finished", + } + + result = service.updateProtocolStepStatusService( + mapper=mapper, + projectId=1, + protocolId=10, + stepIndex=2, + stepStatus="finished", + ) + + assert result == mapper.updateProtocolStepStatusResult + assert protocol.updateStepsCalls == [{"where": "id='102'"}] + assert stepA.status is None + assert stepB.status == "finished" + assert mapper.updateProtocolStepStatusCalls == [ + { + "projectId": 1, + "protocolId": 10, + "stepIndex": 2, + "stepStatus": "finished", + }, + ] + + +def test_UpdateProtocolStepStatusAcceptsObjIdHolderFallback( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "STATUS_NEW", "new") + + step = FakeStep( + index=3, + objId=FakeValueHolder(333), + raiseGetObjId=True, + ) + protocol = FakeProtocolWithSteps([step]) + service.currentProject.protocols[10] = protocol + mapper.updateProtocolStepStatusResult = {"index": 3, "status": "new"} + + result = service.updateProtocolStepStatusService( + mapper=mapper, + projectId=1, + protocolId=10, + stepIndex=3, + stepStatus="new", + ) + + assert result == {"index": 3, "status": "new"} + assert protocol.updateStepsCalls == [{"where": "id='333'"}] + assert step.status == "new" + + +def test_UpdateProtocolStepStatusRejectsInvalidStatus(service, mapper): + with pytest.raises(HTTPException) as exc: + service.updateProtocolStepStatusService( + mapper=mapper, + projectId=1, + protocolId=10, + stepIndex=1, + stepStatus="running", + ) + + assert exc.value.status_code == 422 + assert exc.value.detail == "Invalid step status. Allowed values: new, finished" + + +def test_UpdateProtocolStepStatusRaisesWhenCurrentProjectIsMissing(service, mapper): + service.currentProject = None + + with pytest.raises(HTTPException) as exc: + service.updateProtocolStepStatusService( + mapper=mapper, + projectId=1, + protocolId=10, + stepIndex=1, + stepStatus="finished", + ) + + assert exc.value.status_code == 500 + assert exc.value.detail == "No current project loaded" + + +def test_UpdateProtocolStepStatusRaisesWhenProtocolIsMissing(service, mapper): + with pytest.raises(HTTPException) as exc: + service.updateProtocolStepStatusService( + mapper=mapper, + projectId=1, + protocolId=99, + stepIndex=1, + stepStatus="finished", + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Protocol not found: 99" + + +def test_UpdateProtocolStepStatusRaisesWhenStepIsMissing(service, mapper): + protocol = FakeProtocolWithSteps([FakeStep(index=1, objId=101)]) + service.currentProject.protocols[10] = protocol + + with pytest.raises(HTTPException) as exc: + service.updateProtocolStepStatusService( + mapper=mapper, + projectId=1, + protocolId=10, + stepIndex=2, + stepStatus="finished", + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Step not found: 2" + + +def test_UpdateProtocolStepStatusRaisesWhenPostgresRowIsMissing(service, mapper): + protocol = FakeProtocolWithSteps([FakeStep(index=1, objId=101)]) + service.currentProject.protocols[10] = protocol + + with pytest.raises(HTTPException) as exc: + service.updateProtocolStepStatusService( + mapper=mapper, + projectId=1, + protocolId=10, + stepIndex=1, + stepStatus="finished", + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Protocol step not found in PostgreSQL: 1" \ No newline at end of file diff --git a/tests/unit/backend/mapper/test_postgresql_protocol_steps.py b/tests/unit/backend/mapper/test_postgresql_protocol_steps.py new file mode 100644 index 00000000..d244c2dc --- /dev/null +++ b/tests/unit/backend/mapper/test_postgresql_protocol_steps.py @@ -0,0 +1,74 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from app.backend.mapper.postgresql import PostgresqlFlatMapper + + +class FakeDb: + def __init__(self): + self.fetchOneCalls = [] + self.fetchOneResult = { + "index": 2, + "name": "processStep", + "status": "finished", + } + + def fetchOne(self, query, params): + self.fetchOneCalls.append( + { + "query": query, + "params": params, + }, + ) + return self.fetchOneResult + + +def test_UpdateProtocolStepStatusUpdatesSelectedStepAndReturnsRow(): + mapper = object.__new__(PostgresqlFlatMapper) + mapper.db = FakeDb() + + result = mapper.updateProtocolStepStatus( + projectId=1, + protocolId=10, + stepIndex=2, + stepStatus="finished", + ) + + assert result == { + "index": 2, + "name": "processStep", + "status": "finished", + } + assert len(mapper.db.fetchOneCalls) == 1 + + call = mapper.db.fetchOneCalls[0] + assert "UPDATE protocol_steps" in call["query"] + assert "SET status = %s" in call["query"] + assert '"updatedAt" = NOW()' in call["query"] + assert 'WHERE "projectId" = %s' in call["query"] + assert 'AND "protocolId" = %s' in call["query"] + assert 'AND "stepIndex" = %s' in call["query"] + assert '"stepIndex" AS index' in call["query"] + assert call["params"] == ("finished", 1, "10", 2) \ No newline at end of file