diff --git a/.backend_reload_marker b/.backend_reload_marker new file mode 100644 index 00000000..b79513fc --- /dev/null +++ b/.backend_reload_marker @@ -0,0 +1 @@ +reloadMarker=1780572679.3509667 diff --git a/alembic/env.py b/alembic/env.py index b8910b44..c7fecb6e 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -36,6 +36,7 @@ tag_model, protocol_tag_assignment_model, protocol_step_model, + scipion_object_model, ) target_metadata = Base.metadata diff --git a/alembic/versions/1f8e5b400102_add_scipion_object_persistence_models.py b/alembic/versions/1f8e5b400102_add_scipion_object_persistence_models.py new file mode 100644 index 00000000..e8f2a914 --- /dev/null +++ b/alembic/versions/1f8e5b400102_add_scipion_object_persistence_models.py @@ -0,0 +1,249 @@ +"""add scipion object persistence models + +Revision ID: 1f8e5b400102 +Revises: 6806b9a72a6e +Create Date: 2026-06-19 11:10:02.296385 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = "1f8e5b400102" +down_revision: Union[str, None] = "6806b9a72a6e" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "scipion_object_types", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("className", sa.Text(), nullable=False), + sa.Column("moduleName", sa.Text(), nullable=True), + sa.Column("baseClassName", sa.Text(), nullable=True), + sa.Column("mapperKind", sa.Text(), server_default="tree", nullable=False), + sa.Column( + "schema", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.CheckConstraint("\"mapperKind\" IN ('tree', 'flat_set', 'scalar', 'pointer')", name="ck_scipion_object_types_mapper_kind"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("className"), + ) + op.create_index("idx_scipion_object_types_className", "scipion_object_types", ["className"]) + op.create_index("idx_scipion_object_types_mapperKind", "scipion_object_types", ["mapperKind"]) + op.create_index( + "idx_scipion_object_types_schema_gin", + "scipion_object_types", + ["schema"], + postgresql_using="gin", + ) + op.create_index(op.f("ix_scipion_object_types_id"), "scipion_object_types", ["id"]) + + op.create_table( + "scipion_object_type_properties", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("typeId", sa.Integer(), nullable=False), + sa.Column("propertyPath", sa.Text(), nullable=False), + sa.Column("className", sa.Text(), nullable=True), + sa.Column("valueKind", sa.Text(), nullable=True), + sa.Column("isPointer", sa.Boolean(), server_default="false", nullable=False), + sa.Column("isNested", sa.Boolean(), server_default="false", nullable=False), + sa.Column( + "schema", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["typeId"], ["scipion_object_types.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("typeId", "propertyPath", name="ux_scipion_object_type_properties_type_path"), + ) + op.create_index("idx_scipion_object_type_properties_path", "scipion_object_type_properties", ["propertyPath"]) + op.create_index( + "idx_scipion_object_type_properties_schema_gin", + "scipion_object_type_properties", + ["schema"], + postgresql_using="gin", + ) + op.create_index(op.f("ix_scipion_object_type_properties_id"), "scipion_object_type_properties", ["id"]) + + op.create_table( + "scipion_objects", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("projectId", sa.Integer(), nullable=False), + sa.Column("protocolDbId", sa.Integer(), nullable=True), + sa.Column("scipionObjId", sa.Integer(), nullable=False), + sa.Column("parentObjectId", sa.Integer(), nullable=True), + sa.Column("name", sa.Text(), nullable=True), + sa.Column("path", sa.Text(), nullable=False), + sa.Column("className", sa.Text(), nullable=False), + sa.Column("value", sa.Text(), nullable=True), + sa.Column("label", sa.Text(), nullable=True), + sa.Column("comment", sa.Text(), nullable=True), + sa.Column("creation", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "metadata", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["parentObjectId"], ["scipion_objects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["projectId"], ["projects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["protocolDbId"], ["protocols.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("projectId", "protocolDbId", "path", name="ux_scipion_objects_project_protocol_path"), + sa.UniqueConstraint( + "projectId", + "protocolDbId", + "scipionObjId", + name="ux_scipion_objects_project_protocol_obj", + ), + ) + op.create_index("idx_scipion_objects_metadata_gin", "scipion_objects", ["metadata"], postgresql_using="gin") + op.create_index("idx_scipion_objects_parent", "scipion_objects", ["parentObjectId"]) + op.create_index("idx_scipion_objects_project_class", "scipion_objects", ["projectId", "className"]) + op.create_index("idx_scipion_objects_project_protocol", "scipion_objects", ["projectId", "protocolDbId"]) + op.create_index(op.f("ix_scipion_objects_id"), "scipion_objects", ["id"]) + + op.create_table( + "scipion_object_relations", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("projectId", sa.Integer(), nullable=False), + sa.Column("creatorObjectId", sa.Integer(), nullable=False), + sa.Column("parentObjectId", sa.Integer(), nullable=False), + sa.Column("childObjectId", sa.Integer(), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("parentExtended", sa.Text(), nullable=True), + sa.Column("childExtended", sa.Text(), nullable=True), + sa.Column( + "metadata", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("creation", sa.DateTime(timezone=True), nullable=True), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["childObjectId"], ["scipion_objects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["creatorObjectId"], ["scipion_objects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["parentObjectId"], ["scipion_objects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["projectId"], ["projects.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("idx_scipion_object_relations_child", "scipion_object_relations", ["childObjectId"]) + op.create_index( + "idx_scipion_object_relations_metadata_gin", + "scipion_object_relations", + ["metadata"], + postgresql_using="gin", + ) + op.create_index("idx_scipion_object_relations_parent", "scipion_object_relations", ["parentObjectId"]) + op.create_index("idx_scipion_object_relations_project_name", "scipion_object_relations", ["projectId", "name"]) + op.create_index(op.f("ix_scipion_object_relations_id"), "scipion_object_relations", ["id"]) + + op.create_table( + "scipion_sets", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("projectId", sa.Integer(), nullable=False), + sa.Column("protocolDbId", sa.Integer(), nullable=True), + sa.Column("objectId", sa.Integer(), nullable=True), + sa.Column("outputName", sa.Text(), nullable=False), + sa.Column("setClassName", sa.Text(), nullable=False), + sa.Column("itemClassName", sa.Text(), nullable=False), + sa.Column( + "properties", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["objectId"], ["scipion_objects.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["projectId"], ["projects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["protocolDbId"], ["protocols.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("projectId", "protocolDbId", "outputName", name="ux_scipion_sets_project_protocol_output"), + ) + op.create_index("idx_scipion_sets_project_protocol", "scipion_sets", ["projectId", "protocolDbId"]) + op.create_index("idx_scipion_sets_properties_gin", "scipion_sets", ["properties"], postgresql_using="gin") + op.create_index(op.f("ix_scipion_sets_id"), "scipion_sets", ["id"]) + + op.create_table( + "scipion_set_columns", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("setId", sa.Integer(), nullable=False), + sa.Column("labelProperty", sa.Text(), nullable=False), + sa.Column("columnName", sa.Text(), nullable=False), + sa.Column("className", sa.Text(), nullable=True), + sa.Column("valueType", sa.Text(), nullable=True), + sa.Column("position", sa.Integer(), nullable=False), + sa.Column("indexed", sa.Boolean(), server_default="false", nullable=False), + sa.ForeignKeyConstraint(["setId"], ["scipion_sets.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("setId", "columnName", name="ux_scipion_set_columns_set_column"), + sa.UniqueConstraint("setId", "labelProperty", name="ux_scipion_set_columns_set_label"), + ) + op.create_index("idx_scipion_set_columns_label", "scipion_set_columns", ["labelProperty"]) + op.create_index(op.f("ix_scipion_set_columns_id"), "scipion_set_columns", ["id"]) + + op.create_table( + "scipion_set_items", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("setId", sa.Integer(), nullable=False), + sa.Column("scipionItemId", sa.Integer(), nullable=False), + sa.Column("enabled", sa.Boolean(), server_default="true", nullable=False), + sa.Column("label", sa.Text(), nullable=True), + sa.Column("comment", sa.Text(), nullable=True), + sa.Column("creation", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "values", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["setId"], ["scipion_sets.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("setId", "scipionItemId", name="ux_scipion_set_items_set_item"), + ) + op.create_index("idx_scipion_set_items_set", "scipion_set_items", ["setId"]) + op.create_index("idx_scipion_set_items_values_gin", "scipion_set_items", ["values"], postgresql_using="gin") + op.create_index(op.f("ix_scipion_set_items_id"), "scipion_set_items", ["id"]) + + op.create_table( + "scipion_set_properties", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("setId", sa.Integer(), nullable=False), + sa.Column("key", sa.Text(), nullable=False), + sa.Column("value", sa.Text(), nullable=True), + sa.ForeignKeyConstraint(["setId"], ["scipion_sets.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("setId", "key", name="ux_scipion_set_properties_set_key"), + ) + op.create_index("idx_scipion_set_properties_key", "scipion_set_properties", ["key"]) + op.create_index(op.f("ix_scipion_set_properties_id"), "scipion_set_properties", ["id"]) + + +def downgrade() -> None: + op.drop_table("scipion_set_properties") + op.drop_table("scipion_set_items") + op.drop_table("scipion_set_columns") + op.drop_table("scipion_sets") + op.drop_table("scipion_object_relations") + op.drop_table("scipion_objects") + op.drop_table("scipion_object_type_properties") + op.drop_table("scipion_object_types") \ No newline at end of file diff --git a/alembic/versions/2fc2cd4da2e2_add_protocol_input_refs.py b/alembic/versions/2fc2cd4da2e2_add_protocol_input_refs.py new file mode 100644 index 00000000..1801356e --- /dev/null +++ b/alembic/versions/2fc2cd4da2e2_add_protocol_input_refs.py @@ -0,0 +1,89 @@ +"""add protocol input refs + +Revision ID: 2fc2cd4da2e2 +Revises: 33ffae69565b +Create Date: 2026-06-22 12:06:28.813389 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '2fc2cd4da2e2' +down_revision: Union[str, None] = '33ffae69565b' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "protocol_input_refs", + sa.Column("projectId", sa.Integer(), nullable=False), + sa.Column("protocolDbId", sa.Integer(), nullable=False), + sa.Column("protocolId", sa.Text(), nullable=False), + sa.Column("inputName", sa.Text(), nullable=False), + sa.Column("itemIndex", sa.Integer(), nullable=False, server_default="0"), + sa.Column("parentProtocolDbId", sa.Integer(), nullable=True), + sa.Column("parentProtocolId", sa.Text(), nullable=True), + sa.Column("parentOutputName", sa.Text(), nullable=True), + sa.Column("objectClassName", sa.Text(), nullable=True), + sa.Column("objectId", sa.Text(), nullable=True), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint( + "projectId", + "protocolDbId", + "inputName", + "itemIndex", + name="protocol_input_refs_pkey", + ), + sa.ForeignKeyConstraint( + ["projectId"], + ["projects.id"], + name="protocol_input_refs_projectId_fkey", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["projectId", "protocolDbId"], + ["protocols.projectId", "protocols.id"], + name="protocol_input_refs_protocolDbId_fkey", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["projectId", "parentProtocolDbId"], + ["protocols.projectId", "protocols.id"], + name="protocol_input_refs_parentProtocolDbId_fkey", + ondelete="CASCADE", + ), + ) + + op.create_index( + "idx_protocol_input_refs_protocol", + "protocol_input_refs", + ["projectId", "protocolDbId"], + unique=False, + ) + + op.create_index( + "idx_protocol_input_refs_parent", + "protocol_input_refs", + ["projectId", "parentProtocolDbId", "parentOutputName"], + unique=False, + ) + + op.create_index( + "idx_protocol_input_refs_parent_protocol_id", + "protocol_input_refs", + ["projectId", "parentProtocolId", "parentOutputName"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index("idx_protocol_input_refs_parent_protocol_id", table_name="protocol_input_refs") + op.drop_index("idx_protocol_input_refs_parent", table_name="protocol_input_refs") + op.drop_index("idx_protocol_input_refs_protocol", table_name="protocol_input_refs") + op.drop_table("protocol_input_refs") diff --git a/alembic/versions/33ffae69565b_add_scipion_set_logical_tables.py b/alembic/versions/33ffae69565b_add_scipion_set_logical_tables.py new file mode 100644 index 00000000..27b34ab4 --- /dev/null +++ b/alembic/versions/33ffae69565b_add_scipion_set_logical_tables.py @@ -0,0 +1,117 @@ +"""add scipion set logical tables + +Revision ID: 33ffae69565b +Revises: c3d2b8f4a901 +Create Date: 2026-06-19 22:01:10.327665 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '33ffae69565b' +down_revision: Union[str, None] = 'c3d2b8f4a901' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "scipion_set_tables", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("setId", sa.Integer(), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("alias", sa.Text(), nullable=True), + sa.Column("tableKind", sa.Text(), server_default="root", nullable=False), + sa.Column("parentTableId", sa.Integer(), nullable=True), + sa.Column("parentItemId", sa.Integer(), nullable=True), + sa.Column("itemClassName", sa.Text(), nullable=True), + sa.Column( + "properties", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.CheckConstraint( + "\"tableKind\" IN ('root', 'child', 'properties')", + name="ck_scipion_set_tables_table_kind", + ), + sa.ForeignKeyConstraint(["setId"], ["scipion_sets.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["parentTableId"], ["scipion_set_tables.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("setId", "name", name="ux_scipion_set_tables_set_name"), + ) + op.create_index("idx_scipion_set_tables_set", "scipion_set_tables", ["setId"]) + op.create_index("idx_scipion_set_tables_parent", "scipion_set_tables", ["parentTableId"]) + op.create_index("idx_scipion_set_tables_properties_gin", "scipion_set_tables", ["properties"], postgresql_using="gin") + + op.create_table( + "scipion_set_table_columns", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("tableId", sa.Integer(), nullable=False), + sa.Column("labelProperty", sa.Text(), nullable=False), + sa.Column("columnName", sa.Text(), nullable=False), + sa.Column("className", sa.Text(), nullable=True), + sa.Column("valueType", sa.Text(), nullable=True), + sa.Column("position", sa.Integer(), nullable=False), + sa.Column("indexed", sa.Boolean(), server_default="false", nullable=False), + sa.Column( + "properties", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.ForeignKeyConstraint(["tableId"], ["scipion_set_tables.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("tableId", "labelProperty", name="ux_scipion_set_table_columns_table_label"), + sa.UniqueConstraint("tableId", "columnName", name="ux_scipion_set_table_columns_table_column"), + ) + op.create_index("idx_scipion_set_table_columns_table", "scipion_set_table_columns", ["tableId"]) + op.create_index("idx_scipion_set_table_columns_label", "scipion_set_table_columns", ["labelProperty"]) + + op.create_table( + "scipion_set_table_items", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("tableId", sa.Integer(), nullable=False), + sa.Column("scipionItemId", sa.Integer(), nullable=False), + sa.Column("parentItemId", sa.Integer(), nullable=True), + sa.Column("enabled", sa.Boolean(), server_default="true", nullable=False), + sa.Column("label", sa.Text(), nullable=True), + sa.Column("comment", sa.Text(), nullable=True), + sa.Column("creation", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "values", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["tableId"], ["scipion_set_tables.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("tableId", "scipionItemId", name="ux_scipion_set_table_items_table_item"), + ) + op.create_index("idx_scipion_set_table_items_table", "scipion_set_table_items", ["tableId"]) + op.create_index("idx_scipion_set_table_items_parent", "scipion_set_table_items", ["parentItemId"]) + op.create_index("idx_scipion_set_table_items_values_gin", "scipion_set_table_items", ["values"], postgresql_using="gin") + + +def downgrade() -> None: + op.drop_index("idx_scipion_set_table_items_values_gin", table_name="scipion_set_table_items") + op.drop_index("idx_scipion_set_table_items_parent", table_name="scipion_set_table_items") + op.drop_index("idx_scipion_set_table_items_table", table_name="scipion_set_table_items") + op.drop_table("scipion_set_table_items") + + op.drop_index("idx_scipion_set_table_columns_label", table_name="scipion_set_table_columns") + op.drop_index("idx_scipion_set_table_columns_table", table_name="scipion_set_table_columns") + op.drop_table("scipion_set_table_columns") + + op.drop_index("idx_scipion_set_tables_properties_gin", table_name="scipion_set_tables") + op.drop_index("idx_scipion_set_tables_parent", table_name="scipion_set_tables") + op.drop_index("idx_scipion_set_tables_set", table_name="scipion_set_tables") + op.drop_table("scipion_set_tables") \ No newline at end of file diff --git a/alembic/versions/c3d2b8f4a901_allow_nullable_scipion_object_ids.py b/alembic/versions/c3d2b8f4a901_allow_nullable_scipion_object_ids.py new file mode 100644 index 00000000..caf1f078 --- /dev/null +++ b/alembic/versions/c3d2b8f4a901_allow_nullable_scipion_object_ids.py @@ -0,0 +1,38 @@ +"""allow nullable scipion object ids + +Revision ID: c3d2b8f4a901 +Revises: 1f8e5b400102 +Create Date: 2026-06-19 10:15:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "c3d2b8f4a901" +down_revision: Union[str, None] = "1f8e5b400102" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column( + "scipion_objects", + "scipionObjId", + existing_type=sa.Integer(), + nullable=True, + ) + op.execute('UPDATE scipion_objects SET "scipionObjId" = NULL WHERE "scipionObjId" < 0') + + +def downgrade() -> None: + op.execute('UPDATE scipion_objects SET "scipionObjId" = -id WHERE "scipionObjId" IS NULL') + op.alter_column( + "scipion_objects", + "scipionObjId", + existing_type=sa.Integer(), + nullable=False, + ) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 31cccff0..ecb79ecf 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -26,7 +26,7 @@ WorkflowImportRequest, ) from app.backend.api.schemas.tags_schema import ProtocolTagCreateIn, ProtocolTagUpdateIn, ProtocolTagsSetIn -from app.backend.database import getMapper +from app.backend.database import getMapperDependency as getMapper from app.backend.api.schemas.project_schema import (ProjectCreate, ProjectOut, ProjectUpdate, ProjectShareCreate, ApplyWorkflowToProjectRequest, TiltSeriesNewSetRequest, ProjectImportIn, ProtocolWizardExecuteResponse, @@ -50,6 +50,20 @@ def getProjectService() -> ProjectService: """Return a fresh ProjectService per request to avoid shared state.""" return ProjectService() +def _appendProtocolSyncCounts(response: Dict[str, Any], result: Any) -> Dict[str, Any]: + if not isinstance(result, dict): + return response + + protocolsCount = result.get("protocolsCount", result.get("protocols")) + dependenciesCount = result.get("dependenciesCount", result.get("dependencies")) + + if protocolsCount is not None: + response["protocolsCount"] = protocolsCount + if dependenciesCount is not None: + response["dependenciesCount"] = dependenciesCount + + return response + # ====================================================================== # PROJECT WORKFLOWS # ====================================================================== @@ -156,12 +170,20 @@ def importProject( @router.get("/{projectId}", response_model=Any) def getProject( - projectId: int, # id in the DB + projectId: int, + validateConsistency: bool = Query(False), currentUser=Depends(getCurrentUser), mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=True) + project = service.getProjectById( + mapper, + projectId, + currentUser, + refresh=True, + checkPid=True, + validateConsistency=validateConsistency, + ) if not project: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") return project @@ -189,13 +211,7 @@ def getProjectEffectiveSettings( The project must be accessible by the authenticated user. """ - project = service.getProjectById( - mapper, - projectId, - currentUser, - refresh=False, - checkPid=False, - ) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -217,6 +233,29 @@ def getProjectEffectiveSettings( detail=f"Failed to load project effective settings: {e}", ) + +@router.post( + "/{projectId}/consistency/check", + response_model=Any, + status_code=status.HTTP_200_OK, +) +def checkProjectPostgresqlConsistency( + projectId: int, + refresh: bool = Query(True), + checkPid: bool = Query(True), + currentUser=Depends(getCurrentUser), + mapper: PostgresqlFlatMapper = Depends(getMapper), + service: ProjectService = Depends(getProjectService), +): + return service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=projectId, + currentUser=currentUser, + refresh=refresh, + checkPid=checkPid, + ) + + @router.put("/{projectId}", response_model=Any, status_code=status.HTTP_200_OK) def updateProject( projectId: int, @@ -320,7 +359,7 @@ def loadProtocols( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -345,7 +384,11 @@ async def loadProtocol( if not project: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") - return service.getProtocolParams(projectId, protocolId) + return service.getProtocolParams( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) class ProtocolStepStatusUpdate(BaseModel): @@ -360,7 +403,7 @@ def listProtocolSteps( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") @@ -434,32 +477,34 @@ async def launchProtocol( return JSONResponse( status_code=status.HTTP_404_NOT_FOUND, content={ - "status": 0, + "status": 1, "errors": ["Project not found"], "workflow": [], }, ) - service.launchProtocol( + result = service.launchProtocol( mapper=mapper, projectId=projectId, protocolId=request.getProtocolId(), protocolClassName=request.getProtocolClassName(), params=request.getParams(), executeMode=request.getMode(), - ) + ) or {} - return { + response = { "status": 0, "errors": [], "workflow": [], } + return _appendProtocolSyncCounts(response, result) + except HTTPException as e: return JSONResponse( status_code=e.status_code, content={ - "status": 0, + "status": 1, "errors": _normalizeErrors(e.detail), "workflow": [], }, @@ -469,7 +514,7 @@ async def launchProtocol( return JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={ - "status": 0, + "status": 1, "errors": ["Internal server error"], "workflow": [], }, @@ -544,7 +589,30 @@ def suggestionProtocol( "errors": ["Project not found"], "workflow": []}, ) - return service.getNextProtocolSuggestions(protocolId) + try: + return service.getNextProtocolSuggestions( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + except HTTPException as e: + return JSONResponse( + status_code=e.status_code, + content={ + "status": 1, + "errors": _normalizeErrors(e.detail), + "workflow": [], + }, + ) + except Exception as e: + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={ + "status": 1, + "errors": [str(e)], + "workflow": [], + }, + ) @router.put("/{projectId}/protocols/{protocolId}/rename", response_model=Any, status_code=status.HTTP_200_OK) @@ -592,21 +660,27 @@ def renameProtocol( ) service.renameProtocol( + mapper, + projectId, protocolId, newNameText.strip(), str(newComment or "").strip(), ) - service.syncProjectGraphAfterMutation( + syncResult = service.syncProjectGraphAfterMutation( mapper, projectId, actionLabel="rename protocol", refresh=True, checkPid=True, - ) + ) or {} - return {"status": 0, - "errors": [], - "workflow": []} + response = { + "status": 0, + "errors": [], + "workflow": [], + } + + return _appendProtocolSyncCounts(response, syncResult) except HTTPException as e: return JSONResponse( @@ -651,12 +725,16 @@ def duplicateProtocol( "workflow": []}, ) - result = service.duplicateProtocol(mapper, projectId, items) + result = service.duplicateProtocol(mapper, projectId, items) or {} # Keep 201 on success, but still return unified schema - return {"status": result['status'], - "errors": result['errors'], - "workflow": [], - "duplicated": result['duplicated']} + response = { + "status": result.get("status", 0), + "errors": result.get("errors", []), + "workflow": [], + "duplicated": result.get("duplicated", []), + } + + return _appendProtocolSyncCounts(response, result) except HTTPException as e: return JSONResponse( @@ -695,24 +773,40 @@ def deleteProtocol( protocolIds = getattr(payload, "protocolIds", None) if payload is not None else None if not protocolIds: return JSONResponse( - status_code=status.HTTP_404_NOT_FOUND, + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, content={"status": 1, "errors": ["Missing protocolIds"], "workflow": []}, ) - service.deleteProtocol(mapper, projectId, protocolIds) + result = service.deleteProtocol(mapper, projectId, protocolIds) or {} - return {"status": 0, - "errors": [], - "workflow": []} + response = { + "status": 0, + "errors": [], + "workflow": [], + } + + return _appendProtocolSyncCounts(response, result) + + except HTTPException as e: + return JSONResponse( + status_code=e.status_code, + content={ + "status": 1, + "errors": _normalizeErrors(e.detail), + "workflow": [], + }, + ) except Exception as e: return JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - content={"status": 1, - "errors": _normalizeErrors(str(e)), - "workflow": []}, + content={ + "status": 1, + "errors": _normalizeErrors(str(e)), + "workflow": [], + }, ) @@ -738,18 +832,22 @@ def restartProtocolAll( ) try: - service.restartProtocolAll(protocolId) - service.syncProjectGraphAfterMutation( + service.restartProtocolAll(mapper, projectId, protocolId) + syncResult = service.syncProjectGraphAfterMutation( mapper, projectId, actionLabel="restart protocol subtree", refresh=True, checkPid=True, - ) + ) or {} - return {"status": 0, - "errors": [], - "workflow": []} + response = { + "status": 0, + "errors": [], + "workflow": [], + } + + return _appendProtocolSyncCounts(response, syncResult) except HTTPException as e: return JSONResponse( @@ -788,15 +886,21 @@ def continueProtocolAll( try: service.continueProtocolAll(mapper, projectId, protocolId, currentUser) - service.syncProjectGraphAfterMutation( + syncResult = service.syncProjectGraphAfterMutation( mapper, projectId, actionLabel="continue protocol subtree", refresh=True, checkPid=True, - ) + ) or {} - return {"status": 0, "errors": [], "workflow": []} + response = { + "status": 0, + "errors": [], + "workflow": [], + } + + return _appendProtocolSyncCounts(response, syncResult) except HTTPException as e: return JSONResponse( @@ -830,16 +934,22 @@ def resetProtocolFrom( ) try: - service.resetProtocolFrom(protocolId) - service.syncProjectGraphAfterMutation( + service.resetProtocolFrom(mapper, projectId, protocolId) + syncResult = service.syncProjectGraphAfterMutation( mapper, projectId, actionLabel="reset protocol from node", refresh=True, checkPid=True, - ) + ) or {} + + response = { + "status": 0, + "errors": [], + "workflow": [], + } - return {"status": 0, "errors": [], "workflow": []} + return _appendProtocolSyncCounts(response, syncResult) except HTTPException as e: return JSONResponse( @@ -880,18 +990,22 @@ def stopProtocol( "workflow": []}, ) - service.stopProtocol(protocolIds) - service.syncProjectGraphAfterMutation( + service.stopProtocol(mapper, projectId, protocolIds) + syncResult = service.syncProjectGraphAfterMutation( mapper, projectId, actionLabel="stop protocol", refresh=True, checkPid=True, - ) + ) or {} - return {"status": 0, - "errors": [], - "workflow": []} + response = { + "status": 0, + "errors": [], + "workflow": [], + } + + return _appendProtocolSyncCounts(response, syncResult) except HTTPException as e: return JSONResponse( @@ -1001,7 +1115,7 @@ def listProtocolLogChannels( service: ProjectService = Depends(getProjectService), ): # listProtocolLogChannels - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1009,6 +1123,8 @@ def listProtocolLogChannels( channels = service.listProtocolLogChannelsService( projectId=projectId, protocolId=protocolId, + mapper=mapper, + currentUser=currentUser, ) # normalizeChannels @@ -1051,7 +1167,7 @@ def pollProtocolLogs( service: ProjectService = Depends(getProjectService), ): # pollProtocolLogs - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1072,6 +1188,8 @@ def pollProtocolLogs( offsets=offsets, maxBytes=maxBytes, maxLines=maxLines, + mapper=mapper, + currentUser=currentUser, ) # normalizePollResponse @@ -1183,7 +1301,11 @@ async def getProtocolPath( service: ProjectService = Depends(getProjectService), ): _ensureProjectForFsRequest(projectId, protocolId, currentUser, mapper, service) - return service.getProtocolPath(protocolId) + return service.getProtocolPath( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) @router.get("/{projectId}/protocols/{protocolId}/fs/list", response_model=Any) @@ -1203,7 +1325,12 @@ async def listProtocolDir( service: ProjectService = Depends(getProjectService), ): _ensureProjectForFsRequest(projectId, protocolId, currentUser, mapper, service) - return service.listProtocolDir(protocolId, path) + return service.listProtocolDir( + protocolId=protocolId, + path=path, + mapper=mapper, + projectId=projectId, + ) @router.get("/{projectId}/protocols/{protocolId}/fs/preview2", response_model=None) @@ -1216,7 +1343,12 @@ async def previewProtocolText( service: ProjectService = Depends(getProjectService), ): _ensureProjectForFsRequest(projectId, protocolId, currentUser, mapper, service) - return service.previewProtocolTextFile(protocolId, path) + return service.previewProtocolTextFile( + protocolId=protocolId, + path=path, + mapper=mapper, + projectId=projectId, + ) @router.get("/{projectId}/protocols/{protocolId}/fs/preview", response_model=None) @@ -1237,7 +1369,12 @@ def previewRemoteEntry( refresh=False, checkPid=False, ) - return service.previewRemoteEntry(protocolId, path) + return service.previewRemoteEntry( + protocolId=protocolId, + path=path, + mapper=mapper, + projectId=projectId, + ) @router.get("/{projectId}/protocols/{protocolId}/fs/download", response_model=None) @@ -1251,7 +1388,13 @@ async def previewProtocolImageFile( service: ProjectService = Depends(getProjectService), ): _ensureProjectForFsRequest(projectId, protocolId, currentUser, mapper, service) - return service.previewProtocolImageFile(protocolId, path, inline) + return service.previewProtocolImageFile( + protocolId=protocolId, + path=path, + inline=inline, + mapper=mapper, + projectId=projectId, + ) @router.post( @@ -1406,6 +1549,8 @@ async def writeRemoteFile( return service.writeRemoteFileService( protocolId=protocolId, payload=payload, + mapper=mapper, + projectId=projectId, ) except HTTPException: raise @@ -1445,6 +1590,8 @@ async def previewOutput( outputName=outputName, requestHeaders=dict(request.headers), colormap=cmapHeader or cmapQuery, + mapper=mapper, + projectId=projectId, ) @@ -1468,7 +1615,7 @@ def resolveAnalyzeViewer( service: ProjectService = Depends(getProjectService), ): # resolveAnalyzeViewer - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1481,7 +1628,7 @@ def resolveAnalyzeViewer( projectId=projectId, protocolId=protocolId, ctx=payload, - + mapper=mapper, ) return decision or {"handled": False} except Exception as e: @@ -1506,11 +1653,14 @@ def listOutputVolumes( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") - items = service.listOutputVolumesService(projectId, protocolId, outputName) + items = service.listOutputVolumesService(projectId, + protocolId, + outputName, + mapper=mapper) from fastapi.responses import JSONResponse resp = JSONResponse(items) @@ -1535,11 +1685,15 @@ def getVolumeInfo( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") - info = service.getVolumeInfoService(projectId, protocolId, outputName, volumeId) + info = service.getVolumeInfoService(projectId, + protocolId, + outputName, + volumeId, + mapper=mapper,) from fastapi.responses import JSONResponse resp = JSONResponse(info) @@ -1572,7 +1726,7 @@ def getVolumeHistogram( """ Return intensity histogram for one volume. """ - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1582,6 +1736,7 @@ def getVolumeHistogram( outputName=outputName, volumeId=volumeId, bins=bins, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -1608,7 +1763,7 @@ def renderVolumeSlice( colormapParam: Optional[str] = Query(None, alias="colormap"), formatParam: Optional[str] = Query(None, alias="format"), fmtParam: Optional[str] = Query(None, alias="fmt"), - normalize: Optional[str] = Query(None), + normalize: Optional[str] = Query("minmax"), scale: float = Query(1.0, gt=0), inline: bool = Query(True), thumb: Optional[int] = Query(None, ge=32, le=2048), @@ -1618,11 +1773,11 @@ def renderVolumeSlice( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") - cmap = cmapParam or colormapParam or "viridis" + cmap = cmapParam or colormapParam fmt = fmtParam or formatParam or "webp" resp = service.renderVolumeSliceService( @@ -1640,6 +1795,7 @@ def renderVolumeSlice( thumb=thumb, fast=fast, quality=quality, + mapper=mapper, ) resp.headers["X-Debug-Auth"] = "ok" resp.headers["X-Debug-UserId"] = str(getattr(currentUser, "id", currentUser.get("id", ""))) @@ -1662,7 +1818,7 @@ def getVolumeData3d( currentUser: Dict[str, Any] = Depends(getCurrentUser), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1673,6 +1829,7 @@ def getVolumeData3d( volumeId=volumeId, maxDim=maxDim, method=method, + mapper=mapper, ) @router.get( @@ -1694,20 +1851,22 @@ def getVolumeSurfaceMesh( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") try: - return service.getVolumeSurfaceMesh(protocolId=protocolId, + return service.getVolumeSurfaceMesh(projectId=projectId, + protocolId=protocolId, outputName=outputName, volumeId=volumeId, level=level, maxDim=maxDim, method=method, maxTriangles=maxTriangles, - currentUser=currentUser) + currentUser=currentUser, + mapper=mapper,) except HTTPException: raise @@ -1745,7 +1904,7 @@ def listOutputTiltSeries( """ List tilt series for a SetOfTiltSeries-like output. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1753,6 +1912,7 @@ def listOutputTiltSeries( projectId=projectId, protocolId=protocolId, outputName=outputName, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -1781,7 +1941,7 @@ def getTiltSeriesFrames( """ Return metadata for all tilt images in one tilt series. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1791,6 +1951,7 @@ def getTiltSeriesFrames( protocolId=protocolId, outputName=outputName, tiltSeriesId=tiltSeriesId, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -1848,7 +2009,7 @@ def renderTiltSeriesImage( """ Render one tilt image from a tilt series. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1863,6 +2024,7 @@ def renderTiltSeriesImage( fmt=fmt, applyTransform=applyTransform, inline=inline, + mapper=mapper, ) resp.headers["X-Debug-Auth"] = "ok" @@ -1895,7 +2057,7 @@ def renderTiltSeriesImagesBatch( Render several tilt images from the same tilt series in one request. This is intended for smooth slider prefetching in the web viewer. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1910,6 +2072,7 @@ def renderTiltSeriesImagesBatch( fmt=payload.fmt, applyTransform=payload.applyTransform, inline=payload.inline, + mapper=mapper ) resp = JSONResponse(result or {}) @@ -1942,14 +2105,14 @@ def createNewSetOfTiltSeries( ): """ Create a new tilt-series output applying the given exclusions. - The backend is expected to duplicate the SetOfTiltSeries and - remove excluded views, optionally restacking files on disk. + The backend duplicates the SetOfTiltSeries and removes excluded views, + optionally restacking files on disk. """ project = service.getProjectById( mapper, projectId, currentUser, - refresh=False, + refresh=True, checkPid=False, ) if not project: @@ -1962,6 +2125,7 @@ def createNewSetOfTiltSeries( outputName=outputName, exclusions=payload.exclusions, restack=payload.restack, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -1982,10 +2146,10 @@ def createNewSetOfTiltSeries( detail=f"Failed to create new tilt-series set: {e}", ) - # ============================================================================== # ANALYZE RESULTS: CTF TOMOGRAPHY (SetOfCTFTomoSeries) # ============================================================================== + class CtftomoNewSetRequest(BaseModel): """ Request payload for creating a new SetOfCTFTomoSeries based on exclusions. @@ -2010,11 +2174,16 @@ def listCtftomoSeries( """ List CTFTomoSeries entries for a CTFTomo output. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") - return service.listOutputCtftomoSeriesService(projectId, protocolId, outputName) + return service.listOutputCtftomoSeriesService( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + ) @router.get( @@ -2033,11 +2202,17 @@ def getCtftomoSeriesViews( """ Return all CTF measurements for one tilt-series (identified by tiltSeriesId). """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") - return service.getCtftomoSeriesViewsService(projectId, protocolId, outputName, tiltSeriesId) + return service.getCtftomoSeriesViewsService( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + tiltSeriesId=tiltSeriesId, + mapper=mapper, + ) @router.post( @@ -2076,6 +2251,7 @@ def createNewSetOfCtftomoSeries( outputName=outputName, exclusions=payload.exclusions, restack=payload.restack, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -2136,13 +2312,7 @@ def renderCtftomoPsdImage( Render a PSD image for a CTF-tomography view, given a stack spec (for example '3@/path/to/TS_1.mrc'). """ - project = service.getProjectById( - mapper, - projectId, - currentUser, - refresh=False, - checkPid=False, - ) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2156,6 +2326,7 @@ def renderCtftomoPsdImage( fmt=fmt, applyTransform=applyTransform, inline=inline, + mapper=mapper, ) resp.headers["X-Debug-Auth"] = "ok" resp.headers["X-Debug-UserId"] = str( @@ -2163,6 +2334,8 @@ def renderCtftomoPsdImage( ) resp.headers["Vary"] = "Authorization" return resp + except HTTPException: + raise except Exception as e: logger.exception("Error in renderCtftomoPsdImage: %s", e) raise HTTPException( @@ -2191,7 +2364,7 @@ def listCoordinates3dTomograms( """ List tomograms referenced by a SetOfCoordinates3D output. """ - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2199,6 +2372,7 @@ def listCoordinates3dTomograms( projectId=projectId, protocolId=protocolId, outputName=outputName, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -2229,7 +2403,7 @@ def getCoordinates3dPoints( """ Return all 3D coordinates for one tomogram inside a SetOfCoordinates3D. """ - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2238,6 +2412,7 @@ def getCoordinates3dPoints( protocolId=protocolId, outputName=outputName, tomogramId=tomogramId, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -2310,7 +2485,7 @@ def renderCoords3dTomogramSlice( """ Render a 2D slice from a tomogram referenced by a SetOfCoordinates3D. """ - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2329,6 +2504,7 @@ def renderCoords3dTomogramSlice( thumb=thumb, fast=fast, quality=quality, + mapper=mapper ) resp.headers["X-Debug-Auth"] = "ok" @@ -2378,7 +2554,8 @@ def createCoords3dOutputFromPoints(projectId: int, result = service.createCoords3dOutputFromPointsService(projectId=projectId, protocolId=protocolId, outputName=outputName, - payload=payload) + payload=payload, + mapper=mapper,) resp = JSONResponse(result or {"success": True, "outputName": result['outputName']}) resp.headers["X-Debug-Auth"] = "ok" resp.headers["X-Debug-UserId"] = str(getattr(currentUser, "id", currentUser.get("id", ""))) @@ -2403,7 +2580,7 @@ def getIntegratedAnalyzeContext( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2411,6 +2588,7 @@ def getIntegratedAnalyzeContext( projectId=projectId, protocolId=protocolId, outputName=outputName, + mapper=mapper, ) resp = JSONResponse(payload) @@ -2455,13 +2633,7 @@ def getFscRows( "threshold": 0.143, } """ - project = service.getProjectById( - mapper, - projectId, - currentUser, - refresh=False, - checkPid=False, - ) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2470,6 +2642,7 @@ def getFscRows( projectId=projectId, protocolId=protocolId, outputName=outputName, + mapper=mapper, ) resp = JSONResponse(payload or {"curves": [], "threshold": 0.143}) @@ -2520,7 +2693,7 @@ def listOutputMetadataTables( """ List logical metadata tables (blocks) associated with a given output. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2528,6 +2701,7 @@ def listOutputMetadataTables( projectId=projectId, protocolId=protocolId, outputName=outputName, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -2556,7 +2730,7 @@ def getMetadataTableSchema( """ Return logical schema for one metadata table: columns, renderers, flags. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2565,6 +2739,7 @@ def getMetadataTableSchema( protocolId=protocolId, outputName=outputName, tableName=tableName, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -2591,7 +2766,7 @@ def runMetadataTableAction( service: ProjectService = Depends(getProjectService), ): # runMetadataTableAction - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2684,7 +2859,7 @@ def getMetadataTablePage( """ Return one logical page of rows for a metadata table. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2698,6 +2873,7 @@ def getMetadataTablePage( sortBy=sortBy, asc=asc, selectionOnly=selectionOnly, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -2742,7 +2918,7 @@ def exportMetadataTable( """ Export a metadata table (full or subset) as CSV/XLSX. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2764,6 +2940,7 @@ def exportMetadataTable( fmt=fmt, selectionOnly=selectionOnly, ids=idList, + mapper=mapper, ) resp.headers["X-Debug-Auth"] = "ok" resp.headers["X-Debug-UserId"] = str(getattr(currentUser, "id", currentUser.get("id", ""))) @@ -2827,7 +3004,7 @@ def renderMetadataImageCell( """ Render one image cell from a metadata table using the same logic as ImageRenderer. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2843,6 +3020,7 @@ def renderMetadataImageCell( applyTransform=applyTransform, inline=inline, fmt=fmt, + mapper=mapper, ) resp.headers["X-Debug-Auth"] = "ok" resp.headers["X-Debug-UserId"] = str(getattr(currentUser, "id", currentUser.get("id", ""))) @@ -2889,7 +3067,7 @@ def getMetadataTableWindow( """ Return a window (offset + limit) of rows for a metadata table. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") @@ -2903,6 +3081,7 @@ def getMetadataTableWindow( selectionOnly=selectionOnly, sortBy=sortBy, asc=asc, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -2916,7 +3095,6 @@ def getMetadataTableWindow( # ====================================================================== # ANALYZE RESULTS: EXTERNAL VIEWERS # ====================================================================== - @router.get( "/{projectId}/protocols/{protocolId}/outputs/{outputName}/external-viewers", response_model=Any, @@ -2926,7 +3104,7 @@ def listExternalViewers( projectId: int, protocolId: int, outputName: str, - objectId: Optional[str] = Query(None), + objectId: Optional[Union[str, int]] = Query(None), objectKind: Optional[str] = Query(None), currentUser=Depends(getCurrentUser), mapper: PostgresqlFlatMapper = Depends(getMapper), @@ -2936,7 +3114,7 @@ def listExternalViewers( mapper, projectId, currentUser, - refresh=False, + refresh=True, checkPid=False, ) if not project: @@ -2948,8 +3126,12 @@ def listExternalViewers( outputName=outputName, objectId=objectId, objectKind=objectKind, + mapper=mapper, + projectId=projectId, ) - return {"viewers": viewers} + + return {"viewers": viewers or []} + except HTTPException: raise except Exception as e: @@ -2979,7 +3161,7 @@ def launchExternalViewer( mapper, projectId, currentUser, - refresh=False, + refresh=True, checkPid=False, ) if not project: @@ -2995,7 +3177,10 @@ def launchExternalViewer( objectId=payload.objectId, objectKind=payload.objectKind, params=payload.params or {}, + mapper=mapper, + projectId=projectId, ) + except HTTPException: raise except Exception as e: @@ -3004,7 +3189,6 @@ def launchExternalViewer( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to launch external viewer: {e}", ) - # ====================================================================== # PROTOCOL TAGS # ====================================================================== @@ -3022,7 +3206,7 @@ def listProjectTags( service: ProjectService = Depends(getProjectService), ): # ensureProjectExists - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -3042,7 +3226,7 @@ def createProjectTag( service: ProjectService = Depends(getProjectService), ): # ensureProjectExists - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -3074,7 +3258,7 @@ def updateProjectTag( service: ProjectService = Depends(getProjectService), ): # ensureProjectExists - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -3100,7 +3284,7 @@ def deleteProjectTag( service: ProjectService = Depends(getProjectService), ): # ensureProjectExists - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -3121,7 +3305,7 @@ def listProtocolTags( service: ProjectService = Depends(getProjectService), ): # ensureProjectExists - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -3147,7 +3331,7 @@ def setProtocolTags( service: ProjectService = Depends(getProjectService), ): # ensureProjectExists - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -3171,7 +3355,7 @@ async def getContextMenuVisibilityPolicy( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") return service.getContextMenuVisibilityPolicy() @@ -3393,12 +3577,16 @@ def buildThumbnail(): outputName=outputName, force=False, size=size, + mapper=mapper, + projectId=projectId, ) return service.buildProtocolThumbnail( protocolId=protocolId, force=False, size=size, + mapper=mapper, + projectId=projectId, ) result = _runThumbnailProjectJob( @@ -3453,12 +3641,16 @@ def buildThumbnail(): outputName=outputName, force=True, size=size, + mapper=mapper, + projectId=projectId, ) return service.buildProtocolThumbnail( protocolId=protocolId, force=True, size=size, + mapper=mapper, + projectId=projectId, ) result = _runThumbnailProjectJob( @@ -3517,59 +3709,8 @@ def listProjectThumbnailItems( maxProtocols=maxProtocols, maxOutputsPerProtocol=maxOutputsPerProtocol, inlineImages=inlineImages, + mapper=mapper, ) - - validItems = [] - - for group in items or []: - if not isinstance(group, dict): - continue - - protocolIdValue = group.get("protocolId") - - try: - protocol = service.currentProject.getProtocol(int(protocolIdValue)) - except Exception: - logger.warning( - "Skipping thumbnail group because protocol was not found. " - "projectId=%s protocolId=%s group=%s", - projectId, - protocolIdValue, - group, - ) - continue - - validOutputs = [] - - for output in group.get("outputs") or []: - if not isinstance(output, dict): - continue - - outputNameValue = output.get("outputName") - if not outputNameValue: - continue - - if not hasattr(protocol, str(outputNameValue)): - logger.warning( - "Skipping thumbnail output because it does not belong to protocol. " - "projectId=%s protocolId=%s outputName=%s", - projectId, - protocolIdValue, - outputNameValue, - ) - continue - - validOutputs.append(output) - - if not validOutputs: - continue - - nextGroup = dict(group) - nextGroup["outputs"] = validOutputs - validItems.append(nextGroup) - - items = validItems - response = JSONResponse(items) response.headers["Cache-Control"] = "private, max-age=60, stale-while-revalidate=300" response.headers["Access-Control-Expose-Headers"] = "Cache-Control" @@ -3615,6 +3756,8 @@ def getProtocolOutputThumbnail( outputName=outputName, force=False, size=size, + mapper=mapper, + projectId=projectId, ) thumbPath = result.get("absolutePath") @@ -3689,25 +3832,25 @@ def getProtocolOutputThumbnailsBatch( seen = set() for requestedOutput in requestedOutputs: - protocolId = int(requestedOutput.protocolId) + requestedProtocolId = int(requestedOutput.protocolId) outputName = str(requestedOutput.outputName or "").strip() if not outputName: continue - requestKey = (protocolId, outputName) + requestKey = (requestedProtocolId, outputName) if requestKey in seen: continue seen.add(requestKey) item = { - "protocolId": protocolId, + "protocolId": requestedProtocolId, "outputName": outputName, "outputClassName": None, "exists": False, "cached": False, "thumbnailUrl": ( - f"/projects/{int(projectId)}/protocols/{protocolId}" + f"/projects/{int(projectId)}/protocols/{requestedProtocolId}" f"/outputs/{outputName}/thumbnail" ), "thumbnailDataUrl": None, @@ -3715,17 +3858,17 @@ def getProtocolOutputThumbnailsBatch( } try: - protocol = service.currentProject.getProtocol(protocolId) + scipionProtocolId = service._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=requestedProtocolId, + ) + protocol = service.currentProject.getProtocol(int(scipionProtocolId)) except Exception: item["error"] = "Protocol not found" items.append(item) continue - if not hasattr(protocol, outputName): - item["error"] = "Output not found" - items.append(item) - continue - try: outputObject = getattr(protocol, outputName) item["outputClassName"] = outputObject.__class__.__name__ @@ -3734,16 +3877,18 @@ def getProtocolOutputThumbnailsBatch( try: result = service.buildProtocolOutputThumbnail( - protocolId=protocolId, + protocolId=requestedProtocolId, outputName=outputName, force=False, size=payload.size, + mapper=mapper, + projectId=projectId, ) except Exception as exc: logger.debug( "Failed building batch protocol output thumbnail. projectId=%s protocolId=%s outputName=%s", projectId, - protocolId, + requestedProtocolId, outputName, exc_info=True, ) @@ -3770,7 +3915,7 @@ def getProtocolOutputThumbnailsBatch( logger.debug( "Could not inline batch thumbnail image. projectId=%s protocolId=%s outputName=%s", projectId, - protocolId, + requestedProtocolId, outputName, exc_info=True, ) diff --git a/app/backend/api/routers/protocol_router.py b/app/backend/api/routers/protocol_router.py index 3d264448..ec8f7984 100644 --- a/app/backend/api/routers/protocol_router.py +++ b/app/backend/api/routers/protocol_router.py @@ -30,11 +30,13 @@ from app.backend.api.dependencies import getCurrentUser from app.backend.database import getMapper from app.backend.api.services.project_service import ProjectService -from app.backend.models.protocol_model import ProtocolRequest from app.backend.mapper.postgresql import PostgresqlFlatMapper router = APIRouter(prefix="/protocols", tags=["protocols"]) -service = ProjectService() + + +def getProjectService() -> ProjectService: + return ProjectService() @router.get("/{projectId}/{protocolId}", response_model=Any) @@ -42,13 +44,18 @@ async def loadProtocol( projectId: int, protocolId: int, currentUser=Depends(getCurrentUser), - mapper: PostgresqlFlatMapper = Depends(getMapper) + mapper: PostgresqlFlatMapper = Depends(getMapper), + service: ProjectService = Depends(getProjectService), ): project = service.getProjectById(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") - return service.getProtocolParams(projectId, protocolId) + return service.getProtocolParams( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) @router.get("/{projectId}/protclass/{protClassName}", response_model=Any) @@ -56,7 +63,8 @@ async def loadNewProtocol( projectId: int, protClassName: str, currentUser=Depends(getCurrentUser), - mapper: PostgresqlFlatMapper = Depends(getMapper) + mapper: PostgresqlFlatMapper = Depends(getMapper), + service: ProjectService = Depends(getProjectService), ): project = service.getProjectById(mapper, projectId, currentUser) if not project: @@ -65,26 +73,26 @@ async def loadNewProtocol( return service.getNewProtocolParams(projectId, protClassName) -@router.post("/launch", response_model=Any) -async def launchProtocol(request: ProtocolRequest): - try: - protocolId = request.getProtocolId() - protocolClassName = request.getProtocolClassName() - params = request.getParams() - service.launchProtocol(protocolId, protocolClassName, params) - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@router.post("/save", response_model=Any) -async def saveProtocol(request: ProtocolRequest): - try: - protocolId = request.getProtocolId() - protocolClassName = request.getProtocolClassName() - params = request.getParams() - service.saveProtocol(protocolId, protocolClassName, params) - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) +# @router.post("/launch", response_model=Any) +# async def launchProtocol(request: ProtocolRequest): +# try: +# protocolId = request.getProtocolId() +# protocolClassName = request.getProtocolClassName() +# params = request.getParams() +# service.launchProtocol(protocolId, protocolClassName, params) +# except Exception as e: +# raise HTTPException(status_code=500, detail=str(e)) +# +# +# @router.post("/save", response_model=Any) +# async def saveProtocol(request: ProtocolRequest): +# try: +# protocolId = request.getProtocolId() +# protocolClassName = request.getProtocolClassName() +# params = request.getParams() +# service.saveProtocol(protocolId, protocolClassName, params) +# except Exception as e: +# raise HTTPException(status_code=500, detail=str(e)) @router.get("/logs/{projectId}/{protocolId}/{offset}/{errOffset}/{scheduleOffset}", response_model=Any) @@ -95,10 +103,18 @@ async def getProtocolLogs( errOffset: int = 0, scheduleOffset: int = 0, currentUser=Depends(getCurrentUser), - mapper: PostgresqlFlatMapper = Depends(getMapper) + mapper: PostgresqlFlatMapper = Depends(getMapper), + service: ProjectService = Depends(getProjectService), ): project = service.getProjectById(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") - return service.getProtocolLogs(projectId, protocolId, offset, errOffset, scheduleOffset) + return service.getProtocolLogs( + projectId, + protocolId, + offset, + errOffset, + scheduleOffset, + mapper=mapper, + ) diff --git a/app/backend/api/services/coords2d_service.py b/app/backend/api/services/coords2d_service.py index be97aea6..1d1ee2a9 100644 --- a/app/backend/api/services/coords2d_service.py +++ b/app/backend/api/services/coords2d_service.py @@ -66,26 +66,11 @@ def _loadCoordinatesOutput( detail="Project not found", ) - currentProject = self.projectService.currentProject - if currentProject is None: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Project could not be loaded", - ) - - try: - protocol = currentProject.getProtocol(int(protocolId)) - except Exception as e: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol '{protocolId}' not found: {e}", - ) - - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol '{protocolId}' not found", - ) + protocol = self.projectService._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) if not hasattr(protocol, outputName): raise HTTPException( @@ -295,6 +280,45 @@ def _countCoordinatesByMicrograph(self, coordinatesSet: Any) -> Dict[str, int]: except Exception: return counts + def _getPostgresqlCoords2dReaderIfAvailable( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + protocolId: int, + outputName: str, + ): + if mapper is None: + return None + + try: + from app.backend.viewers.postgresql_coords2d_reader import PostgresqlCoords2dReader + + readerProtocolId = self.projectService._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + reader = PostgresqlCoords2dReader( + db=mapper.db, + projectId=projectId, + protocolId=readerProtocolId, + outputName=outputName, + ) + + if reader.hasOutput(): + return reader + + except Exception: + logger.exception( + "Failed to initialize PostgreSQL Coords2D reader. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + ) + + return None + def listMicrographs( self, mapper: PostgresqlFlatMapper, @@ -303,6 +327,29 @@ def listMicrographs( protocolId: int, outputName: str, ) -> Dict[str, Any]: + pgReader = self._getPostgresqlCoords2dReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.listMicrographs() + if payload is not None: + return payload + + if mapper is not None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=( + "Coordinates2D output is not available in PostgreSQL metadata" + if pgReader is None + else "Coordinates2D micrographs are not available in PostgreSQL metadata: %s" + % getattr(pgReader, "lastSkipReason", None) + ), + ) + _, coordinatesSet = self._loadCoordinatesOutput( mapper, projectId, @@ -357,14 +404,37 @@ def _findMicrograph(self, coordinatesSet: Any, micId: str) -> Any: return micrograph def listCoordinatesForMicrograph( - self, - mapper: PostgresqlFlatMapper, - projectId: int, - currentUser: Any, - protocolId: int, - outputName: str, - micId: str, + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser: Any, + protocolId: int, + outputName: str, + micId: str, ) -> Dict[str, Any]: + pgReader = self._getPostgresqlCoords2dReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.listCoordinatesForMicrograph(micId) + if payload is not None: + return payload + + if mapper is not None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=( + "Coordinates2D output is not available in PostgreSQL metadata" + if pgReader is None + else "Coordinates2D coordinates are not available in PostgreSQL metadata: %s" + % getattr(pgReader, "lastSkipReason", None) + ), + ) + _, coordinatesSet = self._loadCoordinatesOutput( mapper, projectId, diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py new file mode 100644 index 00000000..920d6676 --- /dev/null +++ b/app/backend/api/services/project_consistency_service.py @@ -0,0 +1,1979 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +from __future__ import annotations +from datetime import datetime + +import logging +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple + +from fastapi import HTTPException, status +from pyworkflow.object import PointerList +from pyworkflow.protocol.params import PointerParam, MultiPointerParam, RelationParam + +from app.backend.mapper.postgresql import PostgresqlFlatMapper + +if TYPE_CHECKING: + from app.backend.api.services.project_service import ProjectService + +logger = logging.getLogger(__name__) + + +class ProjectConsistencyService: + def __init__(self, projectService: "ProjectService"): + self.projectService = projectService + + @property + def currentProject(self): + return self.projectService.currentProject + + @currentProject.setter + def currentProject(self, value): + self.projectService.currentProject = value + + def __getattr__(self, name: str) -> Any: + return getattr(self.projectService, name) + + def normalizeStatus(self, value: Any) -> str: + return str(value or "").strip().lower() + + def normalizeClassName(self, value: Any) -> str: + return str(value or "").strip() + + def normalizeProtocolId(self, value: Any) -> str: + return str(value).strip() + + def normalizeOptionalText(self, value: Any) -> Optional[str]: + if value is None or value == "": + return None + + text = str(value).strip() + return text or None + + def toOptionalInt(self, value: Any) -> Optional[int]: + try: + if value is None or value == "": + return None + return int(value) + except Exception: + return None + + def protocolSortKey(self, value: Any): + text = self.normalizeProtocolId(value) + try: + return 0, int(text) + except Exception: + return 1, text + + def dependencySortKey(self, item: Tuple[str, str]): + parentId, childId = item + return self.protocolSortKey(parentId), self.protocolSortKey(childId) + + def stepSortKey(self, item: Tuple[str, int]): + protocolId, stepIndex = item + return self.protocolSortKey(protocolId), int(stepIndex) + + def inputRefSortKey(self, item: Tuple[str, str, int]): + protocolId, inputName, itemIndex = item + return self.protocolSortKey(protocolId), str(inputName), int(itemIndex) + + def paramSortKey(self, item: Tuple[str, str]): + protocolId, paramName = item + return self.protocolSortKey(protocolId), str(paramName) + + def buildDependency(self, parentId: Any, childId: Any) -> Dict[str, str]: + return { + "parentId": self.normalizeProtocolId(parentId), + "childId": self.normalizeProtocolId(childId), + } + + def buildStep(self, protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[str, Any]: + return { + "protocolId": self.normalizeProtocolId(protocolId), + "index": int(stepIndex), + "name": str(payload.get("name") or ""), + "status": self.normalizeStatus(payload.get("status")), + } + + def buildInputRef( + self, + key: Tuple[str, str, int], + payload: Dict[str, Any], + ) -> Dict[str, Any]: + protocolId, inputName, itemIndex = key + return { + "protocolId": self.normalizeProtocolId(protocolId), + "inputName": str(inputName), + "itemIndex": int(itemIndex), + "parentProtocolId": self.normalizeOptionalText(payload.get("parentProtocolId")), + "parentOutputName": self.normalizeOptionalText(payload.get("parentOutputName")), + "objectClassName": self.normalizeOptionalText(payload.get("objectClassName")), + } + + def buildParamIssue( + self, + key: Tuple[str, str], + payload: Dict[str, Any], + ) -> Dict[str, Any]: + protocolId, paramName = key + return { + "protocolId": self.normalizeProtocolId(protocolId), + "paramName": str(paramName), + "value": payload.get("value"), + } + + def buildMissingOutputIssue( + self, + protocolId: Any, + outputName: Any, + runtimeOutputsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]], + ) -> Dict[str, Any]: + protocolIdText = self.normalizeProtocolId(protocolId) + outputNameText = str(outputName) + + return { + "protocolId": protocolIdText, + "outputName": outputNameText, + "className": runtimeOutputsByProtocolId + .get(protocolIdText, {}) + .get(outputNameText, {}) + .get("className"), + } + + def buildExtraOutputIssue( + self, + protocolId: Any, + outputName: Any, + persistedOutputsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]], + ) -> Dict[str, Any]: + protocolIdText = self.normalizeProtocolId(protocolId) + outputNameText = str(outputName) + + return { + "protocolId": protocolIdText, + "outputName": outputNameText, + "mapperKind": persistedOutputsByProtocolId + .get(protocolIdText, {}) + .get(outputNameText, {}) + .get("mapperKind"), + "className": persistedOutputsByProtocolId + .get(protocolIdText, {}) + .get(outputNameText, {}) + .get("className"), + } + + def buildPostgresqlOutputPayloadIssue( + self, + protocolId: Any, + outputName: Any, + payload: Dict[str, Any], + missingFields: List[str], + ) -> Dict[str, Any]: + issue = { + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": payload.get("mapperKind"), + "className": payload.get("className"), + "missingFields": list(missingFields), + } + + for fieldName in ( + "setId", + "rootObjectId", + "scipionObjId", + "itemsCount", + "itemClassName", + ): + if fieldName in payload: + issue[fieldName] = payload.get(fieldName) + + return issue + + def expectedOutputMapperKind(self, className: Any) -> Optional[str]: + classNameText = self.normalizeOptionalText(className) + if classNameText is None: + return None + + if classNameText.startswith("SetOf") or "SetOf" in classNameText: + return "flat_set" + + return "tree" + + def getRuntimeOutputItemsCount(self, outputObj: Any) -> Optional[int]: + if outputObj is None: + return None + + for methodName in ("getSize", "getDim", "__len__"): + try: + if methodName == "__len__": + value = len(outputObj) + else: + method = getattr(outputObj, methodName, None) + if method is None: + continue + value = method() + + if value is None or value == "": + continue + + return int(value) + except Exception: + continue + + return None + + def iterPointerItems(self, attr: Any) -> List[Tuple[int, Any]]: + try: + if isinstance(attr, PointerList): + return [ + (index, pointer) + for index, pointer in enumerate(attr) + ] + except Exception: + pass + + return [(0, attr)] + + def dependencyKeyFromInputRef(self, payload: Dict[str, Any]) -> Optional[Tuple[str, str]]: + parentProtocolId = self.normalizeOptionalText(payload.get("parentProtocolId")) + childProtocolId = self.normalizeOptionalText(payload.get("protocolId")) + + if not parentProtocolId or not childProtocolId: + return None + + if parentProtocolId == "PROJECT" or childProtocolId == "PROJECT": + return None + + if parentProtocolId == childProtocolId: + return None + + return parentProtocolId, childProtocolId + + def normalizeParamValue(self, value: Any) -> Any: + if value is None: + return None + + if isinstance(value, bool): + return value + + if isinstance(value, (int, float)): + return value + + if isinstance(value, str): + text = value.strip() + if text == "": + return "" + lowerText = text.lower() + if lowerText in ("true", "false"): + return lowerText == "true" + + try: + if "." not in text: + return int(text) + except Exception: + pass + + try: + return float(text) + except Exception: + return text + + if isinstance(value, (list, tuple)): + return [self.normalizeParamValue(item) for item in value] + + if isinstance(value, dict): + return { + str(key): self.normalizeParamValue(itemValue) + for key, itemValue in value.items() + } + + try: + if hasattr(value, "get"): + return self.normalizeParamValue(value.get()) + except Exception: + pass + + return str(value) + + def isPointerParam(self, param: Any) -> bool: + return isinstance(param, (PointerParam, MultiPointerParam, RelationParam)) + + def stepValue(self, step: Any, attrName: str, fallback: Any = None) -> Any: + try: + value = getattr(step, attrName, None) + if hasattr(value, "get"): + return value.get() + return value if value is not None else fallback + except Exception: + return fallback + + def getStepName(self, step: Any) -> str: + name = self.stepValue(step, "funcName", None) + if name: + return str(name) + + className = self._safeCall(step, "getClassName", None) + return str(className or "") + + def extractRuntimeInputRef( + self, + protocolId: str, + inputName: str, + itemIndex: int, + pointer: Any, + ) -> Optional[Dict[str, Any]]: + parentProtocolId = None + parentOutputName = None + objectClassName = None + objectId = None + + try: + parentObj = pointer.getObjValue() + parentProtocolId = self.normalizeOptionalText( + self._safeCall(parentObj, "getObjId", None) + ) + except Exception: + parentProtocolId = None + + try: + parentOutputName = self.normalizeOptionalText(pointer.getExtended()) + except Exception: + parentOutputName = None + + try: + targetObj = pointer.get() + if targetObj is not None: + objectClassName = self.normalizeOptionalText( + self._getScipionClassName(targetObj) + ) + objectId = self.normalizeOptionalText( + self._safeCall(targetObj, "getObjId", None) + ) + except Exception: + objectClassName = None + objectId = None + + if parentProtocolId is None and parentOutputName is None: + return None + + return { + "protocolId": self.normalizeProtocolId(protocolId), + "inputName": str(inputName), + "itemIndex": int(itemIndex), + "parentProtocolId": parentProtocolId, + "parentOutputName": parentOutputName, + "objectClassName": objectClassName, + "objectId": objectId, + } + + def extractRuntimeParams(self, protocol: Any) -> Dict[str, Dict[str, Any]]: + paramsByName: Dict[str, Dict[str, Any]] = {} + + try: + self.currentProject._fixProtParamsConfiguration(protocol) + except Exception: + pass + + try: + for paramName, param in protocol.iterParams(): + paramNameText = str(paramName or "").strip() + if not paramNameText: + continue + + if self.isPointerParam(param): + continue + + rawValue = None + try: + rawValue = protocol.getAttributeValue(paramNameText) + except Exception: + try: + rawValue = getattr(protocol, paramNameText, None) + except Exception: + rawValue = None + + paramsByName[paramNameText] = { + "value": self.normalizeParamValue(rawValue), + } + except Exception: + logger.debug( + "Could not inspect runtime protocol params during consistency check.", + exc_info=True, + ) + + return paramsByName + + def extractPostgresqlParams(self, row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + rawParams = row.get("params") + if not isinstance(rawParams, dict): + return {} + + paramsByName: Dict[str, Dict[str, Any]] = {} + for paramName, rawValue in rawParams.items(): + paramNameText = str(paramName or "").strip() + if not paramNameText: + continue + + paramsByName[paramNameText] = { + "value": self.normalizeParamValue(rawValue), + } + + return paramsByName + + def collectRuntimeSnapshot( + self, + projectId: int, + refresh: bool, + checkPid: bool, + ) -> Dict[str, Any]: + runtimeStatuses: Dict[str, str] = {} + runtimeClassNames: Dict[str, str] = {} + runtimeDependencies: Set[Tuple[str, str]] = set() + runtimeOutputsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} + runtimeStepsByProtocolId: Dict[str, Dict[int, Dict[str, Any]]] = {} + runtimeInputRefsByKey: Dict[Tuple[str, str, int], Dict[str, Any]] = {} + runtimeParamsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} + + try: + runs = self.currentProject.getRunsGraph(refresh=refresh, checkPids=checkPid) + nodesDict = getattr(runs, "_nodesDict", {}) or {} + except Exception as e: + logger.exception( + "Failed to load Scipion runtime graph for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load Scipion runtime graph: {e}", + ) + + for nodeId, nodeObj in nodesDict.items(): + protocolId = self.normalizeProtocolId(nodeId) + if not protocolId or protocolId == "PROJECT": + continue + + protocol = getattr(nodeObj, "run", None) + runtimeStatuses[protocolId] = self.normalizeStatus( + self._safeCall(protocol, "getStatus", None) + ) + runtimeClassNames[protocolId] = self.normalizeClassName( + self._safeCall(protocol, "getClassName", None) + ) + runtimeOutputsByProtocolId.setdefault(protocolId, {}) + + if protocol is not None: + try: + for outputItem in protocol.iterOutputAttributes(): + outputName = None + outputObj = None + + if isinstance(outputItem, (tuple, list)) and len(outputItem) >= 2: + outputName = outputItem[0] + outputObj = outputItem[1] + else: + outputName = self._safeCall(outputItem, "getName", None) + outputObj = outputItem + + outputName = str(outputName or "").strip() + if not outputName or outputObj is None: + continue + + outputClassName = self._getScipionClassName(outputObj) + runtimeOutputsByProtocolId[protocolId][outputName] = { + "outputName": outputName, + "className": outputClassName, + "itemsCount": self.getRuntimeOutputItemsCount(outputObj) + if self.expectedOutputMapperKind(outputClassName) == "flat_set" + else None, + } + except Exception: + logger.debug( + "Could not inspect runtime protocol outputs during consistency check. " + "projectId=%s protocolId=%s", + projectId, + protocolId, + exc_info=True, + ) + + runtimeStepsByProtocolId.setdefault(protocolId, {}) + + try: + for stepPayload in self._buildProtocolStepsForPostgresql(protocol): + stepIndex = self.toOptionalInt(stepPayload.get("index")) + if stepIndex is None: + continue + + runtimeStepsByProtocolId[protocolId][stepIndex] = { + "index": stepIndex, + "name": str(stepPayload.get("name") or ""), + "status": self.normalizeStatus(stepPayload.get("status")), + } + except Exception: + logger.debug( + "Could not inspect runtime protocol steps during consistency check. " + "projectId=%s protocolId=%s", + projectId, + protocolId, + exc_info=True, + ) + + try: + for inputName, attr in protocol.iterInputAttributes(): + inputNameText = str(inputName or "").strip() + if not inputNameText: + continue + + for itemIndex, pointer in self.iterPointerItems(attr): + inputRef = self.extractRuntimeInputRef( + protocolId=protocolId, + inputName=inputNameText, + itemIndex=int(itemIndex), + pointer=pointer, + ) + if inputRef is None: + continue + + key = ( + inputRef["protocolId"], + inputRef["inputName"], + inputRef["itemIndex"], + ) + runtimeInputRefsByKey[key] = inputRef + except Exception: + logger.debug( + "Could not inspect runtime protocol input refs during consistency check. " + "projectId=%s protocolId=%s", + projectId, + protocolId, + exc_info=True, + ) + + runtimeParamsByProtocolId[protocolId] = self.extractRuntimeParams(protocol) + + for parent in getattr(nodeObj, "_parents", []) or []: + try: + parentId = self.normalizeProtocolId(parent.getName()) + except Exception: + parentId = self.normalizeProtocolId(parent) + + if not parentId or parentId == "PROJECT": + continue + + runtimeDependencies.add((parentId, protocolId)) + + return { + "statuses": runtimeStatuses, + "classNames": runtimeClassNames, + "dependencies": runtimeDependencies, + "outputsByProtocolId": runtimeOutputsByProtocolId, + "stepsByProtocolId": runtimeStepsByProtocolId, + "inputRefsByKey": runtimeInputRefsByKey, + "paramsByProtocolId": runtimeParamsByProtocolId, + } + + def collectPostgresqlSnapshot( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + ) -> Dict[str, Any]: + try: + protocolRows = mapper.getProtocols(projectId) or [] + except Exception as e: + logger.exception( + "Failed to load PostgreSQL protocols for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL protocols: {e}", + ) + + postgresqlStatuses: Dict[str, str] = {} + postgresqlClassNames: Dict[str, str] = {} + postgresqlParamsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} + + for row in protocolRows: + protocolId = self.normalizeProtocolId(row.get("protocolId")) + if not protocolId: + continue + + postgresqlStatuses[protocolId] = self.normalizeStatus(row.get("status")) + postgresqlClassNames[protocolId] = self.normalizeClassName(row.get("protocolClassName")) + postgresqlParamsByProtocolId[protocolId] = self.extractPostgresqlParams(row) + + try: + adjacencyMap = mapper.getProjectProtocolAdjacencyMap(projectId) or {} + except Exception as e: + logger.exception( + "Failed to load PostgreSQL protocol dependencies for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL protocol dependencies: {e}", + ) + + postgresqlDependencies: Set[Tuple[str, str]] = set() + for childId, refs in adjacencyMap.items(): + childProtocolId = self.normalizeProtocolId(childId) + if not childProtocolId or childProtocolId == "PROJECT": + continue + + for parentIdValue in refs.get("parents") or []: + parentProtocolId = self.normalizeProtocolId(parentIdValue) + if not parentProtocolId or parentProtocolId == "PROJECT": + continue + + postgresqlDependencies.add((parentProtocolId, childProtocolId)) + + try: + persistedOutputsByProtocolId = self._loadPersistedOutputsByProtocolId( + mapper, + projectId, + ) + except Exception as e: + logger.exception( + "Failed to load PostgreSQL persisted outputs for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL persisted outputs: {e}", + ) + + try: + postgresqlStepsByProtocolId = mapper.getProjectProtocolStepsByProtocolId(projectId) or {} + except Exception as e: + logger.exception( + "Failed to load PostgreSQL protocol steps for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL protocol steps: {e}", + ) + + normalizedPostgresqlStepsByProtocolId: Dict[str, Dict[int, Dict[str, Any]]] = {} + for protocolId, steps in postgresqlStepsByProtocolId.items(): + protocolIdText = self.normalizeProtocolId(protocolId) + for step in steps or []: + stepIndex = self.toOptionalInt(step.get("index")) + if stepIndex is None: + continue + + normalizedPostgresqlStepsByProtocolId.setdefault(protocolIdText, {})[stepIndex] = { + "index": stepIndex, + "name": str(step.get("name") or ""), + "status": self.normalizeStatus(step.get("status")), + } + + try: + postgresqlInputRefs = mapper.listProtocolInputRefs(projectId) or [] + except Exception as e: + logger.exception( + "Failed to load PostgreSQL protocol input refs for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL protocol input refs: {e}", + ) + + postgresqlInputRefsByKey: Dict[Tuple[str, str, int], Dict[str, Any]] = {} + for ref in postgresqlInputRefs: + protocolIdText = self.normalizeProtocolId(ref.get("protocolId")) + inputName = str(ref.get("inputName") or "").strip() + itemIndex = self.toOptionalInt(ref.get("itemIndex")) + + if not protocolIdText or not inputName: + continue + + if itemIndex is None or itemIndex < 0: + itemIndex = 0 + + key = (protocolIdText, inputName, int(itemIndex)) + postgresqlInputRefsByKey[key] = { + "protocolId": protocolIdText, + "inputName": inputName, + "itemIndex": int(itemIndex), + "parentProtocolId": self.normalizeOptionalText(ref.get("parentProtocolId")), + "parentOutputName": self.normalizeOptionalText(ref.get("parentOutputName")), + "objectClassName": self.normalizeOptionalText(ref.get("objectClassName")), + "objectId": self.normalizeOptionalText(ref.get("objectId")), + } + + return { + "statuses": postgresqlStatuses, + "classNames": postgresqlClassNames, + "dependencies": postgresqlDependencies, + "outputsByProtocolId": persistedOutputsByProtocolId, + "stepsByProtocolId": normalizedPostgresqlStepsByProtocolId, + "inputRefsByKey": postgresqlInputRefsByKey, + "paramsByProtocolId": postgresqlParamsByProtocolId, + } + + def buildDerivedSets( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeStatuses = runtimeSnapshot["statuses"] + runtimeDependencies = runtimeSnapshot["dependencies"] + runtimeOutputsByProtocolId = runtimeSnapshot["outputsByProtocolId"] + runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] + runtimeInputRefsByKey = runtimeSnapshot["inputRefsByKey"] + runtimeParamsByProtocolId = runtimeSnapshot["paramsByProtocolId"] + + postgresqlStatuses = postgresqlSnapshot["statuses"] + postgresqlDependencies = postgresqlSnapshot["dependencies"] + persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] + normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] + postgresqlInputRefsByKey = postgresqlSnapshot["inputRefsByKey"] + postgresqlParamsByProtocolId = postgresqlSnapshot["paramsByProtocolId"] + + runtimeOutputs: Set[Tuple[str, str]] = set() + for protocolId, outputsByName in runtimeOutputsByProtocolId.items(): + for outputName in outputsByName.keys(): + runtimeOutputs.add((protocolId, outputName)) + + postgresqlOutputs: Set[Tuple[str, str]] = set() + for protocolId, outputsByName in persistedOutputsByProtocolId.items(): + for outputName in outputsByName.keys(): + postgresqlOutputs.add((self.normalizeProtocolId(protocolId), str(outputName))) + + runtimeSteps: Set[Tuple[str, int]] = set() + for protocolId, stepsByIndex in runtimeStepsByProtocolId.items(): + for stepIndex in stepsByIndex.keys(): + runtimeSteps.add((protocolId, int(stepIndex))) + + postgresqlSteps: Set[Tuple[str, int]] = set() + for protocolId, stepsByIndex in normalizedPostgresqlStepsByProtocolId.items(): + for stepIndex in stepsByIndex.keys(): + postgresqlSteps.add((protocolId, int(stepIndex))) + + runtimeProtocolIds = set(runtimeStatuses.keys()) + postgresqlProtocolIds = set(postgresqlStatuses.keys()) + runtimeInputRefs = set(runtimeInputRefsByKey.keys()) + postgresqlInputRefsKeys = set(postgresqlInputRefsByKey.keys()) + + runtimeParams: Set[Tuple[str, str]] = set() + for protocolId, paramsByName in runtimeParamsByProtocolId.items(): + for paramName in paramsByName.keys(): + runtimeParams.add((protocolId, paramName)) + + postgresqlParams: Set[Tuple[str, str]] = set() + for protocolId, paramsByName in postgresqlParamsByProtocolId.items(): + for paramName in paramsByName.keys(): + postgresqlParams.add((protocolId, paramName)) + + runtimeDependenciesFromInputRefs: Set[Tuple[str, str]] = set() + for inputRef in runtimeInputRefsByKey.values(): + dependencyKey = self.dependencyKeyFromInputRef(inputRef) + if dependencyKey is not None: + runtimeDependenciesFromInputRefs.add(dependencyKey) + + postgresqlDependenciesFromInputRefs: Set[Tuple[str, str]] = set() + for inputRef in postgresqlInputRefsByKey.values(): + dependencyKey = self.dependencyKeyFromInputRef(inputRef) + if dependencyKey is not None: + postgresqlDependenciesFromInputRefs.add(dependencyKey) + + return { + "runtimeOutputs": runtimeOutputs, + "postgresqlOutputs": postgresqlOutputs, + "runtimeSteps": runtimeSteps, + "postgresqlSteps": postgresqlSteps, + "runtimeProtocolIds": runtimeProtocolIds, + "postgresqlProtocolIds": postgresqlProtocolIds, + "runtimeInputRefs": runtimeInputRefs, + "postgresqlInputRefsKeys": postgresqlInputRefsKeys, + "runtimeParams": runtimeParams, + "postgresqlParams": postgresqlParams, + "runtimeDependenciesFromInputRefs": runtimeDependenciesFromInputRefs, + "postgresqlDependenciesFromInputRefs": postgresqlDependenciesFromInputRefs, + } + + def compareProtocols( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeStatuses = runtimeSnapshot["statuses"] + runtimeClassNames = runtimeSnapshot["classNames"] + + postgresqlStatuses = postgresqlSnapshot["statuses"] + postgresqlClassNames = postgresqlSnapshot["classNames"] + + runtimeProtocolIds = derivedSets["runtimeProtocolIds"] + postgresqlProtocolIds = derivedSets["postgresqlProtocolIds"] + + missingProtocolIds = sorted( + runtimeProtocolIds - postgresqlProtocolIds, + key=self.protocolSortKey, + ) + extraProtocolIds = sorted( + postgresqlProtocolIds - runtimeProtocolIds, + key=self.protocolSortKey, + ) + + commonProtocolIds = sorted( + runtimeProtocolIds.intersection(postgresqlProtocolIds), + key=self.protocolSortKey, + ) + + statusMismatches = [] + for protocolId in commonProtocolIds: + runtimeStatus = runtimeStatuses.get(protocolId, "") + postgresqlStatus = postgresqlStatuses.get(protocolId, "") + + if runtimeStatus != postgresqlStatus: + statusMismatches.append({ + "protocolId": protocolId, + "runtimeStatus": runtimeStatus, + "postgresqlStatus": postgresqlStatus, + }) + + protocolClassMismatches = [] + for protocolId in commonProtocolIds: + runtimeClassName = self.normalizeClassName(runtimeClassNames.get(protocolId)) + postgresqlClassName = self.normalizeClassName(postgresqlClassNames.get(protocolId)) + + if ( + runtimeClassName + and postgresqlClassName + and runtimeClassName != postgresqlClassName + ): + protocolClassMismatches.append({ + "protocolId": protocolId, + "runtimeClassName": runtimeClassName, + "postgresqlClassName": postgresqlClassName, + }) + + return { + "missingProtocolIds": missingProtocolIds, + "extraProtocolIds": extraProtocolIds, + "commonProtocolIds": commonProtocolIds, + "statusMismatches": statusMismatches, + "protocolClassMismatches": protocolClassMismatches, + } + + def compareDependencies( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeDependencies = runtimeSnapshot["dependencies"] + postgresqlDependencies = postgresqlSnapshot["dependencies"] + + missingDependencies = sorted( + runtimeDependencies - postgresqlDependencies, + key=self.dependencySortKey, + ) + extraDependencies = sorted( + postgresqlDependencies - runtimeDependencies, + key=self.dependencySortKey, + ) + + return { + "missingDependencies": missingDependencies, + "extraDependencies": extraDependencies, + } + + def compareOutputs( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeOutputsByProtocolId = runtimeSnapshot["outputsByProtocolId"] + persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] + + runtimeOutputs = derivedSets["runtimeOutputs"] + postgresqlOutputs = derivedSets["postgresqlOutputs"] + + commonOutputs = sorted( + runtimeOutputs.intersection(postgresqlOutputs), + key=self.dependencySortKey, + ) + + missingOutputs = sorted( + runtimeOutputs - postgresqlOutputs, + key=self.dependencySortKey, + ) + extraOutputs = sorted( + postgresqlOutputs - runtimeOutputs, + key=self.dependencySortKey, + ) + + outputClassMismatches = [] + for protocolId, outputName in commonOutputs: + runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + + runtimeClassName = self.normalizeOptionalText(runtimeOutput.get("className")) + postgresqlClassName = self.normalizeOptionalText(postgresqlOutput.get("className")) + + if ( + runtimeClassName is not None + and postgresqlClassName is not None + and runtimeClassName != postgresqlClassName + ): + outputClassMismatches.append({ + "protocolId": protocolId, + "outputName": outputName, + "runtimeClassName": runtimeClassName, + "postgresqlClassName": postgresqlClassName, + "mapperKind": postgresqlOutput.get("mapperKind"), + }) + + outputMapperKindMismatches = [] + for protocolId, outputName in commonOutputs: + runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + + runtimeClassName = self.normalizeOptionalText(runtimeOutput.get("className")) + expectedMapperKind = self.expectedOutputMapperKind(runtimeClassName) + postgresqlMapperKind = self.normalizeOptionalText(postgresqlOutput.get("mapperKind")) + + if ( + expectedMapperKind is not None + and postgresqlMapperKind is not None + and expectedMapperKind != postgresqlMapperKind + ): + outputMapperKindMismatches.append({ + "protocolId": protocolId, + "outputName": outputName, + "className": runtimeClassName, + "expectedMapperKind": expectedMapperKind, + "postgresqlMapperKind": postgresqlMapperKind, + }) + + outputItemsCountMismatches = [] + for protocolId, outputName in commonOutputs: + runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + + runtimeClassName = self.normalizeOptionalText(runtimeOutput.get("className")) + if self.expectedOutputMapperKind(runtimeClassName) != "flat_set": + continue + + runtimeItemsCount = self.toOptionalInt(runtimeOutput.get("itemsCount")) + postgresqlItemsCount = self.toOptionalInt(postgresqlOutput.get("itemsCount")) + + if runtimeItemsCount is None or postgresqlItemsCount is None: + continue + + if runtimeItemsCount != postgresqlItemsCount: + outputItemsCountMismatches.append({ + "protocolId": protocolId, + "outputName": outputName, + "className": runtimeClassName, + "runtimeItemsCount": runtimeItemsCount, + "postgresqlItemsCount": postgresqlItemsCount, + "mapperKind": postgresqlOutput.get("mapperKind"), + }) + + return { + "missingOutputs": missingOutputs, + "extraOutputs": extraOutputs, + "outputClassMismatches": outputClassMismatches, + "outputMapperKindMismatches": outputMapperKindMismatches, + "outputItemsCountMismatches": outputItemsCountMismatches, + } + + def comparePostgresqlOutputPayloads( + self, + postgresqlSnapshot: Dict[str, Any], + projectId: Optional[int] = None, + ) -> Dict[str, Any]: + persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] + + flatSetOutputsWithIncompletePayload = [] + treeOutputsWithIncompletePayload = [] + flatSetItemsCountMismatches = [] + flatSetMaxItemIdMismatches = [] + flatSetRootTableMismatches = [] + flatSetColumnsCountMismatches = [] + flatSetPropertiesMismatches = [] + flatSetRootObjectMismatches = [] + + for protocolId in sorted(persistedOutputsByProtocolId.keys(), key=self.protocolSortKey): + outputsByName = persistedOutputsByProtocolId.get(protocolId, {}) + + for outputName in sorted(outputsByName.keys()): + payload = outputsByName.get(outputName, {}) + mapperKind = payload.get("mapperKind") + + if mapperKind == "flat_set": + missingFields = [] + + if payload.get("setId") in (None, ""): + missingFields.append("setId") + if payload.get("rootObjectId") in (None, ""): + missingFields.append("rootObjectId") + if self.normalizeOptionalText(payload.get("className")) is None: + missingFields.append("className") + if payload.get("itemsCount") is None: + missingFields.append("itemsCount") + + if missingFields: + flatSetOutputsWithIncompletePayload.append( + self.buildPostgresqlOutputPayloadIssue( + protocolId=protocolId, + outputName=outputName, + payload=payload, + missingFields=missingFields, + ) + ) + + propertiesItemsCount = self.toOptionalInt(payload.get("itemsCount")) + itemsTableCount = self.toOptionalInt(payload.get("itemsTableCount")) + + if ( + propertiesItemsCount is not None + and itemsTableCount is not None + and propertiesItemsCount != itemsTableCount + ): + flatSetItemsCountMismatches.append({ + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": mapperKind, + "className": payload.get("className"), + "setId": payload.get("setId"), + "rootObjectId": payload.get("rootObjectId"), + "itemsCount": propertiesItemsCount, + "itemsTableCount": itemsTableCount, + "itemClassName": payload.get("itemClassName"), + }) + + propertiesMaxItemId = self.toOptionalInt(payload.get("maxItemId")) + maxItemIdFromItems = self.toOptionalInt(payload.get("maxItemIdFromItems")) + itemsIdSignature = self.normalizeOptionalText(payload.get("itemsIdSignature")) + itemsValueSignature = self.normalizeOptionalText(payload.get("itemsValueSignature")) + + if ( + propertiesMaxItemId is not None + and maxItemIdFromItems is not None + and propertiesMaxItemId != maxItemIdFromItems + ): + flatSetMaxItemIdMismatches.append({ + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": mapperKind, + "className": payload.get("className"), + "setId": payload.get("setId"), + "rootObjectId": payload.get("rootObjectId"), + "maxItemId": propertiesMaxItemId, + "maxItemIdFromItems": maxItemIdFromItems, + "itemClassName": payload.get("itemClassName"), + }) + + propertiesColumnsCount = self.toOptionalInt(payload.get("columnsCount")) + setColumnsCount = self.toOptionalInt(payload.get("setColumnsCount")) + + if ( + propertiesColumnsCount is not None + and setColumnsCount is not None + and propertiesColumnsCount != setColumnsCount + ): + flatSetColumnsCountMismatches.append({ + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": mapperKind, + "className": payload.get("className"), + "setId": payload.get("setId"), + "rootObjectId": payload.get("rootObjectId"), + "columnsCount": propertiesColumnsCount, + "setColumnsCount": setColumnsCount, + "itemClassName": payload.get("itemClassName"), + }) + + propertiesPayloadCount = self.toOptionalInt(payload.get("propertiesPayloadCount")) + setPropertiesCount = self.toOptionalInt(payload.get("setPropertiesCount")) + propertiesPayloadSignature = payload.get("propertiesPayloadSignature") or [] + setPropertiesSignature = payload.get("setPropertiesSignature") or [] + + propertiesChangedFields = [] + + if ( + propertiesPayloadCount is not None + and setPropertiesCount is not None + and propertiesPayloadCount != setPropertiesCount + ): + propertiesChangedFields.append("setPropertiesCount") + + if ( + propertiesPayloadSignature + and setPropertiesSignature + and propertiesPayloadSignature != setPropertiesSignature + ): + propertiesChangedFields.append("setPropertiesSignature") + + if propertiesChangedFields: + flatSetPropertiesMismatches.append({ + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": mapperKind, + "className": payload.get("className"), + "setId": payload.get("setId"), + "rootObjectId": payload.get("rootObjectId"), + "fields": propertiesChangedFields, + "propertiesPayloadCount": propertiesPayloadCount, + "setPropertiesCount": setPropertiesCount, + "propertiesPayloadSignature": propertiesPayloadSignature, + "setPropertiesSignature": setPropertiesSignature, + "itemClassName": payload.get("itemClassName"), + }) + + protocolDbId = self.toOptionalInt(payload.get("protocolDbId")) + setId = self.toOptionalInt(payload.get("setId")) + rootObjectId = self.toOptionalInt(payload.get("rootObjectId")) + rootObjectDbId = self.toOptionalInt(payload.get("rootObjectDbId")) + rootObjectProjectId = self.toOptionalInt(payload.get("rootObjectProjectId")) + rootObjectProtocolDbId = self.toOptionalInt(payload.get("rootObjectProtocolDbId")) + rootObjectParentObjectId = self.toOptionalInt(payload.get("rootObjectParentObjectId")) + rootObjectName = self.normalizeOptionalText(payload.get("rootObjectName")) + rootObjectPath = self.normalizeOptionalText(payload.get("rootObjectPath")) + rootObjectClassName = self.normalizeOptionalText(payload.get("rootObjectClassName")) + setClassName = self.normalizeOptionalText(payload.get("className")) + + rootObjectChangedFields = [] + + if rootObjectId is not None and rootObjectDbId is None: + rootObjectChangedFields.append("rootObjectMissing") + else: + if ( + rootObjectProjectId is not None + and projectId is not None + and rootObjectProjectId != projectId + ): + rootObjectChangedFields.append("rootObjectProjectId") + + if ( + rootObjectProtocolDbId is not None + and protocolDbId is not None + and rootObjectProtocolDbId != protocolDbId + ): + rootObjectChangedFields.append("rootObjectProtocolDbId") + + if rootObjectParentObjectId is not None: + rootObjectChangedFields.append("rootObjectParentObjectId") + + if ( + rootObjectName is not None + and str(outputName) != rootObjectName + ): + rootObjectChangedFields.append("rootObjectName") + + if ( + rootObjectPath is not None + and str(outputName) != rootObjectPath + ): + rootObjectChangedFields.append("rootObjectPath") + + if ( + setClassName is not None + and rootObjectClassName is not None + and setClassName != rootObjectClassName + ): + rootObjectChangedFields.append("rootObjectClassName") + + if rootObjectChangedFields: + flatSetRootObjectMismatches.append({ + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": mapperKind, + "className": payload.get("className"), + "setId": setId, + "rootObjectId": rootObjectId, + "fields": rootObjectChangedFields, + "protocolDbId": protocolDbId, + "rootObjectDbId": rootObjectDbId, + "rootObjectProjectId": rootObjectProjectId, + "rootObjectProtocolDbId": rootObjectProtocolDbId, + "rootObjectParentObjectId": rootObjectParentObjectId, + "rootObjectName": rootObjectName, + "rootObjectPath": rootObjectPath, + "rootObjectClassName": rootObjectClassName, + "itemClassName": payload.get("itemClassName"), + }) + + rootTablesCount = self.toOptionalInt(payload.get("rootTablesCount")) + rootTableId = self.toOptionalInt(payload.get("rootTableId")) + rootTableItemsCount = self.toOptionalInt(payload.get("rootTableItemsCount")) + rootTableMaxItemId = self.toOptionalInt(payload.get("rootTableMaxItemId")) + rootTableColumnsCount = self.toOptionalInt(payload.get("rootTableColumnsCount")) + setColumnsSignature = payload.get("setColumnsSignature") or [] + rootTableColumnsSignature = payload.get("rootTableColumnsSignature") or [] + rootTableItemsIdSignature = self.normalizeOptionalText(payload.get("rootTableItemsIdSignature")) + rootTableItemsValueSignature = self.normalizeOptionalText( + payload.get("rootTableItemsValueSignature") + ) + + changedFields = [] + + if rootTablesCount is not None: + if rootTablesCount == 0: + changedFields.append("rootTableMissing") + elif rootTablesCount != 1: + changedFields.append("rootTablesCount") + else: + if ( + itemsTableCount is not None + and rootTableItemsCount is not None + and itemsTableCount != rootTableItemsCount + ): + changedFields.append("rootTableItemsCount") + + if ( + maxItemIdFromItems is not None + and rootTableMaxItemId is not None + and maxItemIdFromItems != rootTableMaxItemId + ): + changedFields.append("rootTableMaxItemId") + + if ( + itemsIdSignature is not None + and rootTableItemsIdSignature is not None + and itemsIdSignature != rootTableItemsIdSignature + ): + changedFields.append("rootTableItemsIdSignature") + + if ( + itemsValueSignature is not None + and rootTableItemsValueSignature is not None + and itemsValueSignature != rootTableItemsValueSignature + ): + changedFields.append("rootTableItemsValueSignature") + + if ( + setColumnsCount is not None + and rootTableColumnsCount is not None + and setColumnsCount != rootTableColumnsCount + ): + changedFields.append("rootTableColumnsCount") + + if ( + setColumnsSignature + and rootTableColumnsSignature + and setColumnsSignature != rootTableColumnsSignature + ): + changedFields.append("rootTableColumnsSignature") + + if changedFields: + issue = { + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": mapperKind, + "className": payload.get("className"), + "setId": payload.get("setId"), + "rootObjectId": payload.get("rootObjectId"), + "rootTableId": rootTableId, + "fields": changedFields, + "rootTablesCount": rootTablesCount, + "itemsTableCount": itemsTableCount, + "rootTableItemsCount": rootTableItemsCount, + "maxItemIdFromItems": maxItemIdFromItems, + "rootTableMaxItemId": rootTableMaxItemId, + "setColumnsCount": setColumnsCount, + "rootTableColumnsCount": rootTableColumnsCount, + "itemClassName": payload.get("itemClassName"), + } + + if "rootTableItemsValueSignature" in changedFields: + issue["itemsValueSignature"] = itemsValueSignature + issue["rootTableItemsValueSignature"] = rootTableItemsValueSignature + + if "rootTableItemsIdSignature" in changedFields: + issue["itemsIdSignature"] = itemsIdSignature + issue["rootTableItemsIdSignature"] = rootTableItemsIdSignature + + if "rootTableColumnsSignature" in changedFields: + issue["setColumnsSignature"] = setColumnsSignature + issue["rootTableColumnsSignature"] = rootTableColumnsSignature + + flatSetRootTableMismatches.append(issue) + + elif mapperKind == "tree": + missingFields = [] + + if payload.get("rootObjectId") in (None, ""): + missingFields.append("rootObjectId") + if self.normalizeOptionalText(payload.get("className")) is None: + missingFields.append("className") + + if missingFields: + treeOutputsWithIncompletePayload.append( + self.buildPostgresqlOutputPayloadIssue( + protocolId=protocolId, + outputName=outputName, + payload=payload, + missingFields=missingFields, + ) + ) + + return { + "postgresqlFlatSetOutputsWithIncompletePayload": flatSetOutputsWithIncompletePayload, + "postgresqlTreeOutputsWithIncompletePayload": treeOutputsWithIncompletePayload, + "postgresqlFlatSetItemsCountMismatches": flatSetItemsCountMismatches, + "postgresqlFlatSetMaxItemIdMismatches": flatSetMaxItemIdMismatches, + "postgresqlFlatSetColumnsCountMismatches": flatSetColumnsCountMismatches, + "postgresqlFlatSetRootTableMismatches": flatSetRootTableMismatches, + "postgresqlFlatSetPropertiesMismatches": flatSetPropertiesMismatches, + "postgresqlFlatSetRootObjectMismatches": flatSetRootObjectMismatches, + } + + def compareSteps(self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] + normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] + + runtimeSteps = derivedSets["runtimeSteps"] + postgresqlSteps = derivedSets["postgresqlSteps"] + + if not runtimeSteps: + return { + "missingSteps": [], + "extraSteps": [], + "stepMismatches": [], + "skipped": True, + "reason": "Runtime protocol steps could not be extracted", + } + + missingSteps = sorted( + runtimeSteps - postgresqlSteps, + key=self.stepSortKey, + ) + extraSteps = sorted( + postgresqlSteps - runtimeSteps, + key=self.stepSortKey, + ) + + stepMismatches = [] + for protocolId, stepIndex in sorted(runtimeSteps.intersection(postgresqlSteps), key=self.stepSortKey): + runtimeStep = runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) + postgresqlStep = normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) + + runtimeName = str(runtimeStep.get("name") or "") + postgresqlName = str(postgresqlStep.get("name") or "") + runtimeStatus = self.normalizeStatus(runtimeStep.get("status")) + postgresqlStatus = self.normalizeStatus(postgresqlStep.get("status")) + + changedFields = [] + if runtimeName != postgresqlName: + changedFields.append("name") + if runtimeStatus != postgresqlStatus: + changedFields.append("status") + + if changedFields: + stepMismatches.append({ + "protocolId": protocolId, + "index": int(stepIndex), + "fields": changedFields, + "runtimeName": runtimeName, + "postgresqlName": postgresqlName, + "runtimeStatus": runtimeStatus, + "postgresqlStatus": postgresqlStatus, + }) + + return { + "missingSteps": missingSteps, + "extraSteps": extraSteps, + "stepMismatches": stepMismatches, + } + + def compareParams( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + + if not derivedSets["runtimeParams"]: + return { + "missingParams": [], + "extraParams": [], + "paramValueMismatches": [], + "skipped": True, + "reason": "Runtime protocol params could not be extracted", + } + + runtimeParamsByProtocolId = runtimeSnapshot["paramsByProtocolId"] + postgresqlParamsByProtocolId = postgresqlSnapshot["paramsByProtocolId"] + + runtimeParams = derivedSets["runtimeParams"] + postgresqlParams = derivedSets["postgresqlParams"] + + missingParams = sorted( + runtimeParams - postgresqlParams, + key=self.paramSortKey, + ) + extraParams = sorted( + postgresqlParams - runtimeParams, + key=self.paramSortKey, + ) + + paramValueMismatches = [] + for key in sorted(runtimeParams.intersection(postgresqlParams), key=self.paramSortKey): + protocolId, paramName = key + + runtimeParam = runtimeParamsByProtocolId.get(protocolId, {}).get(paramName, {}) + postgresqlParam = postgresqlParamsByProtocolId.get(protocolId, {}).get(paramName, {}) + + runtimeValue = self.normalizeParamValue(runtimeParam.get("value")) + postgresqlValue = self.normalizeParamValue(postgresqlParam.get("value")) + + if runtimeValue != postgresqlValue: + paramValueMismatches.append({ + "protocolId": protocolId, + "paramName": paramName, + "runtimeValue": runtimeValue, + "postgresqlValue": postgresqlValue, + }) + + return { + "missingParams": missingParams, + "extraParams": extraParams, + "paramValueMismatches": paramValueMismatches, + } + + def compareInputRefs( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeInputRefsByKey = runtimeSnapshot["inputRefsByKey"] + postgresqlInputRefsByKey = postgresqlSnapshot["inputRefsByKey"] + + runtimeInputRefs = derivedSets["runtimeInputRefs"] + postgresqlInputRefsKeys = derivedSets["postgresqlInputRefsKeys"] + + missingInputRefs = sorted( + runtimeInputRefs - postgresqlInputRefsKeys, + key=self.inputRefSortKey, + ) + extraInputRefs = sorted( + postgresqlInputRefsKeys - runtimeInputRefs, + key=self.inputRefSortKey, + ) + + inputRefMismatches = [] + for key in sorted(runtimeInputRefs.intersection(postgresqlInputRefsKeys), key=self.inputRefSortKey): + runtimeRef = runtimeInputRefsByKey.get(key, {}) + postgresqlRef = postgresqlInputRefsByKey.get(key, {}) + + changedFields = [] + + runtimeParentProtocolId = self.normalizeOptionalText(runtimeRef.get("parentProtocolId")) + postgresqlParentProtocolId = self.normalizeOptionalText(postgresqlRef.get("parentProtocolId")) + + runtimeParentOutputName = self.normalizeOptionalText(runtimeRef.get("parentOutputName")) + postgresqlParentOutputName = self.normalizeOptionalText(postgresqlRef.get("parentOutputName")) + + runtimeObjectClassName = self.normalizeOptionalText(runtimeRef.get("objectClassName")) + postgresqlObjectClassName = self.normalizeOptionalText(postgresqlRef.get("objectClassName")) + + if runtimeParentProtocolId != postgresqlParentProtocolId: + changedFields.append("parentProtocolId") + + if runtimeParentOutputName != postgresqlParentOutputName: + changedFields.append("parentOutputName") + + if ( + runtimeObjectClassName is not None + and postgresqlObjectClassName is not None + and runtimeObjectClassName != postgresqlObjectClassName + ): + changedFields.append("objectClassName") + + if changedFields: + protocolId, inputName, itemIndex = key + inputRefMismatches.append({ + "protocolId": protocolId, + "inputName": inputName, + "itemIndex": int(itemIndex), + "fields": changedFields, + "runtimeParentProtocolId": runtimeParentProtocolId, + "postgresqlParentProtocolId": postgresqlParentProtocolId, + "runtimeParentOutputName": runtimeParentOutputName, + "postgresqlParentOutputName": postgresqlParentOutputName, + "runtimeObjectClassName": runtimeObjectClassName, + "postgresqlObjectClassName": postgresqlObjectClassName, + }) + + return { + "missingInputRefs": missingInputRefs, + "extraInputRefs": extraInputRefs, + "inputRefMismatches": inputRefMismatches, + } + + def compareInputRefDependencies( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeDependencies = runtimeSnapshot["dependencies"] + postgresqlDependencies = postgresqlSnapshot["dependencies"] + + runtimeDependenciesFromInputRefs = derivedSets["runtimeDependenciesFromInputRefs"] + postgresqlDependenciesFromInputRefs = derivedSets["postgresqlDependenciesFromInputRefs"] + + runtimeInputRefDependenciesMissing = sorted( + runtimeDependenciesFromInputRefs - runtimeDependencies, + key=self.dependencySortKey, + ) + runtimeDependenciesWithoutInputRefs = sorted( + runtimeDependencies - runtimeDependenciesFromInputRefs, + key=self.dependencySortKey, + ) + + postgresqlInputRefDependenciesMissing = sorted( + postgresqlDependenciesFromInputRefs - postgresqlDependencies, + key=self.dependencySortKey, + ) + postgresqlDependenciesWithoutInputRefs = sorted( + postgresqlDependencies - postgresqlDependenciesFromInputRefs, + key=self.dependencySortKey, + ) + + return { + "runtimeInputRefDependenciesMissing": runtimeInputRefDependenciesMissing, + "runtimeDependenciesWithoutInputRefs": runtimeDependenciesWithoutInputRefs, + "postgresqlInputRefDependenciesMissing": postgresqlInputRefDependenciesMissing, + "postgresqlDependenciesWithoutInputRefs": postgresqlDependenciesWithoutInputRefs, + } + + def comparePostgresqlInputRefTargets( + self, + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + postgresqlInputRefsByKey = postgresqlSnapshot["inputRefsByKey"] + persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] + + postgresqlProtocolIds = derivedSets["postgresqlProtocolIds"] + + postgresqlInputRefsWithMissingParentProtocols = [] + postgresqlInputRefsWithMissingParentOutputs = [] + runtimeOutputs = derivedSets.get("runtimeOutputs", set()) + + for key in sorted(postgresqlInputRefsByKey.keys(), key=self.inputRefSortKey): + inputRef = postgresqlInputRefsByKey.get(key, {}) + + parentProtocolId = self.normalizeOptionalText(inputRef.get("parentProtocolId")) + parentOutputName = self.normalizeOptionalText(inputRef.get("parentOutputName")) + + if not parentProtocolId or parentProtocolId == "PROJECT": + continue + + if parentProtocolId not in postgresqlProtocolIds: + issue = self.buildInputRef(key, inputRef) + issue["missingParentProtocolId"] = parentProtocolId + postgresqlInputRefsWithMissingParentProtocols.append(issue) + continue + + if not parentOutputName: + continue + + if parentOutputName not in persistedOutputsByProtocolId.get(parentProtocolId, {}): + runtimeOutputKey = (parentProtocolId, parentOutputName) + + if runtimeOutputKey not in runtimeOutputs: + continue + + issue = self.buildInputRef(key, inputRef) + issue["missingParentOutputName"] = parentOutputName + postgresqlInputRefsWithMissingParentOutputs.append(issue) + + return { + "postgresqlInputRefsWithMissingParentProtocols": postgresqlInputRefsWithMissingParentProtocols, + "postgresqlInputRefsWithMissingParentOutputs": postgresqlInputRefsWithMissingParentOutputs, + } + + def buildIssues( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + protocolComparison: Dict[str, Any], + dependencyComparison: Dict[str, Any], + outputComparison: Dict[str, Any], + outputPayloadComparison: Dict[str, Any], + stepComparison: Dict[str, Any], + paramComparison: Dict[str, Any], + inputRefComparison: Dict[str, Any], + inputRefDependencyComparison: Dict[str, Any], + postgresqlInputRefTargetComparison: Dict[str, Any], + ) -> Dict[str, List[Dict[str, Any]]]: + runtimeStatuses = runtimeSnapshot["statuses"] + runtimeOutputsByProtocolId = runtimeSnapshot["outputsByProtocolId"] + runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] + runtimeInputRefsByKey = runtimeSnapshot["inputRefsByKey"] + runtimeParamsByProtocolId = runtimeSnapshot["paramsByProtocolId"] + + postgresqlStatuses = postgresqlSnapshot["statuses"] + persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] + normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] + postgresqlInputRefsByKey = postgresqlSnapshot["inputRefsByKey"] + postgresqlParamsByProtocolId = postgresqlSnapshot["paramsByProtocolId"] + + missingProtocolIds = protocolComparison["missingProtocolIds"] + extraProtocolIds = protocolComparison["extraProtocolIds"] + statusMismatches = protocolComparison["statusMismatches"] + protocolClassMismatches = protocolComparison["protocolClassMismatches"] + + missingDependencies = dependencyComparison["missingDependencies"] + extraDependencies = dependencyComparison["extraDependencies"] + + missingOutputs = outputComparison["missingOutputs"] + extraOutputs = outputComparison["extraOutputs"] + outputClassMismatches = outputComparison["outputClassMismatches"] + outputMapperKindMismatches = outputComparison["outputMapperKindMismatches"] + outputItemsCountMismatches = outputComparison["outputItemsCountMismatches"] + + postgresqlFlatSetOutputsWithIncompletePayload = ( + outputPayloadComparison["postgresqlFlatSetOutputsWithIncompletePayload"] + ) + postgresqlTreeOutputsWithIncompletePayload = ( + outputPayloadComparison["postgresqlTreeOutputsWithIncompletePayload"] + ) + + postgresqlFlatSetPropertiesMismatches = ( + outputPayloadComparison["postgresqlFlatSetPropertiesMismatches"] + ) + + missingSteps = stepComparison["missingSteps"] + extraSteps = stepComparison["extraSteps"] + stepMismatches = stepComparison["stepMismatches"] + + missingParams = paramComparison["missingParams"] + extraParams = paramComparison["extraParams"] + paramValueMismatches = paramComparison["paramValueMismatches"] + + missingInputRefs = inputRefComparison["missingInputRefs"] + extraInputRefs = inputRefComparison["extraInputRefs"] + inputRefMismatches = inputRefComparison["inputRefMismatches"] + + runtimeInputRefDependenciesMissing = inputRefDependencyComparison["runtimeInputRefDependenciesMissing"] + runtimeDependenciesWithoutInputRefs = inputRefDependencyComparison["runtimeDependenciesWithoutInputRefs"] + postgresqlInputRefDependenciesMissing = inputRefDependencyComparison["postgresqlInputRefDependenciesMissing"] + postgresqlDependenciesWithoutInputRefs = inputRefDependencyComparison["postgresqlDependenciesWithoutInputRefs"] + postgresqlInputRefsWithMissingParentProtocols = ( + postgresqlInputRefTargetComparison["postgresqlInputRefsWithMissingParentProtocols"] + ) + postgresqlInputRefsWithMissingParentOutputs = ( + postgresqlInputRefTargetComparison["postgresqlInputRefsWithMissingParentOutputs"] + ) + + postgresqlFlatSetItemsCountMismatches = ( + outputPayloadComparison["postgresqlFlatSetItemsCountMismatches"] + ) + + postgresqlFlatSetMaxItemIdMismatches = ( + outputPayloadComparison["postgresqlFlatSetMaxItemIdMismatches"] + ) + + postgresqlFlatSetColumnsCountMismatches = ( + outputPayloadComparison["postgresqlFlatSetColumnsCountMismatches"] + ) + + postgresqlFlatSetRootTableMismatches = ( + outputPayloadComparison["postgresqlFlatSetRootTableMismatches"] + ) + + postgresqlFlatSetRootObjectMismatches = ( + outputPayloadComparison["postgresqlFlatSetRootObjectMismatches"] + ) + + return { + "missingProtocols": [ + { + "protocolId": protocolId, + "runtimeStatus": runtimeStatuses.get(protocolId, ""), + } + for protocolId in missingProtocolIds + ], + "extraProtocols": [ + { + "protocolId": protocolId, + "postgresqlStatus": postgresqlStatuses.get(protocolId, ""), + } + for protocolId in extraProtocolIds + ], + "statusMismatches": statusMismatches, + "protocolClassMismatches": protocolClassMismatches, + "missingDependencies": [ + self.buildDependency(parentId, childId) + for parentId, childId in missingDependencies + ], + "extraDependencies": [ + self.buildDependency(parentId, childId) + for parentId, childId in extraDependencies + ], + "missingOutputs": [ + self.buildMissingOutputIssue( + protocolId, + outputName, + runtimeOutputsByProtocolId, + ) + for protocolId, outputName in missingOutputs + ], + "extraOutputs": [ + self.buildExtraOutputIssue( + protocolId, + outputName, + persistedOutputsByProtocolId, + ) + for protocolId, outputName in extraOutputs + ], + "outputClassMismatches": outputClassMismatches, + "outputMapperKindMismatches": outputMapperKindMismatches, + "outputItemsCountMismatches": outputItemsCountMismatches, + "postgresqlFlatSetOutputsWithIncompletePayload": postgresqlFlatSetOutputsWithIncompletePayload, + "postgresqlTreeOutputsWithIncompletePayload": postgresqlTreeOutputsWithIncompletePayload, + "postgresqlFlatSetItemsCountMismatches": postgresqlFlatSetItemsCountMismatches, + "missingSteps": [ + self.buildStep( + protocolId, + stepIndex, + runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}), + ) + for protocolId, stepIndex in missingSteps + ], + "extraSteps": [ + self.buildStep( + protocolId, + stepIndex, + normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}), + ) + for protocolId, stepIndex in extraSteps + ], + "stepMismatches": stepMismatches, + "missingInputRefs": [ + self.buildInputRef(key, runtimeInputRefsByKey.get(key, {})) + for key in missingInputRefs + ], + "extraInputRefs": [ + self.buildInputRef(key, postgresqlInputRefsByKey.get(key, {})) + for key in extraInputRefs + ], + "inputRefMismatches": inputRefMismatches, + "postgresqlInputRefsWithMissingParentProtocols": postgresqlInputRefsWithMissingParentProtocols, + "postgresqlInputRefsWithMissingParentOutputs": postgresqlInputRefsWithMissingParentOutputs, + "runtimeInputRefDependenciesMissing": [ + self.buildDependency(parentId, childId) + for parentId, childId in runtimeInputRefDependenciesMissing + ], + "runtimeDependenciesWithoutInputRefs": [ + self.buildDependency(parentId, childId) + for parentId, childId in runtimeDependenciesWithoutInputRefs + ], + "postgresqlInputRefDependenciesMissing": [ + self.buildDependency(parentId, childId) + for parentId, childId in postgresqlInputRefDependenciesMissing + ], + "postgresqlDependenciesWithoutInputRefs": [ + self.buildDependency(parentId, childId) + for parentId, childId in postgresqlDependenciesWithoutInputRefs + ], + "missingParams": [ + self.buildParamIssue(key, runtimeParamsByProtocolId.get(key[0], {}).get(key[1], {})) + for key in missingParams + ], + "extraParams": [ + self.buildParamIssue(key, postgresqlParamsByProtocolId.get(key[0], {}).get(key[1], {})) + for key in extraParams + ], + "paramValueMismatches": paramValueMismatches, + "postgresqlFlatSetMaxItemIdMismatches": postgresqlFlatSetMaxItemIdMismatches, + "postgresqlFlatSetColumnsCountMismatches": postgresqlFlatSetColumnsCountMismatches, + "postgresqlFlatSetRootTableMismatches": postgresqlFlatSetRootTableMismatches, + "postgresqlFlatSetPropertiesMismatches": postgresqlFlatSetPropertiesMismatches, + "postgresqlFlatSetRootObjectMismatches": postgresqlFlatSetRootObjectMismatches, + } + + def buildSummary( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + issuesCount: int, + ) -> Dict[str, int]: + return { + "runtimeProtocols": len(derivedSets["runtimeProtocolIds"]), + "postgresqlProtocols": len(derivedSets["postgresqlProtocolIds"]), + "runtimeDependencies": len(runtimeSnapshot["dependencies"]), + "postgresqlDependencies": len(postgresqlSnapshot["dependencies"]), + "runtimeOutputs": len(derivedSets["runtimeOutputs"]), + "postgresqlOutputs": len(derivedSets["postgresqlOutputs"]), + "runtimeSteps": len(derivedSets["runtimeSteps"]), + "postgresqlSteps": len(derivedSets["postgresqlSteps"]), + "runtimeInputRefs": len(derivedSets["runtimeInputRefs"]), + "postgresqlInputRefs": len(derivedSets["postgresqlInputRefsKeys"]), + "runtimeInputRefDependencies": len(derivedSets["runtimeDependenciesFromInputRefs"]), + "postgresqlInputRefDependencies": len(derivedSets["postgresqlDependenciesFromInputRefs"]), + "runtimeParams": len(derivedSets["runtimeParams"]), + "postgresqlParams": len(derivedSets["postgresqlParams"]), + "issues": issuesCount, + } + + def buildComparisons( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + projectId: Optional[int] = None, + ) -> Dict[str, Dict[str, Any]]: + protocolComparison = self.compareProtocols( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + + dependencyComparison = self.compareDependencies( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + ) + + outputComparison = self.compareOutputs( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + + outputPayloadComparison = self.comparePostgresqlOutputPayloads( + postgresqlSnapshot=postgresqlSnapshot, + projectId=projectId, + ) + + stepComparison = self.compareSteps( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + + paramComparison = self.compareParams( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + + inputRefComparison = self.compareInputRefs( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + + inputRefDependencyComparison = self.compareInputRefDependencies( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + + postgresqlInputRefTargetComparison = self.comparePostgresqlInputRefTargets( + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + + return { + "protocolComparison": protocolComparison, + "dependencyComparison": dependencyComparison, + "outputComparison": outputComparison, + "stepComparison": stepComparison, + "paramComparison": paramComparison, + "inputRefComparison": inputRefComparison, + "inputRefDependencyComparison": inputRefDependencyComparison, + "postgresqlInputRefTargetComparison": postgresqlInputRefTargetComparison, + "outputPayloadComparison": outputPayloadComparison, + } + + def validateProjectPostgresqlConsistency( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser: dict, + refresh: bool = True, + checkPid: bool = True, + ) -> Dict[str, Any]: + dbProj = self.getProjectDbRow(mapper, projectId, currentUser) + if not dbProj: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Project not found", + ) + + self.loadProjectForThumbnails(dbProj) + + runtimeSnapshot = self.collectRuntimeSnapshot( + projectId=projectId, + refresh=refresh, + checkPid=checkPid, + ) + + postgresqlSnapshot = self.collectPostgresqlSnapshot( + mapper=mapper, + projectId=projectId, + ) + + derivedSets = self.buildDerivedSets( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + ) + + comparisons = self.buildComparisons( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + projectId=projectId, + ) + + issues = self.buildIssues( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + **comparisons, + ) + + issuesCount = sum(len(items) for items in issues.values()) + + summary = self.buildSummary( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + issuesCount=issuesCount, + ) + + return { + "ok": issuesCount == 0, + "projectId": projectId, + "checkedAt": datetime.utcnow().isoformat() + "Z", + "summary": summary, + "issues": issues, + } \ No newline at end of file diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index a9f634aa..a2eeed69 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -580,13 +580,803 @@ def _validateImportableScipionProject(self, sourcePath: Path) -> Dict[str, Any]: "status": statusValue or "active", } + def _shouldRegisterProtocolOutputs(self, protocol: Any) -> bool: + try: + outputs = list(protocol.iterOutputAttributes()) + except Exception: + return False + + if not outputs: + return False + + status = protocol.getStatus() + + if status == STATUS_FINISHED: + return True + + if status in (STATUS_LAUNCHED, STATUS_RUNNING, STATUS_SCHEDULED): + return True + + return False + + def _safeCall(self, obj: Any, methodName: str, default: Any = None) -> Any: + try: + method = getattr(obj, methodName, None) + if method is None: + return default + return method() + except Exception: + return default + + def _getScipionClassName(self, obj: Any) -> Optional[str]: + if obj is None: + return None + + className = self._safeCall(obj, "getClassName", None) + if className: + return str(className) + + return obj.__class__.__name__ + + def _getScipionObjectId(self, obj: Any) -> Optional[Any]: + return self._safeCall(obj, "getObjId", None) + + def _iterProtocolInputPointers(self, pointer: Any) -> List[Any]: + if pointer is None: + return [] + + if isinstance(pointer, (list, tuple, set)): + return list(pointer) + + try: + if not isinstance(pointer, (str, bytes, dict)): + items = list(pointer) + if items: + return items + except Exception: + pass + + return [pointer] + + def _getPointerTargetObject(self, pointer: Any) -> Any: + if pointer is None: + return None + + target = self._safeCall(pointer, "get", None) + if target is not None: + return target + + return pointer + + def _getPointerParentProtocolId(self, pointer: Any, targetObj: Any) -> Optional[Any]: + pointerObj = self._safeCall(pointer, "getObjValue", None) + parentProtocolId = self._safeCall(pointerObj, "getObjId", None) + if parentProtocolId is not None: + return parentProtocolId + + parentObj = self._safeCall(targetObj, "getObjParent", None) + parentProtocolId = self._safeCall(parentObj, "getObjId", None) + if parentProtocolId is not None: + return parentProtocolId + + return None + + # ------------------------------------------------------------------ + # Scipion runtime protocol resolution + # ------------------------------------------------------------------ + def _resolveScipionProtocolId( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + ) -> int: + rawProtocolId = str(protocolId).strip() + + if not rawProtocolId: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Missing protocol id", + ) + + if mapper is None or projectId is None: + try: + return int(rawProtocolId) + except Exception: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid Scipion protocol id: {protocolId}", + ) + + protocolDbIdCandidate = None + try: + protocolDbIdCandidate = int(rawProtocolId) + except Exception: + protocolDbIdCandidate = None + + try: + if protocolDbIdCandidate is not None: + row = mapper.db.fetchOne( + """ + SELECT "protocolId" + FROM protocols + WHERE "projectId" = %s + AND (id = %s OR "protocolId" = %s) + LIMIT 1 + """, + (projectId, protocolDbIdCandidate, rawProtocolId), + ) + else: + row = mapper.db.fetchOne( + """ + SELECT "protocolId" + FROM protocols + WHERE "projectId" = %s + AND "protocolId" = %s + LIMIT 1 + """, + (projectId, rawProtocolId), + ) + + if row: + value = row.get("protocolId") if isinstance(row, dict) else row[0] + if value is not None: + return int(value) + + except HTTPException: + raise + except Exception: + logger.exception( + "Failed to resolve Scipion protocol id from PostgreSQL. projectId=%s protocolId=%s", + projectId, + protocolId, + ) + + try: + return int(rawProtocolId) + except Exception: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol not found in PostgreSQL: {protocolId}", + ) + + def _getScipionProtocolByRuntimeId( + self, + protocolId: Union[int, str], + ): + if self.currentProject is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="No current Scipion project loaded", + ) + + try: + protocol = self.currentProject.getProtocol(int(protocolId)) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol not found in Scipion runtime: {protocolId}. {e}", + ) + + if protocol is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol not found in Scipion runtime: {protocolId}", + ) + + return protocol + + def _tryGetScipionProtocolByRuntimeId( + self, + protocolId: Union[int, str], + ): + try: + return self._getScipionProtocolByRuntimeId(protocolId) + except Exception: + return None + + def _getScipionProtocolForRuntime( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + ): + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + return self._getScipionProtocolByRuntimeId(scipionProtocolId) + + def _tryGetScipionProtocolForRuntime( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + ): + try: + return self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + except Exception: + return None + + def _resolvePostgresqlProtocolDbId( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + ) -> Optional[int]: + if mapper is None or projectId is None or protocolId is None: + return None + + rawProtocolId = str(protocolId).strip() + if not rawProtocolId: + return None + + protocolDbIdCandidate = None + try: + protocolDbIdCandidate = int(rawProtocolId) + except Exception: + protocolDbIdCandidate = None + + try: + if protocolDbIdCandidate is not None: + row = mapper.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE "projectId" = %s + AND (id = %s OR "protocolId" = %s) + LIMIT 1 + """, + (projectId, protocolDbIdCandidate, rawProtocolId), + ) + else: + row = mapper.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE "projectId" = %s + AND "protocolId" = %s + LIMIT 1 + """, + (projectId, rawProtocolId), + ) + + if row: + value = row.get("id") if isinstance(row, dict) else row[0] + if value is not None: + return int(value) + + except Exception: + logger.exception( + "Failed to resolve PostgreSQL protocol id. projectId=%s protocolId=%s", + projectId, + protocolId, + ) + + return None + + def _raisePostgresqlViewerUnavailable( + self, + viewerName: str, + projectId: int, + protocolId: Union[int, str], + outputName: str, + reason: Optional[str] = None, + **extra, + ) -> None: + extraText = " ".join( + "%s=%s" % (key, value) + for key, value in extra.items() + if value is not None + ) + + logger.warning( + "%s output is not available in PostgreSQL metadata. projectId=%s protocolId=%s outputName=%s reason=%s %s", + viewerName, + projectId, + protocolId, + outputName, + reason, + extraText, + ) + + detail = "%s output is not available in PostgreSQL metadata" % viewerName + if reason: + detail = "%s: %s" % (detail, reason) + + raise HTTPException( + status_code=404, + detail=detail, + ) + + def _resolvePostgresqlReaderProtocolId( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + ) -> Union[int, str]: + if mapper is None: + return protocolId + + return self._resolvePostgresqlProtocolDbId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) or protocolId + + def _storeGeneratedSetInPostgresql( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + outputName: str, + scipionSet, + contextLabel: str, + ) -> Dict[str, Any]: + postgresqlSync = None + postgresqlError = None + + if mapper is None: + return { + "postgresqlSync": postgresqlSync, + "postgresqlError": postgresqlError, + } + + try: + from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + protocolDbId = self._resolvePostgresqlProtocolDbId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) or protocolId + + setMapper = ScipionSetPostgresqlMapper(mapper.db) + postgresqlSync = setMapper.storeSet( + projectId=projectId, + protocolDbId=protocolDbId, + outputName=outputName, + scipionSet=scipionSet, + ) + + except Exception as e: + postgresqlError = str(e) + logger.exception( + "Failed to persist generated %s output to PostgreSQL. projectId=%s protocolId=%s outputName=%s", + contextLabel, + projectId, + protocolId, + outputName, + ) + + return { + "postgresqlSync": postgresqlSync, + "postgresqlError": postgresqlError, + } + + def _deletePersistedProtocolOutputsFromPostgresql( + self, + mapper, + projectId: int, + protocolId: Union[int, str], + ) -> Dict[str, Any]: + protocolDbId = self._resolvePostgresqlProtocolDbId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + if protocolDbId is None: + return { + "protocolDbId": None, + "setsDeleted": 0, + "objectsDeleted": 0, + "skipped": True, + "reason": "protocol_not_found", + } + + setRows = mapper.db.fetchAll( + """ + SELECT id + FROM scipion_sets + WHERE "projectId" = %s + AND "protocolDbId" = %s + """, + (projectId, protocolDbId), + ) + + setIds = [ + int(row.get("id") if isinstance(row, dict) else row[0]) + for row in (setRows or []) + if (row.get("id") if isinstance(row, dict) else row[0]) is not None + ] + + setsDeleted = 0 + objectsDeleted = 0 + + with mapper.db.transaction(): + if setIds: + mapper.db.execute( + """ + DELETE FROM scipion_set_table_items + WHERE "tableId" IN ( + SELECT id + FROM scipion_set_tables + WHERE "setId" = ANY(%s) + ) + """, + (setIds,), + commit=False, + ) + + mapper.db.execute( + """ + DELETE FROM scipion_set_table_columns + WHERE "tableId" IN ( + SELECT id + FROM scipion_set_tables + WHERE "setId" = ANY(%s) + ) + """, + (setIds,), + commit=False, + ) + + mapper.db.execute( + """ + DELETE FROM scipion_set_tables + WHERE "setId" = ANY(%s) + """, + (setIds,), + commit=False, + ) + + mapper.db.execute( + """ + DELETE FROM scipion_set_items + WHERE "setId" = ANY(%s) + """, + (setIds,), + commit=False, + ) + + mapper.db.execute( + """ + DELETE FROM scipion_set_columns + WHERE "setId" = ANY(%s) + """, + (setIds,), + commit=False, + ) + + mapper.db.execute( + """ + DELETE FROM scipion_set_properties + WHERE "setId" = ANY(%s) + """, + (setIds,), + commit=False, + ) + + cur = mapper.db.execute( + """ + DELETE FROM scipion_sets + WHERE id = ANY(%s) + """, + (setIds,), + commit=False, + ) + setsDeleted = int(cur.rowcount or 0) + + cur = mapper.db.execute( + """ + WITH RECURSIVE object_tree AS ( + SELECT id + FROM scipion_objects + WHERE "projectId" = %s + AND "protocolDbId" = %s + + UNION ALL + + SELECT child.id + FROM scipion_objects child + JOIN object_tree parent + ON child."parentObjectId" = parent.id + ) + DELETE FROM scipion_objects + WHERE id IN (SELECT id FROM object_tree) + """, + (projectId, protocolDbId), + commit=False, + ) + objectsDeleted = int(cur.rowcount or 0) + + return { + "protocolDbId": protocolDbId, + "setsDeleted": setsDeleted, + "objectsDeleted": objectsDeleted, + "skipped": False, + } + + def _deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql( + self, + mapper, + projectId: int, + protocols: List[Any], + ) -> Dict[str, Any]: + cleanupItems = [] + totalSetsDeleted = 0 + totalObjectsDeleted = 0 + + for protocol in protocols or []: + protocolId = None + + try: + protocolId = protocol.getObjId() + except Exception: + protocolId = protocol + + if protocolId is None: + continue + + cleanupInfo = self._deletePersistedProtocolOutputsFromPostgresql( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + cleanupItems.append({ + "protocolId": str(protocolId), + **cleanupInfo, + }) + + totalSetsDeleted += int(cleanupInfo.get("setsDeleted") or 0) + totalObjectsDeleted += int(cleanupInfo.get("objectsDeleted") or 0) + + return { + "protocolsCount": len(cleanupItems), + "setsDeleted": totalSetsDeleted, + "objectsDeleted": totalObjectsDeleted, + "items": cleanupItems, + } + + def _getPointerOutputName(self, pointer: Any) -> Optional[str]: + outputName = self._safeCall(pointer, "getExtended", None) + if outputName is None: + return None + + outputNameText = str(outputName).strip() + return outputNameText or None + + def _buildProtocolInputRefsForPostgresql( + self, + projectId: int, + protocol: Any, + protocolDbIdByScipionId: Dict[str, int], + ) -> List[Dict[str, Any]]: + protocolId = self._getScipionObjectId(protocol) + if protocolId is None: + return [] + + protocolIdText = str(protocolId) + protocolDbId = protocolDbIdByScipionId.get(protocolIdText) + if protocolDbId is None: + return [] + + try: + inputAttributes = list(protocol.iterInputAttributes()) + except Exception: + return [] + + refs: List[Dict[str, Any]] = [] + + for inputName, pointer in inputAttributes: + pointerItems = self._iterProtocolInputPointers(pointer) + + for itemIndex, pointerItem in enumerate(pointerItems): + targetObj = self._getPointerTargetObject(pointerItem) + if targetObj is None: + continue + + parentProtocolId = self._getPointerParentProtocolId(pointerItem, targetObj) + parentProtocolIdText = str(parentProtocolId) if parentProtocolId is not None else None + parentProtocolDbId = ( + protocolDbIdByScipionId.get(parentProtocolIdText) + if parentProtocolIdText is not None + else None + ) + + refs.append({ + "projectId": int(projectId), + "protocolDbId": int(protocolDbId), + "protocolId": protocolIdText, + "inputName": str(inputName), + "itemIndex": int(itemIndex), + "parentProtocolDbId": parentProtocolDbId, + "parentProtocolId": parentProtocolIdText, + "parentOutputName": self._getPointerOutputName(pointerItem), + "objectClassName": self._getScipionClassName(targetObj), + "objectId": str(self._getScipionObjectId(targetObj)) + if self._getScipionObjectId(targetObj) is not None else None, + }) + + return refs + + def _safeProtocolStepValue(self, value: Any, default=None): + try: + if value is None: + return default + + if hasattr(value, "hasValue") and not value.hasValue(): + return default + + if hasattr(value, "get"): + try: + return value.get(default) + except TypeError: + return value.get() + + return value + except Exception: + return default + + def _safeProtocolStepCall(self, step: Any, methodName: str, default=None): + try: + method = getattr(step, methodName, None) + if not callable(method): + return default + value = method() + return value if value is not None else default + except Exception: + return default + + def _safeProtocolStepJsonValue(self, value: Any): + if value in (None, ""): + return None + + if isinstance(value, (dict, list, tuple)): + return value + + try: + return json.loads(str(value)) + except Exception: + return str(value) + + def _loadProtocolStepsForPostgresql(self, protocol: Any) -> List[Any]: + for methodName in ("loadSteps", "getSteps"): + try: + method = getattr(protocol, methodName, None) + if not callable(method): + continue + + steps = method() or [] + steps = list(steps) + if steps: + return steps + except Exception: + logger.debug( + "Could not load protocol steps using %s.", + methodName, + exc_info=True, + ) + + for attrName in ("_steps", "steps"): + try: + steps = getattr(protocol, attrName, None) + if not steps: + continue + + if isinstance(steps, dict): + steps = list(steps.values()) + else: + steps = list(steps) + + if steps: + return steps + except Exception: + logger.debug( + "Could not load protocol steps from attribute %s.", + attrName, + exc_info=True, + ) + + return [] + + def _buildProtocolStepsForPostgresql(self, protocol: Any) -> List[Dict[str, Any]]: + result: List[Dict[str, Any]] = [] + + steps = self._loadProtocolStepsForPostgresql(protocol) + if not steps: + return result + + for step in steps: + try: + stepIndex = self._safeProtocolStepCall(step, "getIndex", None) + if stepIndex is None: + continue + + elapsedSeconds = None + elapsed = self._safeProtocolStepCall(step, "getElapsedTime", None) + try: + if elapsed is not None: + elapsedSeconds = elapsed.total_seconds() + except Exception: + elapsedSeconds = None + + stepName = self._safeProtocolStepValue( + getattr(step, "funcName", None), + None, + ) + + if not stepName: + stepName = self._safeProtocolStepCall(step, "getClassName", "") + + prerequisites = [] + rawPrerequisites = self._safeProtocolStepCall( + step, + "getPrerequisites", + [], + ) + + try: + prerequisites = [ + int(prerequisite) + for prerequisite in (rawPrerequisites or []) + ] + except Exception: + prerequisites = [] + + rawArgs = self._safeProtocolStepValue( + getattr(step, "argsStr", None), + None, + ) + + needsGpu = self._safeProtocolStepCall(step, "needsGPU", None) + if needsGpu is None: + needsGpu = True + + result.append({ + "index": int(stepIndex), + "name": str(stepName or ""), + "status": self._safeProtocolStepCall(step, "getStatus", ""), + "prerequisites": prerequisites, + "args": self._safeProtocolStepJsonValue(rawArgs), + "initTime": self._safeProtocolStepValue( + getattr(step, "initTime", None), + None, + ), + "endTime": self._safeProtocolStepValue( + getattr(step, "endTime", None), + None, + ), + "elapsedSeconds": elapsedSeconds, + "error": self._safeProtocolStepCall(step, "getErrorMessage", None), + "interactive": bool( + self._safeProtocolStepCall(step, "isInteractive", False) + ), + "needsGpu": bool(needsGpu), + "event": "snapshot", + }) + except Exception: + logger.debug( + "Could not serialize protocol step for PostgreSQL.", + exc_info=True, + ) + + return result + def syncProjectProtocolsAndDependencies( self, mapper: PostgresqlFlatMapper, projectId: int, refresh: bool = False, checkPid: bool = False, - ) -> Dict[str, int]: + ) -> Dict[str, Any]: if self.currentProject is None: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -598,6 +1388,16 @@ def syncProjectProtocolsAndDependencies( protocolDbIdByScipionId: Dict[str, int] = {} currentProtocolIds: Set[str] = set() + protocolsByScipionId: Dict[str, Any] = {} + + outputSyncResults: List[Dict[str, Any]] = [] + outputSyncErrors: List[Dict[str, Any]] = [] + outputSyncDeclared: List[Dict[str, Any]] = [] + outputSyncMissing: List[Dict[str, Any]] = [] + + stepsSyncCount = 0 + stepsSyncProtocolsCount = 0 + stepsSyncErrors: List[Dict[str, Any]] = [] # 1) Save all protocol nodes that are currently present in the real Scipion graph for nodeId, nodeObj in nodesDict.items(): @@ -607,10 +1407,7 @@ def syncProjectProtocolsAndDependencies( protocol = getattr(nodeObj, "run", None) if protocol is None: - try: - protocol = self.currentProject.getProtocol(int(nodeId)) - except Exception: - protocol = None + protocol = self._tryGetScipionProtocolByRuntimeId(nodeId) if protocol is None: continue @@ -618,8 +1415,92 @@ def syncProjectProtocolsAndDependencies( protocolContext = self._buildProtocolContext(projectId, protocol) protocolDbId = mapper.saveProtocol(protocolContext) + try: + protocolSteps = self._buildProtocolStepsForPostgresql(protocol) + protocolScipionId = self._getScipionObjectId(protocol) + + if protocolSteps and protocolScipionId is not None: + mapper.replaceProtocolSteps( + projectId=projectId, + protocolDbId=int(protocolDbId), + protocolId=int(protocolScipionId), + steps=protocolSteps, + ) + + stepsSyncCount += len(protocolSteps) + stepsSyncProtocolsCount += 1 + except Exception as exc: + stepsSyncErrors.append({ + "protocolId": nodeIdText, + "error": str(exc), + }) + logger.exception( + "Failed to sync protocol steps. projectId=%s protocolId=%s", + projectId, + nodeIdText, + ) + + if self._shouldRegisterProtocolOutputs(protocol): + try: + outputReport = self.registerOutput( + projectId=projectId, + protocol=protocol, + mapper=mapper, + returnReport=True, + ) + + outputSyncResults.extend(outputReport.get("persisted") or []) + declaredOutputs = outputReport.get("declared") or [] + persistedOutputs = outputReport.get("persisted") or [] + skippedOutputs = outputReport.get("skipped") or [] + erroredOutputs = outputReport.get("errors") or [] + + for declaredOutput in declaredOutputs: + outputSyncDeclared.append({ + "protocolId": nodeIdText, + "outputName": declaredOutput.get("outputName"), + "outputClassName": declaredOutput.get("outputClassName"), + }) + + outputSyncMissing.extend( + self._buildMissingOutputSyncItems( + protocolId=nodeIdText, + declaredOutputs=declaredOutputs, + persistedOutputs=persistedOutputs, + skippedOutputs=skippedOutputs, + outputErrors=erroredOutputs, + ) + ) + + for skippedOutput in outputReport.get("skipped") or []: + outputSyncErrors.append({ + "protocolId": nodeIdText, + "outputName": skippedOutput.get("outputName"), + "outputClassName": skippedOutput.get("outputClassName"), + "reason": skippedOutput.get("reason"), + }) + + for outputError in outputReport.get("errors") or []: + outputSyncErrors.append({ + "protocolId": nodeIdText, + "outputName": outputError.get("outputName"), + "outputClassName": outputError.get("outputClassName"), + "error": outputError.get("error"), + }) + except Exception as exc: + outputSyncErrors.append({ + "protocolId": nodeIdText, + "error": str(exc), + }) + logger.exception( + "Failed to sync protocol outputs. projectId=%s protocolId=%s", + projectId, + nodeIdText, + ) + currentProtocolIds.add(nodeIdText) protocolDbIdByScipionId[nodeIdText] = int(protocolDbId) + protocolsByScipionId[nodeIdText] = protocol # 2) Purge stale protocol rows that are no longer present in the real graph mapper.deleteProjectProtocolsNotInProtocolIds( @@ -648,9 +1529,40 @@ def syncProjectProtocolsAndDependencies( savedEdges = mapper.replaceProjectProtocolDependencies(projectId, edges) + inputRefs: List[Dict[str, Any]] = [] + + for protocolIdText, protocol in protocolsByScipionId.items(): + inputRefs.extend( + self._buildProtocolInputRefsForPostgresql( + projectId=projectId, + protocol=protocol, + protocolDbIdByScipionId=protocolDbIdByScipionId, + ) + ) + + savedInputRefs = 0 + replaceInputRefs = getattr(mapper, "replaceProjectProtocolInputRefs", None) + if callable(replaceInputRefs): + savedInputRefs = replaceInputRefs(projectId, inputRefs) + + outputResultsByKind: Dict[str, int] = {} + for item in outputSyncResults: + mapperKind = str(item.get("mapperKind") or "unknown") + outputResultsByKind[mapperKind] = outputResultsByKind.get(mapperKind, 0) + 1 + return { "protocols": len(protocolDbIdByScipionId), "dependencies": int(savedEdges), + "inputRefs": int(savedInputRefs), + "steps": int(stepsSyncCount), + "stepsProtocols": int(stepsSyncProtocolsCount), + "stepErrors": stepsSyncErrors, + "outputsDeclared": len(outputSyncDeclared), + "outputs": len(outputSyncResults), + "outputsMissing": len(outputSyncMissing), + "outputsByKind": outputResultsByKind, + "outputMissing": outputSyncMissing, + "outputErrors": outputSyncErrors, } def syncProjectGraphAfterMutation( @@ -660,7 +1572,7 @@ def syncProjectGraphAfterMutation( actionLabel: str, refresh: bool = True, checkPid: bool = True, - ) -> Dict[str, int]: + ) -> Dict[str, Any]: try: return self.syncProjectProtocolsAndDependencies( mapper, @@ -996,6 +1908,8 @@ def listProjects(self, mapper: PostgresqlFlatMapper, currentUser) -> List[dict]: if not storedProjectPath: continue + projectId = dbProj["id"] + storedPathObj = Path(storedProjectPath).expanduser() if not storedPathObj.is_absolute(): storedPathObj = Path(self.manager.getProjectPath(str(storedPathObj))) @@ -1005,7 +1919,6 @@ def listProjects(self, mapper: PostgresqlFlatMapper, currentUser) -> List[dict]: try: realProjectPathObj = storedPathObj.resolve(strict=True) except FileNotFoundError: - # Broken entry or missing project on disk; keep it visible but degraded realProjectPathObj = storedPathObj except Exception: realProjectPathObj = storedPathObj @@ -1018,16 +1931,28 @@ def listProjects(self, mapper: PostgresqlFlatMapper, currentUser) -> List[dict]: except Exception: sizeGB = 0.0 - try: - protCount = self.countProtocols(runsPath) - except Exception: - protCount = 0 + protCount = None + + countProjectProtocols = getattr(mapper, "countProjectProtocols", None) + if callable(countProjectProtocols): + try: + protCount = int(countProjectProtocols(projectId) or 0) + except Exception: + logger.exception( + "Failed to count project protocols from PostgreSQL. projectId=%s", + projectId, + ) + + if protCount is None: + try: + protCount = self.countProtocols(runsPath) + except Exception: + protCount = 0 isOwner = dbProj.get("isOwner", dbProj.get("ownerId") == currentUser["id"]) isShared = dbProj.get("isShared", False) permission = dbProj.get("permission", "owner" if isOwner else "full") projectOwnerId = dbProj.get("ownerId") - projectId = dbProj["id"] updatedAt = dbProj.get("updatedAt") thumbnailVersion = self._buildProjectThumbnailVersion( @@ -1151,7 +2076,16 @@ def revokeProjectShareForUser( return {"success": True} - def getProjectById(self, mapper: PostgresqlFlatMapper, projectId: int, currentUser, refresh=True, checkPid=True) -> Optional[dict]: + def getProjectById( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser, + refresh=True, + checkPid=True, + validateConsistency: bool = False, + failOnConsistencyError: bool = False, + ) -> Optional[dict]: # Retrieve project from PostgreSQL using the mapper userId = currentUser["id"] dbProj = mapper.getProject(projectId=projectId, userId=userId) @@ -1161,7 +2095,48 @@ def getProjectById(self, mapper: PostgresqlFlatMapper, projectId: int, currentUs if not os.path.exists(projectPath): return None - return self.loadProject(dbProj, mapper, refresh=refresh, checkPid=checkPid) + project = self.loadProject(dbProj, mapper, refresh=refresh, checkPid=checkPid) + + if validateConsistency: + try: + from app.backend.api.services.project_consistency_service import ( + ProjectConsistencyService, + ) + + consistencyService = ProjectConsistencyService(self) + project["postgresqlConsistency"] = ( + consistencyService.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=projectId, + currentUser=currentUser, + refresh=refresh, + checkPid=checkPid, + ) + ) + except HTTPException: + if failOnConsistencyError: + raise + logger.exception( + "PostgreSQL consistency validation failed. projectId=%s", + projectId, + ) + project["postgresqlConsistency"] = { + "ok": False, + "error": "PostgreSQL consistency validation failed", + } + except Exception as e: + if failOnConsistencyError: + raise + logger.exception( + "PostgreSQL consistency validation failed. projectId=%s", + projectId, + ) + project["postgresqlConsistency"] = { + "ok": False, + "error": str(e), + } + + return project def getProjectEffectiveSettings( self, @@ -1170,13 +2145,7 @@ def getProjectEffectiveSettings( currentUser: Any, ) -> Dict[str, Any]: # getProjectEffectiveSettings - project = self.getProjectById( - mapper, - projectId, - currentUser, - refresh=False, - checkPid=False, - ) + project = self.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -1281,6 +2250,430 @@ def _buildProjectThumbnailVersion( return f"{projectId}:{updatedText}:{protocolsCount}:{runsMtime}" + def _loadPersistedOutputSummariesByProtocolId( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + ) -> Dict[str, Dict[str, Dict[str, Any]]]: + def toOptionalInt(value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except Exception: + return None + + result: Dict[str, Dict[str, Dict[str, Any]]] = {} + + setRows = mapper.db.fetchAll( + """ + SELECT + p."protocolId", + s.id, + s."objectId", + s."outputName", + s."setClassName", + s."itemClassName", + s.properties, + s."createdAt", + s."updatedAt" + FROM scipion_sets s + JOIN protocols p + ON p.id = s."protocolDbId" + WHERE s."projectId" = %s + ORDER BY p."protocolId", s."outputName" + """, + (projectId,), + ) + + for row in setRows: + protocolId = str(row.get("protocolId")) + outputName = str(row.get("outputName") or "") + if not protocolId or not outputName: + continue + + properties = row.get("properties") or {} + + result.setdefault(protocolId, {})[outputName] = { + "mapperKind": "flat_set", + "setId": row.get("id"), + "rootObjectId": row.get("objectId"), + "className": row.get("setClassName"), + "itemClassName": row.get("itemClassName"), + "itemsCount": toOptionalInt(properties.get("itemsCount")) if isinstance(properties, dict) else None, + "maxItemId": toOptionalInt(properties.get("maxItemId")) if isinstance(properties, dict) else None, + "columnsCount": toOptionalInt(properties.get("columnsCount")) if isinstance(properties, dict) else None, + "lastSyncAt": properties.get("lastSyncAt") if isinstance(properties, dict) else None, + "lastCheckedAt": properties.get("lastCheckedAt") if isinstance(properties, dict) else None, + "skippedLastSync": properties.get("skippedLastSync") if isinstance(properties, dict) else None, + "createdAt": row.get("createdAt"), + "updatedAt": row.get("updatedAt"), + } + + treeRows = mapper.db.fetchAll( + """ + SELECT + p."protocolId", + o.id, + o."scipionObjId", + o.name, + o.path, + o."className", + o.value, + o.label, + o.comment, + o.metadata, + o."createdAt", + o."updatedAt" + FROM scipion_objects o + JOIN protocols p + ON p.id = o."protocolDbId" + WHERE o."projectId" = %s + AND o."parentObjectId" IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM scipion_sets s + WHERE s."objectId" = o.id + ) + ORDER BY p."protocolId", o.path + """, + (projectId,), + ) + + for row in treeRows: + protocolId = str(row.get("protocolId")) + outputName = str(row.get("path") or row.get("name") or "") + if not protocolId or not outputName: + continue + + result.setdefault(protocolId, {})[outputName] = { + "mapperKind": "tree", + "rootObjectId": row.get("id"), + "scipionObjId": row.get("scipionObjId"), + "className": row.get("className"), + "value": row.get("value"), + "label": row.get("label"), + "comment": row.get("comment"), + "metadata": row.get("metadata") or {}, + "createdAt": row.get("createdAt"), + "updatedAt": row.get("updatedAt"), + } + + return result + + def _loadPersistedOutputsByProtocolId( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + ) -> Dict[str, Dict[str, Dict[str, Any]]]: + def toOptionalInt(value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except Exception: + return None + + result: Dict[str, Dict[str, Dict[str, Any]]] = {} + + setRows = mapper.db.fetchAll( + """ + SELECT + p.id AS "protocolDbId", + p."protocolId", + s.id, + s."objectId", + s."outputName", + s."setClassName", + s."itemClassName", + s.properties, + root_object.id AS "rootObjectDbId", + root_object."projectId" AS "rootObjectProjectId", + root_object."protocolDbId" AS "rootObjectProtocolDbId", + root_object."parentObjectId" AS "rootObjectParentObjectId", + root_object.name AS "rootObjectName", + root_object.path AS "rootObjectPath", + root_object."className" AS "rootObjectClassName", + COALESCE(items_stats."itemsTableCount", 0) AS "itemsTableCount", + items_stats."maxItemIdFromItems" AS "maxItemIdFromItems", + items_stats."itemsIdSignature" AS "itemsIdSignature", + items_stats."itemsValueSignature" AS "itemsValueSignature", + COALESCE(columns_stats."setColumnsCount", 0) AS "setColumnsCount", + columns_stats."setColumnsSignature" AS "setColumnsSignature", + COALESCE(root_table_stats."rootTablesCount", 0) AS "rootTablesCount", + root_table_stats."rootTableId" AS "rootTableId", + COALESCE(root_table_stats."rootTableItemsCount", 0) AS "rootTableItemsCount", + root_table_stats."rootTableMaxItemId" AS "rootTableMaxItemId", + root_table_stats."rootTableItemsIdSignature" AS "rootTableItemsIdSignature", + root_table_stats."rootTableItemsValueSignature" AS "rootTableItemsValueSignature", + COALESCE(root_table_columns_stats."rootTableColumnsCount", 0) AS "rootTableColumnsCount", + root_table_columns_stats."rootTableColumnsSignature" AS "rootTableColumnsSignature", + COALESCE(properties_payload_stats."propertiesPayloadCount", 0) AS "propertiesPayloadCount", + properties_payload_stats."propertiesPayloadSignature" AS "propertiesPayloadSignature", + COALESCE(set_properties_stats."setPropertiesCount", 0) AS "setPropertiesCount", + set_properties_stats."setPropertiesSignature" AS "setPropertiesSignature", + s."createdAt", + s."updatedAt" + FROM scipion_sets s + JOIN protocols p + ON p.id = s."protocolDbId" + LEFT JOIN scipion_objects root_object + ON root_object.id = s."objectId" + LEFT JOIN ( + SELECT + "setId", + COUNT(*)::int AS "itemsTableCount", + MAX("scipionItemId")::int AS "maxItemIdFromItems", + md5( + string_agg( + "scipionItemId"::text, + ',' + ORDER BY "scipionItemId" + ) + ) AS "itemsIdSignature", + md5( + string_agg( + jsonb_build_object( + 'scipionItemId', "scipionItemId", + 'enabled', enabled, + 'label', label, + 'comment', comment, + 'creation', creation, + 'values', "values" + )::text, + ',' + ORDER BY "scipionItemId" + ) + ) AS "itemsValueSignature" + FROM scipion_set_items + GROUP BY "setId" + ) items_stats + ON items_stats."setId" = s.id + LEFT JOIN ( + SELECT + "setId", + COUNT(*)::int AS "setColumnsCount", + jsonb_agg( + jsonb_build_object( + 'labelProperty', "labelProperty", + 'columnName', "columnName", + 'className', "className", + 'valueType', "valueType", + 'position', position, + 'indexed', indexed + ) + ORDER BY position ASC, "labelProperty" ASC + ) AS "setColumnsSignature" + FROM scipion_set_columns + GROUP BY "setId" + ) columns_stats + ON columns_stats."setId" = s.id + LEFT JOIN ( + SELECT + t."setId", + COUNT(DISTINCT t.id)::int AS "rootTablesCount", + MIN(t.id)::int AS "rootTableId", + COUNT(ti.id)::int AS "rootTableItemsCount", + MAX(ti."scipionItemId")::int AS "rootTableMaxItemId", + md5( + string_agg( + ti."scipionItemId"::text, + ',' + ORDER BY ti."scipionItemId" + ) FILTER (WHERE ti.id IS NOT NULL) + ) AS "rootTableItemsIdSignature", + md5( + string_agg( + jsonb_build_object( + 'scipionItemId', ti."scipionItemId", + 'enabled', ti.enabled, + 'label', ti.label, + 'comment', ti.comment, + 'creation', ti.creation, + 'values', ti."values" + )::text, + ',' + ORDER BY ti."scipionItemId" + ) FILTER (WHERE ti.id IS NOT NULL) + ) AS "rootTableItemsValueSignature" + FROM scipion_set_tables t + LEFT JOIN scipion_set_table_items ti + ON ti."tableId" = t.id + WHERE t."tableKind" = 'root' + GROUP BY t."setId" + ) root_table_stats + ON root_table_stats."setId" = s.id + LEFT JOIN ( + SELECT + t."setId", + COUNT(tc.id)::int AS "rootTableColumnsCount", + jsonb_agg( + jsonb_build_object( + 'labelProperty', tc."labelProperty", + 'columnName', tc."columnName", + 'className', tc."className", + 'valueType', tc."valueType", + 'position', tc.position, + 'indexed', tc.indexed + ) + ORDER BY tc.position ASC, tc."labelProperty" ASC + ) FILTER (WHERE tc.id IS NOT NULL) AS "rootTableColumnsSignature" + FROM scipion_set_tables t + LEFT JOIN scipion_set_table_columns tc + ON tc."tableId" = t.id + WHERE t."tableKind" = 'root' + GROUP BY t."setId" + ) root_table_columns_stats + ON root_table_columns_stats."setId" = s.id + LEFT JOIN ( + SELECT + s2.id AS "setId", + COUNT(*)::int AS "propertiesPayloadCount", + jsonb_agg( + jsonb_build_object( + 'key', stable_keys.key, + 'value', s2.properties ->> stable_keys.key + ) + ORDER BY stable_keys.key ASC + ) AS "propertiesPayloadSignature" + FROM scipion_sets s2 + CROSS JOIN ( + VALUES + ('columnsCount'), + ('itemsCount'), + ('nestedTablesVersion') + ) AS stable_keys(key) + WHERE s2.properties ? stable_keys.key + GROUP BY s2.id + ) properties_payload_stats + ON properties_payload_stats."setId" = s.id + LEFT JOIN ( + SELECT + "setId", + COUNT(*)::int AS "setPropertiesCount", + jsonb_agg( + jsonb_build_object( + 'key', key, + 'value', value + ) + ORDER BY key ASC + ) AS "setPropertiesSignature" + FROM scipion_set_properties + WHERE key IN ( + 'columnsCount', + 'itemsCount', + 'nestedTablesVersion' + ) + GROUP BY "setId" + ) set_properties_stats + ON set_properties_stats."setId" = s.id + WHERE s."projectId" = %s + ORDER BY p."protocolId", s."outputName" + """, + (projectId,), + ) + + for row in setRows: + protocolId = str(row.get("protocolId")) + outputName = str(row.get("outputName") or "") + if not protocolId or not outputName: + continue + + properties = row.get("properties") or {} + + result.setdefault(protocolId, {})[outputName] = { + "mapperKind": "flat_set", + "setId": row.get("id"), + "protocolDbId": toOptionalInt(row.get("protocolDbId")), + "rootObjectId": row.get("objectId"), + "rootObjectDbId": toOptionalInt(row.get("rootObjectDbId")), + "rootObjectProjectId": toOptionalInt(row.get("rootObjectProjectId")), + "rootObjectProtocolDbId": toOptionalInt(row.get("rootObjectProtocolDbId")), + "rootObjectParentObjectId": toOptionalInt(row.get("rootObjectParentObjectId")), + "rootObjectName": row.get("rootObjectName"), + "rootObjectPath": row.get("rootObjectPath"), + "rootObjectClassName": row.get("rootObjectClassName"), + "className": row.get("setClassName"), + "itemClassName": row.get("itemClassName"), + "itemsCount": toOptionalInt(properties.get("itemsCount")) if isinstance(properties, dict) else None, + "itemsTableCount": toOptionalInt(row.get("itemsTableCount")), + "maxItemIdFromItems": toOptionalInt(row.get("maxItemIdFromItems")), + "itemsIdSignature": row.get("itemsIdSignature"), + "itemsValueSignature": row.get("itemsValueSignature"), + "maxItemId": toOptionalInt(properties.get("maxItemId")) if isinstance(properties, dict) else None, + "columnsCount": toOptionalInt(properties.get("columnsCount")) if isinstance(properties, dict) else None, + "setColumnsCount": toOptionalInt(row.get("setColumnsCount")), + "setColumnsSignature": row.get("setColumnsSignature") or [], + "rootTablesCount": toOptionalInt(row.get("rootTablesCount")), + "rootTableId": toOptionalInt(row.get("rootTableId")), + "rootTableItemsCount": toOptionalInt(row.get("rootTableItemsCount")), + "rootTableMaxItemId": toOptionalInt(row.get("rootTableMaxItemId")), + "rootTableItemsIdSignature": row.get("rootTableItemsIdSignature"), + "rootTableItemsValueSignature": row.get("rootTableItemsValueSignature"), + "rootTableColumnsCount": toOptionalInt(row.get("rootTableColumnsCount")), + "rootTableColumnsSignature": row.get("rootTableColumnsSignature") or [], + "propertiesPayloadCount": toOptionalInt(row.get("propertiesPayloadCount")), + "propertiesPayloadSignature": row.get("propertiesPayloadSignature") or [], + "setPropertiesCount": toOptionalInt(row.get("setPropertiesCount")), + "setPropertiesSignature": row.get("setPropertiesSignature") or [], + "lastSyncAt": properties.get("lastSyncAt") if isinstance(properties, dict) else None, + "lastCheckedAt": properties.get("lastCheckedAt") if isinstance(properties, dict) else None, + "skippedLastSync": properties.get("skippedLastSync") if isinstance(properties, dict) else None, + "createdAt": row.get("createdAt"), + "updatedAt": row.get("updatedAt"), + } + + treeRows = mapper.db.fetchAll( + """ + SELECT + p."protocolId", + o.id, + o."scipionObjId", + o.name, + o.path, + o."className", + o.value, + o.label, + o.comment, + o.metadata, + o."createdAt", + o."updatedAt" + FROM scipion_objects o + JOIN protocols p + ON p.id = o."protocolDbId" + WHERE o."projectId" = %s + AND o."parentObjectId" IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM scipion_sets s + WHERE s."objectId" = o.id + ) + ORDER BY p."protocolId", o.path + """, + (projectId,), + ) + + for row in treeRows: + protocolId = str(row.get("protocolId")) + outputName = str(row.get("path") or row.get("name") or "") + if not protocolId or not outputName: + continue + + result.setdefault(protocolId, {})[outputName] = { + "mapperKind": "tree", + "rootObjectId": row.get("id"), + "scipionObjId": row.get("scipionObjId"), + "className": row.get("className"), + "value": row.get("value"), + "label": row.get("label"), + "comment": row.get("comment"), + "metadata": row.get("metadata") or {}, + "createdAt": row.get("createdAt"), + "updatedAt": row.get("updatedAt"), + } + + return result + def buildProtocolsGraph( self, projectId: int, @@ -1288,11 +2681,13 @@ def buildProtocolsGraph( tags: Dict[str, List[str]], dependencyMap: Optional[Dict[str, Dict[str, List[str]]]] = None, runMap: Optional[Dict[str, Any]] = None, + persistedOutputsByProtocolId: Optional[Dict[str, Dict[str, Dict[str, Any]]]] = None, ) -> dict: """Assemble protocol graph using PostgreSQL as source of truth for nodes + edges.""" graphData: Dict[str, Any] = {} adjacency = dependencyMap or {} liveRuns = runMap or {} + persistedOutputsByProtocolId = persistedOutputsByProtocolId or {} def sortKey(row: Dict[str, Any]): raw = str(row.get("protocolId") or "") @@ -1349,6 +2744,7 @@ def sortKey(row: Dict[str, Any]): continue nodeId = str(rawNodeId) + persistedOutputsByName = persistedOutputsByProtocolId.get(nodeId, {}) nodeDeps = adjacency.get(nodeId, {"parents": [], "children": []}) childrenIds = list(nodeDeps.get("children") or []) parentIds = list(nodeDeps.get("parents") or []) @@ -1376,10 +2772,7 @@ def sortKey(row: Dict[str, Any]): protocol = liveRuns.get(nodeId) if protocol is None: - try: - protocol = self.currentProject.getProtocol(int(nodeId)) - except Exception: - protocol = None + protocol = self._tryGetScipionProtocolByRuntimeId(nodeId) if protocol is not None: try: @@ -1469,7 +2862,12 @@ def sortKey(row: Dict[str, Any]): inputs = [] try: + seenOutputNames = set() + for key, attr in protocol.iterOutputAttributes(): + outputName = str(key) + seenOutputNames.add(outputName) + outputItem = {} outputItem["name"] = key outputItem["paramClass"] = "PointerParam" @@ -1487,7 +2885,27 @@ def sortKey(row: Dict[str, Any]): outputItem["value"] = "" outputItem["parentId"] = None + persistedOutput = persistedOutputsByName.get(outputName) + outputItem["persisted"] = bool(persistedOutput) + outputItem["persistence"] = persistedOutput + outputs.append(outputItem) + + for outputName, persistedOutput in persistedOutputsByName.items(): + if outputName in seenOutputNames: + continue + + outputs.append({ + "name": outputName, + "paramClass": "PointerParam", + "pointerClass": persistedOutput.get("className") or "", + "info": "", + "value": "%s.%s" % (nodeId, outputName), + "parentId": nodeId, + "persisted": True, + "persistence": persistedOutput, + }) + except Exception: outputs = [] else: @@ -1533,6 +2951,18 @@ def loadProject(self, dbProj: dict, mapper: PostgresqlFlatMapper = None, refresh scipionProtocolCount = 0 scipionEdgeCount = 0 + liveProtocolStatusById: Dict[str, str] = {} + activeOutputProtocolIds: Set[str] = set() + + def normalizeStatus(value: Any) -> str: + return str(value or "").strip().lower() + + activeOutputStatuses = { + normalizeStatus(STATUS_LAUNCHED), + normalizeStatus(STATUS_RUNNING), + normalizeStatus(STATUS_SCHEDULED), + } + try: runs = self.currentProject.getRunsGraph(refresh=refresh, checkPids=checkPid) nodesDict = getattr(runs, "_nodesDict", {}) or {} @@ -1542,7 +2972,24 @@ def loadProject(self, dbProj: dict, mapper: PostgresqlFlatMapper = None, refresh continue scipionProtocolCount += 1 - runMap[str(nodeId)] = getattr(nodeObj, "run", None) + protocol = getattr(nodeObj, "run", None) + runMap[str(nodeId)] = protocol + + if protocol is not None: + try: + liveStatus = protocol.getStatus() + liveProtocolStatusById[str(nodeId)] = normalizeStatus(liveStatus) + except Exception: + liveStatus = None + + try: + if ( + normalizeStatus(liveStatus) in activeOutputStatuses + and self._shouldRegisterProtocolOutputs(protocol) + ): + activeOutputProtocolIds.add(str(nodeId)) + except Exception: + pass for parent in getattr(nodeObj, "_parents", []) or []: parentNodeId = str(parent.getName()) @@ -1593,9 +3040,23 @@ def loadProject(self, dbProj: dict, mapper: PostgresqlFlatMapper = None, refresh dbProtocolCount = len(protocolRows) dbEdgeCount = sum(len(v.get("parents") or []) for v in dependencyMap.values()) + dbStatusByProtocolId = { + str(row.get("protocolId")): normalizeStatus(row.get("status")) + for row in protocolRows + if row.get("protocolId") is not None + } + + statusChangedProtocolIds = [ + protocolId + for protocolId, liveStatus in liveProtocolStatusById.items() + if dbStatusByProtocolId.get(protocolId) != liveStatus + ] + shouldResyncGraph = ( scipionProtocolCount != dbProtocolCount or - scipionEdgeCount != dbEdgeCount + scipionEdgeCount != dbEdgeCount or + bool(statusChangedProtocolIds) or + bool(activeOutputProtocolIds) ) if shouldResyncGraph: @@ -1625,12 +3086,27 @@ def loadProject(self, dbProj: dict, mapper: PostgresqlFlatMapper = None, refresh dbProj['id'], ) + persistedOutputsByProtocolId = {} + if mapper is not None: + try: + persistedOutputsByProtocolId = self._loadPersistedOutputSummariesByProtocolId( + mapper, + dbProj['id'], + ) + except Exception: + logger.exception( + "Failed to load persisted Scipion outputs for graph. projectId=%s", + dbProj['id'], + ) + persistedOutputsByProtocolId = {} + graphData = self.buildProtocolsGraph( dbProj['id'], protocolRows, tags, dependencyMap=dependencyMap, runMap=runMap, + persistedOutputsByProtocolId=persistedOutputsByProtocolId, ) stats = projPath.stat() @@ -1651,6 +3127,24 @@ def loadProject(self, dbProj: dict, mapper: PostgresqlFlatMapper = None, refresh "thumbnailItemsUrl": self.buildProjectThumbnailItemsUrl(dbProj['id']), } + def validateProjectPostgresqlConsistency( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser: dict, + refresh: bool = True, + checkPid: bool = True, + ) -> Dict[str, Any]: + from app.backend.api.services.project_consistency_service import ProjectConsistencyService + + return ProjectConsistencyService(self).validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=projectId, + currentUser=currentUser, + refresh=refresh, + checkPid=checkPid, + ) + def listProjectWorkflows(self, raw: bool = False): """ Return available workflow templates. @@ -2108,27 +3602,324 @@ def buildProjectThumbnailItemsUrl(projectId: int) -> str: # buildProjectThumbnailItemsUrl return f"/projects/{projectId}/thumbnail-items" - @staticmethod - def buildProtocolThumbnailUrl(projectId: int, protocolId: int) -> str: - # buildProtocolThumbnailUrl - return f"/projects/{projectId}/protocols/{protocolId}/thumbnail" + @staticmethod + def buildProtocolThumbnailUrl(projectId: int, protocolId: int) -> str: + # buildProtocolThumbnailUrl + return f"/projects/{projectId}/protocols/{protocolId}/thumbnail" + + @staticmethod + def buildProtocolThumbnailRebuildUrl(projectId: int, protocolId: int) -> str: + # buildProtocolThumbnailRebuildUrl + return f"/projects/{projectId}/protocols/{protocolId}/thumbnail/rebuild" + + @staticmethod + def buildProtocolOutputThumbnailUrl(projectId: int, protocolId: int, outputName: str) -> str: + # buildProtocolOutputThumbnailUrl + return f"/projects/{projectId}/protocols/{protocolId}/outputs/{outputName}/thumbnail" + + def _getPostgresqlStoredSetForIntegratedContext( + self, + mapper, + projectId: int, + protocolId: int, + outputName: str, + ) -> Optional[Dict[str, Any]]: + if mapper is None: + return None + + try: + from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + setMapper = ScipionSetPostgresqlMapper(mapper.db) + return setMapper.getStoredSet( + projectId=projectId, + protocolDbId=protocolId, + outputName=outputName, + limit=None, + offset=0, + ) + except Exception: + logger.debug( + "Could not load PostgreSQL stored set for integrated context. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + exc_info=True, + ) + return None + + def _getPostgresqlIntegratedKind(self, storedSet: Dict[str, Any]) -> Optional[str]: + classText = "%s %s" % ( + storedSet.get("setClassName") or "", + storedSet.get("itemClassName") or "", + ) + classText = classText.replace(" ", "").lower() + + if "setofcoordinates3d" in classText or "coordinate3d" in classText: + return "coordinates3d" + + if "setofctftomoseries" in classText or "ctftomoseries" in classText: + return "ctf" + + if "setoftiltseries" in classText or "tiltseries" in classText: + return "tiltSeries" + + if "setoftomograms" in classText or "tomogram" in classText: + return "tomogram" + + return None + + def _getPostgresqlStoredSetProperty( + self, + storedSet: Dict[str, Any], + key: str, + default=None, + ): + properties = storedSet.get("properties") or {} + + if isinstance(properties, str): + try: + properties = json.loads(properties) + except Exception: + properties = {} + + if isinstance(properties, dict) and key in properties: + return properties.get(key) + + for item in storedSet.get("setProperties") or []: + if str(item.get("key")) == str(key): + return item.get("value") + + return default + + def _buildPostgresqlIntegratedLink( + self, + projectId: int, + protocolId: int, + outputName: str, + storedSet: Dict[str, Any], + label: Optional[str] = None, + statusValue: str = "available", + ) -> Dict[str, Any]: + return { + "protocolId": protocolId, + "outputName": outputName, + "itemId": storedSet.get("objectId") or storedSet.get("id"), + "label": label or outputName, + "status": statusValue, + } + + def _buildPostgresqlIntegratedSummary( + self, + storedSet: Dict[str, Any], + extra: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + items = storedSet.get("items") or [] + itemsCount = self._getPostgresqlStoredSetProperty(storedSet, "itemsCount", None) + + if itemsCount is None: + itemsCount = len(items) + + summary = { + "objectClass": storedSet.get("setClassName") or storedSet.get("itemClassName"), + "objectId": storedSet.get("objectId") or storedSet.get("id"), + "size": itemsCount, + "fileName": self._getPostgresqlStoredSetProperty(storedSet, "fileName", None), + "samplingRate": self._getPostgresqlStoredSetProperty(storedSet, "samplingRate", None), + } + + if extra: + summary.update(extra) + + return self._safeScipionValue(summary) + + def _firstPostgresqlValueByName( + self, + values: Dict[str, Any], + names: List[str], + ): + normalizedNames = { + str(name).replace("_", "").replace(".", "").replace("-", "").lower() + for name in names + } + + for key, value in (values or {}).items(): + normalizedKey = str(key).replace("_", "").replace(".", "").replace("-", "").lower() + if normalizedKey in normalizedNames: + return value + + return None + + def _addPostgresqlIntegratedRelation( + self, + relationsByKey: Dict[str, Dict[str, Any]], + keyValue: Any, + **values: Any, + ) -> None: + key = str(keyValue) if keyValue is not None else "" + if not key: + return + + relation = relationsByKey.setdefault( + key, + { + "key": key, + "label": key, + }, + ) + + for name, value in values.items(): + if value is not None: + relation[name] = value + + def _addPostgresqlTiltSeriesRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tiltSeriesId = item.get("tiltSeriesId") or item.get("id") or index + label = item.get("label") or str(tiltSeriesId) + + self._addPostgresqlIntegratedRelation( + relationsByKey, + tiltSeriesId, + tiltSeriesId=tiltSeriesId, + label=label, + ) + + def _addPostgresqlCtftomoRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tiltSeriesId = item.get("tiltSeriesId") or item.get("id") or index + label = item.get("label") or str(tiltSeriesId) + + self._addPostgresqlIntegratedRelation( + relationsByKey, + tiltSeriesId, + ctfSeriesId=tiltSeriesId, + tiltSeriesId=tiltSeriesId, + label=label, + ) + + def _addPostgresqlTomogramRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tomogramId = ( + item.get("tomoId") + or item.get("tomogramId") + or item.get("id") + or item.get("label") + or index + ) + + label = item.get("label") or item.get("name") or str(tomogramId) + volumeId = item.get("tomogramVolumeId") or item.get("volumeId") or index + + self._addPostgresqlIntegratedRelation( + relationsByKey, + tomogramId, + tomogramId=tomogramId, + tomogramVolumeId=volumeId, + label=label, + ) + + def _addPostgresqlCoordinates3dRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tomogramId = ( + item.get("tomoId") + or item.get("tomogramId") + or item.get("id") + or item.get("label") + or index + ) + + label = item.get("label") or item.get("name") or str(tomogramId) + volumeId = item.get("tomogramVolumeId") or item.get("volumeId") or index + + self._addPostgresqlIntegratedRelation( + relationsByKey, + tomogramId, + coordinatesTomogramId=tomogramId, + tomogramId=tomogramId, + tomogramVolumeId=volumeId, + label=label, + ) + + def _getPostgresqlIntegratedAnalyzeContextIfAvailable( + self, + mapper, + projectId: int, + protocolId: int, + outputName: str, + ) -> Optional[Dict[str, Any]]: + if mapper is None: + return None + + try: + from app.backend.viewers.postgresql_integrated_context_reader import PostgresqlIntegratedContextReader + + readerProtocolId = self._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) - @staticmethod - def buildProtocolThumbnailRebuildUrl(projectId: int, protocolId: int) -> str: - # buildProtocolThumbnailRebuildUrl - return f"/projects/{projectId}/protocols/{protocolId}/thumbnail/rebuild" + reader = PostgresqlIntegratedContextReader( + db=mapper.db, + projectId=projectId, + protocolId=readerProtocolId, + outputName=outputName, + ) - @staticmethod - def buildProtocolOutputThumbnailUrl(projectId: int, protocolId: int, outputName: str) -> str: - # buildProtocolOutputThumbnailUrl - return f"/projects/{projectId}/protocols/{protocolId}/outputs/{outputName}/thumbnail" + return reader.getContext() + except Exception: + logger.debug( + "Could not build PostgreSQL integrated analyze context. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + exc_info=True, + ) + return None def getIntegratedAnalyzeContextService( self, projectId: int, protocolId: int, outputName: str, + mapper=None, ) -> Dict[str, Any]: + + pgContext = self._getPostgresqlIntegratedAnalyzeContextIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgContext is not None: + return pgContext + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Integrated Analyze Context", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="context_not_available", + ) + if self.currentProject is None: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -2600,14 +4391,22 @@ def addSetRelations(kind: str, obj: Any) -> None: def buildProtocolOutputThumbnail( self, - protocolId: int, + protocolId: Union[int, str], outputName: str, force: bool = False, size: int = 320, + mapper=None, + projectId: Optional[int] = None, ) -> Dict[str, Any]: - service = ThumbnailService(self.currentProject) - return service.buildProtocolOutputThumbnail( + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, protocolId=protocolId, + ) + + thumbnailService = ThumbnailService(self.currentProject) + return thumbnailService.buildProtocolOutputThumbnail( + protocolId=scipionProtocolId, outputName=outputName, force=force, size=size, @@ -2621,9 +4420,10 @@ def listProjectThumbnailItems( maxProtocols: int = 12, maxOutputsPerProtocol: int = 4, inlineImages: bool = False, + mapper=None, ): thumbnailService = ThumbnailService(self.currentProject) - return thumbnailService.listProtocolThumbnailItems( + items = thumbnailService.listProtocolThumbnailItems( projectId=projectId, force=force, size=size, @@ -2632,6 +4432,336 @@ def listProjectThumbnailItems( inlineImages=inlineImages, ) + if mapper is None: + return items + + validItems = [] + + for group in items or []: + if not isinstance(group, dict): + continue + + protocolIdValue = group.get("protocolId") + + try: + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolIdValue, + ) + protocol = self.currentProject.getProtocol(int(scipionProtocolId)) + except Exception: + logger.warning( + "Skipping thumbnail group because protocol was not found. " + "projectId=%s protocolId=%s group=%s", + projectId, + protocolIdValue, + group, + ) + continue + + validOutputs = [] + + for output in group.get("outputs") or []: + if not isinstance(output, dict): + continue + + outputNameValue = output.get("outputName") + if not outputNameValue: + continue + + if not hasattr(protocol, str(outputNameValue)): + logger.warning( + "Skipping thumbnail output because it does not belong to protocol. " + "projectId=%s protocolId=%s outputName=%s", + projectId, + protocolIdValue, + outputNameValue, + ) + continue + + validOutputs.append(output) + + if not validOutputs: + continue + + nextGroup = dict(group) + nextGroup["outputs"] = validOutputs + validItems.append(nextGroup) + + return validItems + + def _buildMissingOutputSyncItems( + self, + protocolId: Union[int, str], + declaredOutputs: List[Dict[str, Any]], + persistedOutputs: List[Dict[str, Any]], + skippedOutputs: List[Dict[str, Any]], + outputErrors: List[Dict[str, Any]], + ) -> List[Dict[str, Any]]: + persistedByName = { + str(item.get("outputName") or ""): item + for item in persistedOutputs or [] + if item.get("outputName") is not None + } + + skippedByName = { + str(item.get("outputName") or ""): item + for item in skippedOutputs or [] + if item.get("outputName") is not None + } + + errorsByName = { + str(item.get("outputName") or ""): item + for item in outputErrors or [] + if item.get("outputName") is not None + } + + missingOutputs: List[Dict[str, Any]] = [] + + for declaredOutput in declaredOutputs or []: + outputName = str(declaredOutput.get("outputName") or "") + if not outputName or outputName in persistedByName: + continue + + missingItem = { + "protocolId": str(protocolId), + "outputName": outputName, + "outputClassName": declaredOutput.get("outputClassName"), + } + + skippedOutput = skippedByName.get(outputName) + outputError = errorsByName.get(outputName) + + if skippedOutput is not None: + missingItem["reason"] = skippedOutput.get("reason") or "skipped" + elif outputError is not None: + missingItem["reason"] = "persistence_error" + missingItem["error"] = outputError.get("error") + else: + missingItem["reason"] = "not_persisted" + + missingOutputs.append(missingItem) + + return missingOutputs + + def registerOutput( + self, + projectId: int, + protocol: Any, + raiseOnError: bool = False, + mapper=None, + returnReport: bool = False, + ) -> Union[List[Dict[str, Any]], Dict[str, Any]]: + """ + Persist all Scipion protocol outputs in PostgreSQL. + + This method does not special-case concrete output classes. It decides + how to persist each output by object capabilities: + + - Set-like Scipion outputs -> ScipionSetPostgresqlMapper + - Regular Scipion objects -> ScipionObjectPostgresqlMapper + + Protocols are not stored here; only protocol outputs. + """ + from app.backend.database import getMapper + from app.backend.mapper import ScipionObjectPostgresqlMapper, ScipionSetPostgresqlMapper + + results: List[Dict[str, Any]] = [] + skippedOutputs: List[Dict[str, Any]] = [] + outputErrors: List[Dict[str, Any]] = [] + declaredOutputs: List[Dict[str, Any]] = [] + + closeMapper = mapper is None + if mapper is None: + mapper = getMapper() + + try: + protocolDbId = self._resolveProtocolDbIdForOutputPersistence( + mapper.db, + projectId, + protocol, + ) + + objectMapper = ScipionObjectPostgresqlMapper(mapper.db) + setMapper = ScipionSetPostgresqlMapper(mapper.db) + + for outputName, outputObj in protocol.iterOutputAttributes(): + outputClassName = self._getOutputClassName(outputObj) + + declaredOutputs.append({ + "outputName": outputName, + "outputClassName": outputClassName, + }) + + if outputObj is None: + skippedOutputs.append({ + "outputName": outputName, + "outputClassName": outputClassName, + "reason": "empty_output", + }) + continue + + try: + if self._isScipionSetOutput(outputObj): + result = setMapper.storeSet( + projectId=projectId, + protocolDbId=protocolDbId, + outputName=outputName, + scipionSet=outputObj, + ) + result["mapperKind"] = "flat_set" + + elif self._isScipionObjectOutput(outputObj): + result = objectMapper.storeObjectTree( + projectId=projectId, + protocolDbId=protocolDbId, + outputName=outputName, + scipionObj=outputObj, + includeNestedProperties=True, + ) + result["mapperKind"] = "tree" + + + else: + + skippedOutputs.append({ + "outputName": outputName, + "outputClassName": self._getOutputClassName(outputObj), + "reason": "unsupported_output_type", + }) + + continue + + result["outputName"] = outputName + result["outputClassName"] = self._getOutputClassName(outputObj) + results.append(result) + + except Exception as exc: + if raiseOnError: + raise + + outputErrors.append({ + "outputName": outputName, + "outputClassName": self._getOutputClassName(outputObj), + "error": str(exc), + }) + logger.warning( + "Could not persist Scipion output. projectId=%s protocolId=%s outputName=%s outputClass=%s error=%s", + projectId, + self._getScipionObjId(protocol), + outputName, + self._getOutputClassName(outputObj), + exc, + exc_info=True, + ) + finally: + if closeMapper: + mapper.db.close() + + if returnReport: + return { + "declared": declaredOutputs, + "persisted": results, + "skipped": skippedOutputs, + "errors": outputErrors, + } + + return results + + def _resolveProtocolDbIdForOutputPersistence(self, db, projectId: int, protocol: Any) -> int: + scipionProtocolId = self._getScipionObjId(protocol) + if scipionProtocolId is None: + raise ValueError("Cannot persist Scipion outputs without protocol getObjId()/getId()") + + row = db.fetchOne( + """ + SELECT id + FROM protocols + WHERE "projectId" = %s + AND "protocolId" = %s + """, + (projectId, str(scipionProtocolId)), + ) + if row is not None: + return int(row["id"]) + + row = db.fetchOne( + """ + SELECT id + FROM protocols + WHERE id = %s + AND "projectId" = %s + """, + (scipionProtocolId, projectId), + ) + if row is not None: + return int(row["id"]) + + raise ValueError( + "Protocol %s was not found in PostgreSQL protocols table for project %s" + % (scipionProtocolId, projectId) + ) + + def _isScipionSetOutput(self, outputObj: Any) -> bool: + className = self._getOutputClassName(outputObj) + + if className.startswith("SetOf"): + return True + + iterItems = getattr(outputObj, "iterItems", None) + if callable(iterItems): + return True + + return False + + def _isScipionObjectOutput(self, outputObj: Any) -> bool: + if outputObj is None: + return False + + getClassName = getattr(outputObj, "getClassName", None) + if callable(getClassName): + return True + + getAttributesToStore = getattr(outputObj, "getAttributesToStore", None) + if callable(getAttributesToStore): + return True + + return False + + def _getOutputClassName(self, outputObj: Any) -> str: + getClassName = getattr(outputObj, "getClassName", None) + if callable(getClassName): + try: + className = getClassName() + if className: + return str(className) + except Exception: + pass + + return outputObj.__class__.__name__ if outputObj is not None else "" + + def _getScipionObjId(self, obj: Any) -> Optional[int]: + for getterName in ("getObjId", "getId"): + getter = getattr(obj, getterName, None) + if not callable(getter): + continue + + try: + value = getter() + except Exception: + continue + + if value is None: + continue + + try: + return int(value) + except Exception: + continue + + return None + def _buildProtocolContext(self, projectId, protocol) -> dict: """ Build the common context dictionary for a protocol, @@ -3031,7 +5161,7 @@ def getNewProtocolParams(self, projectId, protocolClassName: str) -> dict: # getNewProtocolParams _invalidateNewProtocolCacheIfNeeded() - key = str(protocolClassName) + key = "%s:%s" % (str(projectId), str(protocolClassName)) with _newProtocolLock: cached = _newProtocolCache.get(key) @@ -3058,22 +5188,33 @@ def getNewProtocolParams(self, projectId, protocolClassName: str) -> dict: return copy.deepcopy(ctx) - def getProtocolParams(self, projectId: int, protocolId: int) -> dict: + def getProtocolParams( + self, + mapper, + projectId: int, + protocolId: int, + ) -> dict: """ Returns the parameters of an existing protocol given its ID. """ - protocol = self.currentProject.getProtocol(int(protocolId)) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + protocol.getPlugin() self.currentProject._fixProtParamsConfiguration(protocol) return self._buildProtocolContext(projectId, protocol) - def getNextProtocolSuggestions(self, protocolId): + def getNextProtocolSuggestions(self, mapper, projectId, protocolId): """ Returns the suggestions from the Scipion website for the next protocols to the protocol passed """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) protName = protocol.getClassName() try: url = Config.SCIPION_STATS_SUGGESTION % protName # protocol.getClassName() @@ -3156,7 +5297,13 @@ def castParamValue(self, param, rawValue): else: return rawValue - def applyParamsToProtocol(self, protocol, params): + def applyParamsToProtocol( + self, + mapper, + projectId: int, + protocol, + params, + ): """Apply pointer parameters to protocol.""" errorList = [] for key, value in params.items(): @@ -3172,7 +5319,12 @@ def applyParamsToProtocol(self, protocol, params): parentId, rawValue = v.split('.') if v else ("", "") if rawValue: try: - parentProtocol = self.currentProject.getProtocol(int(parentId)) + parentScipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=parentId, + ) + parentProtocol = self._getScipionProtocolByRuntimeId(parentScipionProtocolId) extended = rawValue newInputs.append(Pointer(parentProtocol, extended=extended)) logger.info(f"[INFO] Pointer param {key} set from parent {parentId} output {rawValue}") @@ -3189,9 +5341,14 @@ def applyParamsToProtocol(self, protocol, params): parentId, rawValue = value.split('.') if value else ("", "") if rawValue: try: - parentProtocol = self.currentProject.getProtocol(int(parentId)) - val = value - output = val.split('.')[-1] + parentScipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=parentId, + ) + parentProtocol = self._getScipionProtocolByRuntimeId(parentScipionProtocolId) + output = rawValue + val = f"{parentScipionProtocolId}.{output}" param.set(val) parentOutput = hasattr(parentProtocol, output) if parentOutput: @@ -3227,16 +5384,37 @@ def applyParamsToProtocol(self, protocol, params): param.set(None) return errorList - def setPointerParam(self, protocol, key, value, parentId): + def setPointerParam( + self, + mapper, + projectId: int, + protocol, + key, + value, + parentId, + ): """Resolve and set a pointer param from parent protocol outputs.""" param = protocol.getParam(key) if not isinstance(param, PointerParam): logger.warning(f"[WARN] Param {key} is not a PointerParam") return - parentProtocol = self.currentProject.getProtocol(int(parentId)) - param.set(value['editableValue']) + + parentScipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=parentId, + ) + parentProtocol = self._getScipionProtocolByRuntimeId(parentScipionProtocolId) + + editableValue = value.get("editableValue") if isinstance(value, dict) else value + + if editableValue and "." in str(editableValue): + _rawParentId, outputName = str(editableValue).split(".", 1) + editableValue = f"{parentScipionProtocolId}.{outputName}" + + param.set(editableValue) protocol.setAttributeValue(key, parentProtocol) - param.default.set(value['editableValue']) + param.default.set(editableValue) def saveProtocol(self, mapper, projectId, protocolId, protocolClassName, params, setToSave=True): errorList = [] @@ -3250,12 +5428,11 @@ def saveProtocol(self, mapper, projectId, protocolId, protocolClassName, params, ) protocol = self.currentProject.newProtocol(protClass) else: - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) protectedParams = ['_objComment', '_useQueue', '_prerequisites', 'gpuList', 'numberOfThreads'] for paramName in protectedParams: @@ -3296,7 +5473,12 @@ def saveProtocol(self, mapper, projectId, protocolId, protocolClassName, params, cleaned = re.sub(r'[^A-Za-z0-9\s+\-*/=<>!&|^%()\[\]{}_,.;:]', '', str(e)) errorList.append('**' + param.label.get() + '** ' + cleaned) - errorList += self.applyParamsToProtocol(protocol, params) + errorList += self.applyParamsToProtocol( + mapper=mapper, + projectId=projectId, + protocol=protocol, + params=params, + ) # Persist protocol in Scipion always. # The setToSave flag only controls whether we also sync the graph to PostgreSQL now. @@ -3340,7 +5522,13 @@ def saveProtocol(self, mapper, projectId, protocolId, protocolClassName, params, return protocol, errorList def listProtocolStepsService(self, mapper, projectId: int, protocolId: int): - return mapper.listProtocolSteps(projectId, protocolId) + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + return mapper.listProtocolSteps(projectId, scipionProtocolId) def updateProtocolStepStatusService( self, @@ -3364,22 +5552,13 @@ def updateProtocolStepStatusService( targetStatus = statusMap[normalizedStatus] - if self.currentProject is None: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="No current project loaded", - ) - - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - protocol = None - - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + protocol = self._getScipionProtocolByRuntimeId(scipionProtocolId) try: steps = protocol.loadSteps() or [] @@ -3438,7 +5617,7 @@ def updateProtocolStepStatusService( row = mapper.updateProtocolStepStatus( projectId=projectId, - protocolId=protocolId, + protocolId=scipionProtocolId, stepIndex=stepIndex, stepStatus=targetStatus, ) @@ -3456,6 +5635,11 @@ def launchProtocol(self, mapper, projectId, protocolId, protocolClassName, param Save, validate, and execute a protocol action. Supported execute modes: launch, restart, schedule, stop. """ + modeAliases = { + None: "launch", + "resume": "launch", + } + executeMode = modeAliases.get(executeMode, executeMode) allowedModes = {"launch", "restart", "schedule", "stop"} if executeMode not in allowedModes: raise HTTPException( @@ -3465,15 +5649,14 @@ def launchProtocol(self, mapper, projectId, protocolId, protocolClassName, param if executeMode == "stop": try: - self.stopProtocol([protocolId]) + self.stopProtocol(mapper, projectId, [protocolId]) - self.syncProjectProtocolsAndDependencies( + return self.syncProjectProtocolsAndDependencies( mapper, projectId, refresh=True, checkPid=True, ) - return except HTTPException: raise except Exception as e: @@ -3543,9 +5726,23 @@ def launchProtocol(self, mapper, projectId, protocolId, protocolClassName, param } runMode = modeToRunMode[executeMode] protocol.runMode.set(runMode) + + if executeMode == "restart": + cleanupInfo = self._deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql( + mapper=mapper, + projectId=projectId, + protocols=[protocol], + ) + logger.info( + "Deleted persisted protocol outputs before restart. projectId=%s protocolId=%s cleanup=%s", + projectId, + getattr(protocol, "getObjId", lambda: protocolId)(), + cleanupInfo, + ) + self.currentProject.launchProtocol(protocol) - self.syncProjectProtocolsAndDependencies( + return self.syncProjectProtocolsAndDependencies( mapper, projectId, refresh=True, @@ -3760,102 +5957,517 @@ def _buildProtocolsTreeInSubprocess(self) -> Dict[str, Any]: operationName="Build protocols tree", ) - def getProtocols(self, mapper: PostgresqlFlatMapper, projectId: int, currentUser) -> Optional[dict]: - # getProtocols - _invalidateProtocolsTreeCacheIfNeeded() + def getProtocols(self, mapper: PostgresqlFlatMapper, projectId: int, currentUser) -> Optional[dict]: + # getProtocols + _invalidateProtocolsTreeCacheIfNeeded() + + cacheKey = "protocolsTree" + with _protocolsTreeLock: + cached = _protocolsTreeCache.get(cacheKey) + if cached is not None: + tree = copy.deepcopy(cached) + self.walkAndReplaceProtocols(tree, self.getProtocolName) + return tree + + # computeFreshTreeOncePerRevision + protocolsTree = self._buildProtocolsTreeInSubprocess() + + with _protocolsTreeLock: + _protocolsTreeCache[cacheKey] = protocolsTree + + tree = copy.deepcopy(protocolsTree) + self.walkAndReplaceProtocols(tree, self.getProtocolName) + return tree + + def replaceDefaultProtocolText(self, node: dict, resolverFn): + # Determine type and extract text, tag, and children + if isinstance(node, dict): + text = node.get("text") + tag = node.get("tag") + children = node.get("childs", []) + else: + text = getattr(node, "text", None) + tag = getattr(node, "tag", None) + children = getattr(node, "childs", []) + + # Replace text if conditions are met + if text == "default" and tag == "protocol": + newText = resolverFn(node) + if newText: + if isinstance(node, dict): + node["text"] = newText + else: + setattr(node, "text", newText) + + # Recursively process children + for child in children: + self.replaceDefaultProtocolText(child, resolverFn) + + def walkAndReplaceProtocols(self, data: dict, resolverFn): + """ + Walk through the entire JSON/tree structure and replace 'default' texts for protocol nodes. + + Parameters: + - data: the root of the tree (can be a dictionary or a list of nodes) + - resolverFn: a function to determine the new text for 'default' protocol nodes + """ + if isinstance(data, dict): + # Iterate over key/value pairs in a dictionary + for key, value in data.items(): + if isinstance(value, dict): + self.replaceDefaultProtocolText(value, resolverFn) + elif isinstance(value, list): + for item in value: + if isinstance(item, dict): + self.replaceDefaultProtocolText(item, resolverFn) + elif isinstance(data, list): + for item in data: + if isinstance(item, dict): + self.replaceDefaultProtocolText(item, resolverFn) + + def _getProtocolClassForTreeLabel(self, protClassName: str): + try: + if self.currentProject is not None: + domain = self.currentProject.getDomain() + protocolClass = domain.getProtocols().get(protClassName, None) + if protocolClass is not None: + return protocolClass + except Exception: + pass + + try: + return Config.getDomain().getProtocols().get(protClassName, None) + except Exception: + logger.warning( + "Protocol className '%s' not found while resolving protocol tree label.", + protClassName, + ) + return None + + def getProtocolName(self, node): + text = node.get('text') + if text: + value = node.get('value') if node.get('value') is not None else text + protClassName = value.split('.')[-1] + prot = self._getProtocolClassForTreeLabel(protClassName) + + if node.get('tag') == 'protocol' and text == 'default': + if prot is None: + logger.warning("Protocol className '%s' not found!!!. \n" + "Fix your config/protocols.conf configuration." + % protClassName) + return + + text = prot.getClassLabel() + return text + + return 'default' + + def _resolvePostgresqlProjectPathForFilesystem( + self, + mapper, + projectId: int, + ) -> Optional[str]: + if mapper is None or not hasattr(mapper, "db"): + return None + + try: + row = mapper.db.fetchOne( + """ + SELECT name + FROM projects + WHERE id = %s + """, + (projectId,), + ) + except Exception: + logger.debug( + "Could not resolve project path from PostgreSQL. projectId=%s", + projectId, + exc_info=True, + ) + return None + + if not row: + return None + + rawPath = row.get("name") if isinstance(row, dict) else row[0] + if not rawPath: + return None + + try: + path = self._normalizeProjectPath(str(rawPath)) + except Exception: + path = str(rawPath) + + if not os.path.isabs(path): + try: + path = self.manager.getProjectPath(path) + except Exception: + pass + + return os.path.realpath(path) + + def _resolvePostgresqlProtocolRunPath( + self, + mapper, + projectId: int, + scipionProtocolId: Union[int, str], + ) -> Optional[str]: + projectPath = self._resolvePostgresqlProjectPathForFilesystem( + mapper=mapper, + projectId=projectId, + ) + if not projectPath: + return None + + runsPath = os.path.join(projectPath, "Runs") + if not os.path.isdir(runsPath): + return None + + runtimeIdText = str(scipionProtocolId).strip() + runtimeIdInt = None + try: + runtimeIdInt = int(runtimeIdText) + except Exception: + pass + + matches: List[str] = [] + + try: + for entry in os.scandir(runsPath): + if not entry.is_dir(): + continue + + name = entry.name + + if name.startswith("%s_" % runtimeIdText): + matches.append(entry.path) + continue + + if runtimeIdInt is not None: + match = re.match(r"^0*(\d+)_", name) + if match and int(match.group(1)) == runtimeIdInt: + matches.append(entry.path) + continue + except Exception: + logger.debug( + "Could not scan Runs folder for protocol logs. runsPath=%s", + runsPath, + exc_info=True, + ) + return None + + if not matches: + return None + + return sorted(matches)[0] + + @staticmethod + def _firstExistingLogPath( + protocolPath: str, + candidates: List[str], + ) -> Optional[str]: + for candidate in candidates: + path = os.path.join(protocolPath, candidate) + if os.path.exists(path): + return path + + return None + + def _resolvePostgresqlProtocolLogPaths( + self, + mapper, + projectId: int, + protocolId: int, + ) -> Optional[Dict[str, Any]]: + if mapper is None: + return None + + try: + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + except Exception: + logger.debug( + "Could not resolve PostgreSQL protocol id for logs. projectId=%s protocolId=%s", + projectId, + protocolId, + exc_info=True, + ) + return None + + protocolPath = self._resolvePostgresqlProtocolRunPath( + mapper=mapper, + projectId=projectId, + scipionProtocolId=scipionProtocolId, + ) + if not protocolPath: + return None + + logCandidates = { + "stdout": [ + "logs/run.stdout", + "logs/stdout.log", + "logs/stdout.txt", + "run.stdout", + "stdout.log", + "stdout.txt", + ], + "stderr": [ + "logs/run.stderr", + "logs/stderr.log", + "logs/stderr.txt", + "run.stderr", + "stderr.log", + "stderr.txt", + ], + "schedule": [ + "logs/schedule.log", + "logs/schedule.txt", + "logs/run.schedule", + "schedule.log", + "schedule.txt", + "run.schedule", + ], + } + + paths = { + channelId: self._firstExistingLogPath(protocolPath, candidates) + for channelId, candidates in logCandidates.items() + } + + # If we cannot find any known log file, do not guess. Let runtime fallback + # resolve the exact Scipion log paths. + if not any(paths.values()): + return None + + return { + "protocolId": scipionProtocolId, + "protocolPath": protocolPath, + "paths": paths, + } + + def _ensureRuntimeProjectForLogs( + self, + mapper, + projectId: int, + currentUser: Optional[dict], + ) -> None: + if getattr(self, "currentProject", None) is not None: + return + + if currentUser is None: + return + + self.getProjectById( + mapper, + projectId, + currentUser, + refresh=False, + checkPid=False, + ) + + @staticmethod + def _buildProtocolLogChannel( + channelId: str, + label: str, + order: int, + filePath: Optional[str], + ) -> Dict[str, Any]: + return { + "id": channelId, + "label": label, + "order": order, + } + + def _buildProtocolLogChannelsPayload( + self, + projectId: int, + protocolId: Union[int, str], + logPaths: Dict[str, Optional[str]], + ) -> Dict[str, Any]: + return { + "projectId": projectId, + "protocolId": int(protocolId), + "channels": [ + self._buildProtocolLogChannel("stdout", "Output", 1, logPaths.get("stdout")), + self._buildProtocolLogChannel("stderr", "Errors", 2, logPaths.get("stderr")), + self._buildProtocolLogChannel("schedule", "Schedule", 3, logPaths.get("schedule")), + ], + } + + @staticmethod + def _normalizeProtocolLogOffsets(rawOffsets: Dict[str, int]) -> Dict[str, int]: + if not isinstance(rawOffsets, dict): + return {"stdout": 0, "stderr": 0, "schedule": 0} + + keyMap = { + "stdout": "stdout", + "stdoutLog": "stdout", + "out": "stdout", + "stderr": "stderr", + "stderrLog": "stderr", + "err": "stderr", + "schedule": "schedule", + "scheduleLog": "schedule", + } + + normalized = {"stdout": 0, "stderr": 0, "schedule": 0} + for key, value in rawOffsets.items(): + canonical = keyMap.get(str(key), None) + if canonical is None: + continue + + try: + normalized[canonical] = max(0, int(value)) + except Exception: + normalized[canonical] = 0 + + return normalized + + @staticmethod + def _readProtocolLogChunk( + filePath: Optional[str], + startOffset: int, + maxBytes: Optional[int] = 65536, + maxLines: Optional[int] = 2000, + ) -> Dict[str, Any]: + if not filePath or not os.path.exists(filePath): + return { + "content": "", + "offset": int(startOffset or 0), + } + + try: + sizeBytes = int(os.path.getsize(filePath)) + except Exception: + sizeBytes = 0 + + safeOffset = int(startOffset or 0) + if safeOffset < 0: + safeOffset = 0 - cacheKey = "protocolsTree" - with _protocolsTreeLock: - cached = _protocolsTreeCache.get(cacheKey) - if cached is not None: - tree = copy.deepcopy(cached) - self.walkAndReplaceProtocols(tree, self.getProtocolName) - return tree + if safeOffset > sizeBytes: + safeOffset = 0 - # computeFreshTreeOncePerRevision - protocolsTree = self._buildProtocolsTreeInSubprocess() + bytesCap = None if maxBytes is None else max(1, int(maxBytes)) + linesCap = None if maxLines is None else max(1, int(maxLines)) - with _protocolsTreeLock: - _protocolsTreeCache[cacheKey] = protocolsTree + contentParts: List[str] = [] + bytesRead = 0 + linesRead = 0 - tree = copy.deepcopy(protocolsTree) - self.walkAndReplaceProtocols(tree, self.getProtocolName) - return tree + try: + with open(filePath, "rb") as handle: + handle.seek(safeOffset) - def replaceDefaultProtocolText(self, node: dict, resolverFn): - # Determine type and extract text, tag, and children - if isinstance(node, dict): - text = node.get("text") - tag = node.get("tag") - children = node.get("childs", []) - else: - text = getattr(node, "text", None) - tag = getattr(node, "tag", None) - children = getattr(node, "childs", []) + while True: + if linesCap is not None and linesRead >= linesCap: + break + if bytesCap is not None and bytesRead >= bytesCap: + break - # Replace text if conditions are met - if text == "default" and tag == "protocol": - newText = resolverFn(node) - if newText: - if isinstance(node, dict): - node["text"] = newText - else: - setattr(node, "text", newText) + posBefore = handle.tell() + lineBytes = handle.readline() + if not lineBytes: + break - # Recursively process children - for child in children: - self.replaceDefaultProtocolText(child, resolverFn) + if bytesCap is not None and (bytesRead + len(lineBytes)) > bytesCap: + handle.seek(posBefore) + break - def walkAndReplaceProtocols(self, data: dict, resolverFn): - """ - Walk through the entire JSON/tree structure and replace 'default' texts for protocol nodes. + contentParts.append(lineBytes.decode("utf-8", errors="ignore")) + bytesRead += len(lineBytes) + linesRead += 1 - Parameters: - - data: the root of the tree (can be a dictionary or a list of nodes) - - resolverFn: a function to determine the new text for 'default' protocol nodes - """ - if isinstance(data, dict): - # Iterate over key/value pairs in a dictionary - for key, value in data.items(): - if isinstance(value, dict): - self.replaceDefaultProtocolText(value, resolverFn) - elif isinstance(value, list): - for item in value: - if isinstance(item, dict): - self.replaceDefaultProtocolText(item, resolverFn) - elif isinstance(data, list): - for item in data: - if isinstance(item, dict): - self.replaceDefaultProtocolText(item, resolverFn) + newOffset = handle.tell() - def getProtocolName(self, node): - text = node.get('text') - if text: - value = node.get('value') if node.get('value') is not None else text - protClassName = value.split('.')[-1] - emProtocolsDict = self.currentProject.getDomain().getProtocols() - prot = emProtocolsDict.get(protClassName, None) + except Exception as e: + return { + "content": "", + "offset": safeOffset, + "error": str(e), + } - if node.get('tag') == 'protocol' and text == 'default': - if prot is None: - logger.warning("Protocol className '%s' not found!!!. \n" - "Fix your config/protocols.conf configuration." - % protClassName) - return + return { + "content": "".join(contentParts), + "offset": int(newOffset), + } - text = prot.getClassLabel() - return text + def _pollProtocolLogPaths( + self, + projectId: int, + protocolId: Union[int, str], + logPaths: Dict[str, Optional[str]], + offsets: Dict[str, int], + maxBytes: Optional[int] = 65536, + maxLines: Optional[int] = 2000, + ) -> Dict[str, Any]: + normalizedOffsets = self._normalizeProtocolLogOffsets(offsets or {}) - return 'default' + return { + "projectId": projectId, + "protocolId": int(protocolId), + "channels": { + "stdout": self._readProtocolLogChunk( + logPaths.get("stdout"), + normalizedOffsets.get("stdout", 0), + maxBytes=maxBytes, + maxLines=maxLines, + ), + "stderr": self._readProtocolLogChunk( + logPaths.get("stderr"), + normalizedOffsets.get("stderr", 0), + maxBytes=maxBytes, + maxLines=maxLines, + ), + "schedule": self._readProtocolLogChunk( + logPaths.get("schedule"), + normalizedOffsets.get("schedule", 0), + maxBytes=maxBytes, + maxLines=maxLines, + ), + }, + } - def listProtocolLogChannelsService(self, projectId: int, protocolId: int): + def listProtocolLogChannelsService( + self, + projectId: int, + protocolId: int, + mapper=None, + currentUser: Optional[dict] = None, + ): """ Return available log channels for a protocol, including paths and basic file stats. """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + pgLogs = self._resolvePostgresqlProtocolLogPaths( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + if pgLogs is not None: + return self._buildProtocolLogChannelsPayload( + projectId=projectId, + protocolId=pgLogs["protocolId"], + logPaths=pgLogs["paths"], + ) + + self._ensureRuntimeProjectForLogs( + mapper=mapper, + projectId=projectId, + currentUser=currentUser, + ) + + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + protocol = self._getScipionProtocolByRuntimeId(scipionProtocolId) # Resolve log paths from Scipion protocol object stdoutPath = protocol.getStdoutLog() if hasattr(protocol, "getStdoutLog") else None @@ -3898,7 +6510,7 @@ def buildChannel(channelId: str, label: str, order, filePath: Optional[str]) -> # Keep a consistent list but allow UI to filter by exists==True return { "projectId": projectId, - "protocolId": int(protocolId), + "protocolId": int(scipionProtocolId), "channels": channels, } @@ -3909,14 +6521,40 @@ def pollProtocolLogsService( offsets: Dict[str, int], maxBytes: Optional[int] = 65536, maxLines: Optional[int] = 2000, + mapper=None, + currentUser: Optional[dict] = None, ): """ Incrementally read protocol logs from the given offsets, applying maxBytes and maxLines limits. """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + pgLogs = self._resolvePostgresqlProtocolLogPaths( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + if pgLogs is not None: + return self._pollProtocolLogPaths( + projectId=projectId, + protocolId=pgLogs["protocolId"], + logPaths=pgLogs["paths"], + offsets=offsets, + maxBytes=maxBytes, + maxLines=maxLines, + ) + + self._ensureRuntimeProjectForLogs( + mapper=mapper, + projectId=projectId, + currentUser=currentUser, + ) + + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + protocol = self._getScipionProtocolByRuntimeId(scipionProtocolId) stdoutPath = protocol.getStdoutLog() if hasattr(protocol, "getStdoutLog") else None stderrPath = protocol.getStderrLog() if hasattr(protocol, "getStderrLog") else None @@ -4067,7 +6705,7 @@ def readChunk(filePath: Optional[str], startOffset: int) -> Dict[str, Any]: return { "projectId": projectId, - "protocolId": int(protocolId), + "protocolId": int(scipionProtocolId), "channels": { "stdout": stdoutRes, "stderr": stderrRes, @@ -4075,15 +6713,43 @@ def readChunk(filePath: Optional[str], startOffset: int) -> Dict[str, Any]: }, } - def getProtocolLogs(self, projectId: int, protocolId: int, - offset: int = 0, - errOffset: int = 0, - scheduleOffset: int = 0): - protocol = self.currentProject.getProtocol(int(protocolId)) + def getProtocolLogs( + self, + projectId: int, + protocolId: int, + offset: int = 0, + errOffset: int = 0, + scheduleOffset: int = 0, + mapper=None, + ): + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + protocol = self._getScipionProtocolByRuntimeId(scipionProtocolId) logPath = protocol.getStdoutLog() errLogPath = protocol.getStderrLog() scheduleLogPath = protocol.getScheduleLog() + def normalizeOffset(value, filePath): + safeOffset = max(0, int(value or 0)) + + if filePath and os.path.exists(filePath): + try: + fileSize = os.path.getsize(filePath) + if safeOffset > fileSize: + return 0 + except Exception: + pass + + return safeOffset + + offset = normalizeOffset(offset, logPath) + errOffset = normalizeOffset(errOffset, errLogPath) + scheduleOffset = normalizeOffset(scheduleOffset, scheduleLogPath) + stdoutContent, stderrContent, scheduleContent = "", "", "" newOffsetOut, newOffsetErr, newOffsetSchedule = offset, errOffset, scheduleOffset @@ -4136,8 +6802,19 @@ def _buildProtocolMutationResult(message: str, **extra) -> Dict[str, Any]: return result - def renameProtocol(self, protocolId, newName, newComment): - protocol = self.currentProject.getProtocol(int(protocolId)) + def renameProtocol( + self, + mapper, + projectId: int, + protocolId, + newName, + newComment, + ): + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) if protocol is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -4172,12 +6849,11 @@ def duplicateProtocol(self, mapper, projectId, protocols): if protocolId is None: continue sourceIds.append(protocolId) - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) protocolList.append(protocol) @@ -4193,19 +6869,17 @@ def duplicateProtocol(self, mapper, projectId, protocols): protId = str(prot.getObjId()) duplicated.append({"sourceId": sourceIds[index], "newId": protId}) + except Exception as e: - errors.append("Failed to duplicate protocols. projectId=%s protocolIds=%s" %projectId, - [getattr(p, "getObjId", lambda: None)() for p in protocolList]) + protocolIds = [ + getattr(p, "getObjId", lambda: None)() + for p in protocolList + ] + logger.exception("Failed to duplicate protocols. projectId=%s protocolIds=%s", + projectId, protocolIds,) - logger.exception( - "Failed to duplicate protocols. projectId=%s protocolIds=%s", - projectId, - [getattr(p, "getObjId", lambda: None)() for p in protocolList], - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to duplicate protocols: {e}", - ) + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to duplicate protocols: {e}",) try: syncResult = self.syncProjectProtocolsAndDependencies( @@ -4238,8 +6912,20 @@ def duplicateProtocol(self, mapper, projectId, protocols): def deleteProtocol(self, mapper, projectId, protocols: Any): try: protList = [] - for protocol in protocols: - protList.append(self.currentProject.getProtocol(int(protocol))) + + for protocolId in protocols or []: + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + protList.append(protocol) + + if not protList: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="No valid protocols to delete", + ) self.currentProject.deleteProtocol(*protList) mapper.deleteProtocol(projectId, protList) @@ -4257,22 +6943,25 @@ def deleteProtocol(self, mapper, projectId, protocols: Any): "protocolsCount": syncInfo.get("protocols"), "dependenciesCount": syncInfo.get("dependencies"), } + + except HTTPException: + raise except Exception as e: raise HTTPException(status_code=500, detail=str(e)) - def restartProtocolAll(self, protocolId): - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + def restartProtocolAll(self, mapper, projectId: int, protocolId): + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) try: workflowProtocolList, _activeProtocolList = self.currentProject._getSubworkflow(protocol) except Exception as e: logger.exception( - "Failed to resolve subworkflow for restart-all. protocolId=%s", + "Failed to resolve subworkflow for restart-all. projectId=%s protocolId=%s", + projectId, protocolId, ) raise HTTPException( @@ -4280,12 +6969,25 @@ def restartProtocolAll(self, protocolId): detail=f"Failed to resolve protocol subworkflow: {e}", ) + cleanupInfo = self._deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql( + mapper=mapper, + projectId=projectId, + protocols=workflowProtocolList, + ) + logger.info( + "Deleted persisted protocol outputs before restart-all. projectId=%s protocolId=%s cleanup=%s", + projectId, + protocolId, + cleanupInfo, + ) + errorList = [] try: self.currentProject._restartWorkflow(errorList, workflowProtocolList) except Exception as e: logger.exception( - "Failed to restart workflow subtree. protocolId=%s", + "Failed to restart workflow subtree. projectId=%s protocolId=%s", + projectId, protocolId, ) raise HTTPException( @@ -4299,15 +7001,17 @@ def restartProtocolAll(self, protocolId): detail=[str(e) for e in errorList], ) - return self._buildProtocolMutationResult("Protocol subtree restarted successfully") + return self._buildProtocolMutationResult( + "Protocol subtree restarted successfully", + postgresqlCleanup=cleanupInfo, + ) def continueProtocolAll(self, mapper, projectId, protocolId, currentUser): - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) try: workflowProtocolList, activeProtocolList = self.currentProject._getSubworkflow(protocol) @@ -4330,19 +7034,11 @@ def continueProtocolAll(self, mapper, projectId, protocolId, currentUser): protocolToLaunch = item if not hasattr(protocolToLaunch, "runMode"): - try: - protocolToLaunch = self.currentProject.getProtocol(int(item)) - except Exception: - logger.exception( - "Failed to resolve protocol to continue. projectId=%s protocolId=%s item=%s", - projectId, - protocolId, - item, - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to resolve protocol to continue: {item}", - ) + protocolToLaunch = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=item, + ) try: protocolToLaunch.runMode.set(MODE_RESUME) @@ -4371,19 +7067,19 @@ def continueProtocolAll(self, mapper, projectId, protocolId, currentUser): return self._buildProtocolMutationResult("Protocol subtree continued successfully") - def resetProtocolFrom(self, protocolId): - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + def resetProtocolFrom(self, mapper, projectId: int, protocolId): + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) try: workflowProtocolList, _activeProtocolList = self.currentProject._getSubworkflow(protocol) except Exception as e: logger.exception( - "Failed to resolve subworkflow for reset-from. protocolId=%s", + "Failed to resolve subworkflow for reset-from. projectId=%s protocolId=%s", + projectId, protocolId, ) raise HTTPException( @@ -4391,11 +7087,24 @@ def resetProtocolFrom(self, protocolId): detail=f"Failed to resolve protocol subworkflow: {e}", ) + cleanupInfo = self._deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql( + mapper=mapper, + projectId=projectId, + protocols=workflowProtocolList, + ) + logger.info( + "Deleted persisted protocol outputs before reset-from. projectId=%s protocolId=%s cleanup=%s", + projectId, + protocolId, + cleanupInfo, + ) + try: resetErrors = self.currentProject.resetWorkFlow(workflowProtocolList) or [] except Exception as e: logger.exception( - "Failed to reset workflow subtree. protocolId=%s", + "Failed to reset workflow subtree. projectId=%s protocolId=%s", + projectId, protocolId, ) raise HTTPException( @@ -4409,18 +7118,20 @@ def resetProtocolFrom(self, protocolId): detail=[str(e) for e in resetErrors], ) - return self._buildProtocolMutationResult("Protocol subtree reset successfully") + return self._buildProtocolMutationResult( + "Protocol subtree reset successfully", + postgresqlCleanup=cleanupInfo, + ) - def stopProtocol(self, protocolIds): + def stopProtocol(self, mapper, projectId: int, protocolIds): resolvedProtocols = [] for protocolId in protocolIds or []: - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) resolvedProtocols.append(protocol) if not resolvedProtocols: @@ -4434,7 +7145,8 @@ def stopProtocol(self, protocolIds): self.currentProject.stopProtocol(protocol) except Exception as e: logger.exception( - "Failed to stop protocols. protocolIds=%s", + "Failed to stop protocols. projectId=%s protocolIds=%s", + projectId, protocolIds, ) raise HTTPException( @@ -4949,6 +7661,44 @@ def _normalizeProtocolIdsForExport( return out + def _resolveRuntimeProtocolsForExport( + self, + mapper, + projectId: int, + protocolIds: List[Union[int, str]], + ) -> List[Any]: + protocolList = [] + missing = [] + + for protocolId in protocolIds or []: + try: + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + except HTTPException as e: + if e.status_code == status.HTTP_404_NOT_FOUND: + missing.append(str(protocolId)) + continue + raise + + protocolList.append(protocol) + + if missing: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol(s) not found: {', '.join(missing)}", + ) + + if not protocolList: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="No valid protocols to export", + ) + + return protocolList + def _sanitizeExportFilename(self, rawFilename: str) -> str: filename = str(rawFilename or "").strip() filename = filename.replace("\\", "/").split("/")[-1].strip() @@ -4971,8 +7721,17 @@ def _sanitizeExportFilename(self, rawFilename: str) -> str: return filename - def _resolveFsRootForWrite(self, protocolId: Union[int, str]) -> Path: - pathInfo = self.getProtocolPath(protocolId) + def _resolveFsRootForWrite( + self, + protocolId: Union[int, str], + mapper=None, + projectId: Optional[int] = None, + ) -> Path: + pathInfo = self.getProtocolPath( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) rootAbs = str((pathInfo or {}).get("rootAbs") or "").strip() if not rootAbs: raise HTTPException( @@ -5009,16 +7768,27 @@ def _guardFsPathWithinRootForWrite( return candidate - def getProtocolPath(self, protocolId): + def getProtocolPath( + self, + protocolId, + mapper=None, + projectId: Optional[int] = None, + ): if self._isGlobalFsBrowserMode(protocolId): root = self._getGlobalFsBrowserRoot() return { "rootAbs": str(root), "startPath": "", } + fakeProtocolId = 'fake-protocol-id-for-browser-paths-resolution' + if protocolId != fakeProtocolId: - protocol = self.currentProject.getProtocol(int(protocolId)) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) protocolAbsPath = os.path.abspath(protocol.getPath()) rootAbsPath = self._inferProjectRootAbs(protocolAbsPath) else: @@ -5065,14 +7835,26 @@ def _inferProjectRootAbs(self, protocolAbsPath: str) -> str: return os.path.abspath(projectPath) if projectPath else "" - def _protocolRoot(self, protocolId: Union[int, str]) -> FsPath: + def _protocolRoot( + self, + protocolId: Union[int, str], + mapper=None, + projectId: Optional[int] = None, + ) -> FsPath: """ Resolve the absolute root folder for a protocol, using your service. """ - root = self.getProtocolPath(str(protocolId)) - if not root: + pathInfo = self.getProtocolPath( + protocolId=str(protocolId), + mapper=mapper, + projectId=projectId, + ) + + rootAbs = str((pathInfo or {}).get("rootAbs") or "").strip() + if not rootAbs: raise HTTPException(status_code=404, detail="Protocol path not found") - return FsPath(root).resolve() + + return FsPath(rootAbs).resolve() @staticmethod def _guardJoin(root: FsPath, relPath: str) -> FsPath: @@ -5093,7 +7875,30 @@ def _guessMime(p: FsPath) -> str: mt, _ = mimetypes.guess_type(str(p)) return mt or "application/octet-stream" - def listProtocolDir(self, protocolId: str, path: str): + def _resolveRuntimeProtocolIdForFs( + self, + protocolId: Union[int, str], + mapper=None, + projectId: Optional[int] = None, + ) -> str: + if self._isGlobalFsBrowserMode(protocolId): + return str(protocolId) + + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + return str(scipionProtocolId) + + def listProtocolDir( + self, + protocolId: str, + path: str, + mapper=None, + projectId: Optional[int] = None, + ): """Return the directory file list.""" fileHandlers = FileHandlers(self.currentProject) @@ -5101,9 +7906,21 @@ def listProtocolDir(self, protocolId: str, path: str): root = self._getGlobalFsBrowserRoot() return fileHandlers.listRemoteDirectoryUnderRoot(root, path) - return fileHandlers.listProtocolDir(protocolId, path) + runtimeProtocolId = self._resolveRuntimeProtocolIdForFs( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) + + return fileHandlers.listProtocolDir(runtimeProtocolId, path) - def previewProtocolTextFile(self, protocolId: str, path: str): + def previewProtocolTextFile( + self, + protocolId: str, + path: str, + mapper=None, + projectId: Optional[int] = None, + ): """ Return a lightweight preview for a file inside a protocol workspace. """ @@ -5113,9 +7930,21 @@ def previewProtocolTextFile(self, protocolId: str, path: str): root = self._getGlobalFsBrowserRoot() return fileHandlers.previewTextFileUnderRoot(root, path) - return fileHandlers.previewProtocolTextFile(protocolId, path) + runtimeProtocolId = self._resolveRuntimeProtocolIdForFs( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) + + return fileHandlers.previewProtocolTextFile(runtimeProtocolId, path) - def previewRemoteEntry(self, protocolId: str, path: str): + def previewRemoteEntry( + self, + protocolId: str, + path: str, + mapper=None, + projectId: Optional[int] = None, + ): """ Return a preview. """ @@ -5129,13 +7958,26 @@ def previewRemoteEntry(self, protocolId: str, path: str): databaseInspector=self._inspectScipionSqliteDatabase, ) + runtimeProtocolId = self._resolveRuntimeProtocolIdForFs( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) + return fileHandlers.previewProtocolRemoteEntry( - protocolId, + runtimeProtocolId, path, databaseInspector=self._inspectScipionSqliteDatabase, ) - def previewProtocolImageFile(self, protocolId, path, inline: bool): + def previewProtocolImageFile( + self, + protocolId, + path, + inline: bool, + mapper=None, + projectId: Optional[int] = None, + ): """ inline == False: - attachment download (binary as-is) @@ -5151,13 +7993,41 @@ def previewProtocolImageFile(self, protocolId, path, inline: bool): root = self._getGlobalFsBrowserRoot() return fileHandlers.previewImageFileUnderRoot(root, path, inline) - return fileHandlers.previewProtocolImageFile(protocolId, path, inline) + runtimeProtocolId = self._resolveRuntimeProtocolIdForFs( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) + + return fileHandlers.previewProtocolImageFile(runtimeProtocolId, path, inline) + + def outputPreview( + self, + protocolId, + outputName, + requestHeaders=None, + colormap=None, + mapper=None, + projectId=None, + ): + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + protocol = self._getScipionProtocolByRuntimeId(scipionProtocolId) + + if not hasattr(protocol, outputName): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Output '{outputName}' not found in protocol", + ) - def outputPreview(self, protocolId: int, outputName: str, requestHeaders: dict = None, colormap: str = None): - protocol = self.currentProject.getProtocol(protocolId) output = getattr(protocol, outputName) + outputPath = output.getFileName() - outputPreview = OutputsPreview( + preview = OutputsPreview( self.currentProject, protocol, output, @@ -5165,31 +8035,27 @@ def outputPreview(self, protocolId: int, outputName: str, requestHeaders: dict = colormapOverride=colormap, ) - getFileName = getattr(output, "getFileName", None) - if not callable(getFileName): - return outputPreview.renderOutputFallbackPreview() - - try: - outputPath = getFileName() - except Exception: - return outputPreview.renderOutputFallbackPreview() - - if not outputPath: - return outputPreview.renderOutputFallbackPreview() - objMgr = self._createObjectManager() - return outputPreview.preview(protocolId, outputPath, objMgr) + return preview.preview(scipionProtocolId, outputPath, objMgr) def buildProtocolThumbnail( self, - protocolId: int, + protocolId: Union[int, str], force: bool = False, size: int = 320, outputName: Optional[str] = None, + mapper=None, + projectId: Optional[int] = None, ) -> Dict[str, Any]: - service = ThumbnailService(self.currentProject) - return service.buildProtocolThumbnail( + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, protocolId=protocolId, + ) + + thumbnailService = ThumbnailService(self.currentProject) + return thumbnailService.buildProtocolThumbnail( + protocolId=scipionProtocolId, force=force, size=size, outputName=outputName, @@ -5237,34 +8103,20 @@ def exportProtocolsService( ) try: - protocolIdInts = [int(pid) for pid in protocolIds] - except Exception: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail="protocolIds must be numeric", + protocolList = self._resolveRuntimeProtocolsForExport( + mapper=mapper, + projectId=projectId, + protocolIds=protocolIds, ) - try: - protocolList = [] - missing: List[str] = [] - - for protId in protocolIdInts: - protocol = self.currentProject.getProtocol(protId) - if protocol is None: - missing.append(str(protId)) - continue - protocolList.append(protocol) - - if missing: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol(s) not found: {', '.join(missing)}", - ) - rawExport = self.currentProject.getProtocolsJson(protocolList) content = self._buildWorkflowExportJsonContent(rawExport, protocolList) - rootPath = self._resolveFsRootForWrite("-1") + rootPath = self._resolveFsRootForWrite( + "-1", + mapper=mapper, + projectId=projectId, + ) targetDir = self._guardFsPathWithinRootForWrite(rootPath, directoryPath) if targetDir.exists() and not targetDir.is_dir(): @@ -5480,29 +8332,11 @@ def exportWorkflowProtocolsService( detail="Missing protocolIds", ) - try: - protocolIdInts = [int(pid) for pid in protocolIds] - except Exception: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail="protocolIds must be numeric", - ) - - protocolList = [] - missing: List[str] = [] - - for protocolId in protocolIdInts: - protocol = self.currentProject.getProtocol(protocolId) - if protocol is None: - missing.append(str(protocolId)) - continue - protocolList.append(protocol) - - if missing: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol(s) not found: {', '.join(missing)}", - ) + protocolList = self._resolveRuntimeProtocolsForExport( + mapper=mapper, + projectId=projectId, + protocolIds=protocolIds, + ) rawExport = self.currentProject.getProtocolsJson(protocolList) workflow = self._decodeExportJsonPayload(rawExport) @@ -5597,8 +8431,14 @@ def writeRemoteFileService( self, protocolId: Union[int, str], payload: Any, + mapper=None, + projectId: Optional[int] = None, ) -> Dict[str, Any]: - rootPath = self._resolveFsRootForWrite(protocolId) + rootPath = self._resolveFsRootForWrite( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) targetPath = self._guardFsPathWithinRootForWrite( rootPath, getattr(payload, "path", ""), @@ -5628,15 +8468,23 @@ def writeRemoteFileService( # Internal helpers for TiltSeries (SetOfTiltSeries) # ---------------------------------------------------------------------- - @lru_cache - def _resolveOutputForTiltSeries(self, protocolId: int, outputName: str): + def _resolveOutputForTiltSeries( + self, + protocolId: int, + outputName: str, + projectId: Optional[int] = None, + mapper=None, + ): """ Resolve protocol + SetOfTiltSeries-like output for tilt series operations. + + protocolId can be either the PostgreSQL protocols.id or the Scipion protocolId. """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) if not hasattr(protocol, outputName): raise HTTPException( @@ -5653,43 +8501,114 @@ def _resolveOutputForTiltSeries(self, protocolId: int, outputName: str): return protocol, output + def _getPostgresqlTiltSeriesReaderIfAvailable( + self, + mapper, + projectId: int, + protocolId: int, + outputName: str, + ): + if mapper is None: + return None + + try: + from app.backend.viewers.postgresql_tiltseries_reader import PostgresqlTiltSeriesReader + + readerProtocolId = self._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + reader = PostgresqlTiltSeriesReader( + db=mapper.db, + projectId=projectId, + protocolId=readerProtocolId, + outputName=outputName, + ) + + if reader.hasOutput(): + return reader + + except Exception: + logger.exception( + "Failed to initialize PostgreSQL TiltSeries reader. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + ) + + return None + + def _parseTiltSeriesFramePath( + self, + framePath: Any, + fallbackIndex: int, + ) -> Tuple[str, int]: + pathText = str(framePath or "").strip() + if not pathText: + raise HTTPException( + status_code=404, + detail="Tilt image path not found in PostgreSQL metadata", + ) + + imageIndex = int(fallbackIndex) + imagePath = pathText + + if "@" in pathText: + indexText, imagePath = pathText.split("@", 1) + try: + imageIndex = int(float(indexText)) + except Exception: + imageIndex = int(fallbackIndex) + + return os.path.abspath(imagePath), imageIndex + # ====================================================================== # Analyze Results: Resolve viewer # ====================================================================== - def resolveAnalyzeViewerDecision(self, projectId: int, protocolId: int, ctx: Dict[str, Any]) -> Dict[str, Any]: - # resolveAnalyzeViewerDecision - return { - "handled": False, - } + def resolveAnalyzeViewerDecision( + self, + projectId: int, + protocolId: int, + ctx: Dict[str, Any], + mapper=None, + ) -> Dict[str, Any]: + # This resolver is reserved for external developer-provided viewers. + # Built-in React viewers are selected on the frontend side. + return {"handled": False} # ====================================================================== # Analyze Results: CTF Tomography (CTFTomoSeries) # ====================================================================== - - @lru_cache - def _resolveOutputForCtftomoSeries(self, protocolId: int, outputName: str): + def _resolveOutputForCtftomoSeries( + self, + protocolId: int, + outputName: str, + mapper=None, + projectId: Optional[int] = None, + ): """ Resolve protocol + CTFTomoSeries-like output for CTF tomography operations. - The output can be: - * a single CTFTomoSeries (one tilt-series) - * a container of CTFTomoSeries objects (SetOfCTFTomoSeries) + protocolId can be either the PostgreSQL protocols.id or the Scipion protocolId. """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) if not hasattr(protocol, outputName): raise HTTPException( - status_code=404, + status_code=status.HTTP_404_NOT_FOUND, detail=f"Output '{outputName}' not found in protocol", ) output = getattr(protocol, outputName) if output is None: raise HTTPException( - status_code=404, + status_code=status.HTTP_404_NOT_FOUND, detail=f"Output '{outputName}' is None", ) @@ -5786,6 +8705,7 @@ def listOutputCtftomoSeriesService( projectId: int, protocolId: int, outputName: str, + mapper=None, ): """ List all CTFTomoSeries in a CTFTomo output. @@ -5803,7 +8723,31 @@ def listOutputCtftomoSeriesService( ... ] """ - protocol, output = self._resolveOutputForCtftomoSeries(protocolId, outputName) + pgReader = self._getPostgresqlCtftomoReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + return pgReader.listCtftomoSeries() + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="CTFTomo", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="reader_not_available", + ) + + protocol, output = self._resolveOutputForCtftomoSeries( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) seriesList: List[Dict[str, Any]] = [] @@ -5823,6 +8767,7 @@ def getCtftomoSeriesViewsService( protocolId: int, outputName: str, tiltSeriesId: Union[int, str], + mapper=None, ): """ Return all CTF measurements for one tilt series inside a CTFTomo output. @@ -5856,7 +8801,38 @@ def getCtftomoSeriesViewsService( ] } """ - protocol, output = self._resolveOutputForCtftomoSeries(protocolId, outputName) + pgReader = self._getPostgresqlCtftomoReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.getCtftomoSeriesViews(tiltSeriesId) + if payload is not None: + return payload + + raise HTTPException( + status_code=404, + detail="CTF tomo series not found in PostgreSQL metadata", + ) + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="CTFTomo views", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="views_not_available" if pgReader is not None else "reader_not_available", + tiltSeriesId=tiltSeriesId, + ) + + protocol, output = self._resolveOutputForCtftomoSeries( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) targetKey = str(tiltSeriesId) setOfTiltSeries = output.getSetOfTiltSeries() @@ -5899,6 +8875,7 @@ def renderCtfTomoPsdImageService( applyTransform: bool = False, rot=None, shifts=None, + mapper=None, ) -> Response: """ Render a single CTFtomo PSD image using the OutputsPreview pipeline. @@ -5909,7 +8886,66 @@ def renderCtfTomoPsdImageService( - index is used when the PSD is stored in a stack file. - applyTransform/rot/shifts are optional alignment parameters. """ - protocol, output = self._resolveOutputForCtftomoSeries(protocolId, outputName) + if mapper is not None and psdPath: + try: + splitPath = str(psdPath).split("@") + imageIndex = int(index) + + if len(splitPath) > 1: + try: + imageIndex = int(float(splitPath[0])) + except Exception: + imageIndex = int(index) + + candidatePath = Path(splitPath[-1]).expanduser() + + if candidatePath.is_absolute(): + absPath = candidatePath.resolve() + + if absPath.exists() and absPath.is_file(): + preview = OutputsPreview( + currentProject=self.currentProject, + protocol=None, + output=None, + ) + + return preview.renderImageFromFilePath( + filePath=str(absPath), + size=size, + fmt=fmt, + index=imageIndex, + inline=inline, + quality=quality, + applyTransform=applyTransform and rot is not None and shifts is not None, + rot=rot, + shifts=shifts, + ) + + except Exception: + logger.exception( + "Failed to render CTFTomo PSD from PostgreSQL metadata. projectId=%s protocolId=%s outputName=%s psdPath=%s", + projectId, + protocolId, + outputName, + psdPath, + ) + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="CTFTomo PSD", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="psd_file_not_available", + psdPath=psdPath, + ) + + protocol, output = self._resolveOutputForCtftomoSeries( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) if protocol is None: raise HTTPException( status_code=404, @@ -5984,15 +9020,61 @@ def _buildTiltSeriesSummary(self, ts) -> Dict[str, Any]: return item + def _getPostgresqlCtftomoReaderIfAvailable( + self, + mapper, + projectId: int, + protocolId: int, + outputName: str, + ): + if mapper is None: + return None + + try: + from app.backend.viewers.postgresql_ctftomo_reader import PostgresqlCtftomoReader + + readerProtocolId = self._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + reader = PostgresqlCtftomoReader( + db=mapper.db, + projectId=projectId, + protocolId=readerProtocolId, + outputName=outputName, + ) + + if reader.hasOutput(): + return reader + + except Exception: + logger.exception( + "Failed to initialize PostgreSQL CTFTomo reader. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + ) + + return None + # ====================================================================== # Analyze Results: Volumes (Volume / VolumeMask / SetOfVolumes) # ====================================================================== - def _resolveOutputForVolumes(self, protocolId: int, outputName: str): - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + def _resolveOutputForVolumes( + self, + protocolId: int, + outputName: str, + mapper=None, + projectId: Optional[int] = None, + ): + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) # Try exact + common alternates (singular/plural/alias) candidates = [outputName] @@ -6017,13 +9099,158 @@ def _resolveOutputForVolumes(self, protocolId: int, outputName: str): raise HTTPException(status_code=404, detail=f"Output '{outputName}' not found in protocol (tried {candidates})") - def listOutputVolumesService(self, projectId: int, protocolId: int, outputName: str): - protocol, output = self._resolveOutputForVolumes(protocolId, outputName) + def _getPostgresqlVolumeReaderIfAvailable( + self, + mapper, + projectId: int, + protocolId: int, + outputName: str, + ): + if mapper is None: + return None + + readerProtocolId = self._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + try: + from app.backend.viewers.postgresql_coords3d_tomogram_volume_reader import \ + PostgresqlCoords3dTomogramVolumeReader + + reader = PostgresqlCoords3dTomogramVolumeReader( + db=mapper.db, + projectId=projectId, + protocolId=readerProtocolId, + outputName=outputName, + ) + + if reader.hasOutput(): + return reader + + except Exception: + logger.debug( + "PostgreSQL Coords3D-derived tomogram volume reader is not available. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + exc_info=True, + ) + + try: + from app.backend.viewers.postgresql_volume_reader import PostgresqlVolumeReader + + reader = PostgresqlVolumeReader( + db=mapper.db, + projectId=projectId, + protocolId=readerProtocolId, + outputName=outputName, + ) + + if reader.hasOutput(): + return reader + + except Exception: + logger.exception( + "Failed to initialize PostgreSQL volume reader. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + ) + + return None + + def listOutputVolumesService( + self, + projectId: int, + protocolId: int, + outputName: str, + mapper=None, + ): + pgReader = self._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.listVolumes() + if payload is not None: + return payload + + logger.info( + "Skipping PostgreSQL volume list reader. projectId=%s protocolId=%s outputName=%s reason=%s", + projectId, + protocolId, + outputName, + getattr(pgReader, "lastSkipReason", None), + ) + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Volume", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + ) + + protocol, output = self._resolveOutputForVolumes( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) op = OutputsPreview(self.currentProject, protocol, output) return op.listOutputVolumes() - def getVolumeInfoService(self, projectId: int, protocolId: int, outputName: str, volumeId: int): - protocol, output = self._resolveOutputForVolumes(protocolId, outputName) + def getVolumeInfoService( + self, + projectId: int, + protocolId: int, + outputName: str, + volumeId: Union[int, str], + mapper=None, + ): + pgReader = self._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.getVolumeInfo(volumeId) + if payload is not None: + return payload + + logger.info( + "Skipping PostgreSQL volume info reader. projectId=%s protocolId=%s outputName=%s volumeId=%s reason=%s", + projectId, + protocolId, + outputName, + volumeId, + getattr(pgReader, "lastSkipReason", None), + ) + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Volume", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + volumeId=volumeId, + ) + + protocol, output = self._resolveOutputForVolumes( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) op = OutputsPreview(self.currentProject, protocol, output) return op.getVolumeInfo(volumeId) @@ -6034,24 +9261,54 @@ def getVolumeHistogramService( outputName: str, volumeId: Union[int, str], bins: int = 128, + mapper=None, ): - """ - Compute or retrieve an intensity histogram for a single volume. + pgReader = self._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) - It delegates to OutputsPreview so all volume handling stays in one place. - The result is normalized to always expose: - { - "binEdges": [...], - "counts": [...] - } - """ - protocol, output = self._resolveOutputForVolumes(protocolId, outputName) + if pgReader is not None: + payload = pgReader.getHistogram(volumeId=volumeId, bins=bins) + if payload is not None: + return { + "binEdges": payload.get("binEdges") or [], + "counts": payload.get("counts") or [], + } + + logger.info( + "Skipping PostgreSQL volume histogram reader. projectId=%s protocolId=%s outputName=%s volumeId=%s reason=%s", + projectId, + protocolId, + outputName, + volumeId, + getattr(pgReader, "lastSkipReason", None), + ) + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Volume histogram", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + volumeId=volumeId, + ) + + protocol, output = self._resolveOutputForVolumes( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) op = OutputsPreview(self.currentProject, protocol, output) if isinstance(output, SetOfVolumes): - output = output.getItem('_objId', volumeId + 1) - raw = op.getVolumeHistogram(volumePath=output.getFileName(), bins=bins) + output = output.getItem('_objId', int(volumeId) + 1) + raw = op.getVolumeHistogram(volumePath=output.getFileName(), bins=bins) if not raw: return {"binEdges": [], "counts": []} @@ -6062,23 +9319,87 @@ def getVolumeHistogramService( binEdges = list(binEdges) except Exception: binEdges = [] + try: counts = list(counts) except Exception: counts = [] - return { - "binEdges": binEdges, - "counts": counts, - } + return { + "binEdges": binEdges, + "counts": counts, + } + + def renderVolumeSliceService( + self, + projectId: int, + protocolId: int, + outputName: str, + volumeId: Union[int, str], + sliceIndex: int, + axis: str, + colormap: Optional[str], + normalize: Optional[str], + scale: float, + inline: bool, + fmt: str = "webp", + thumb: Optional[int] = None, + fast: bool = True, + quality: int = 75, + mapper=None, + ) -> Response: + pgReader = self._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + info = pgReader.getVolumeFile(volumeId) + if info is not None: + volumePath = info.get("fileName") or info.get("path") + if volumePath and os.path.exists(str(volumePath)): + return self._renderTomogramSliceFromPath( + volumePath=str(volumePath), + tomogramId=volumeId, + sliceIndex=sliceIndex, + axis=axis, + colormap=colormap, + normalize=normalize or "minmax", + scale=scale, + inline=inline, + fmt=fmt, + thumb=thumb, + fast=fast, + quality=quality, + ) + + logger.info( + "Skipping PostgreSQL volume slice reader. projectId=%s protocolId=%s outputName=%s volumeId=%s reason=%s", + projectId, + protocolId, + outputName, + volumeId, + getattr(pgReader, "lastSkipReason", None), + ) + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Volume slice", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + volumeId=volumeId, + ) - def renderVolumeSliceService( - self, projectId: int, protocolId: int, outputName: str, volumeId: int, - sliceIndex: int, axis: str, colormap: Optional[str], normalize: Optional[str], - scale: float, inline: bool, fmt: str = "webp", - thumb: Optional[int] = None, fast: bool = True, quality: int = 75, - ) -> Response: - protocol, output = self._resolveOutputForVolumes(protocolId, outputName) + protocol, output = self._resolveOutputForVolumes( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) op = OutputsPreview(self.currentProject, protocol, output) return op.renderVolumeSlice( volumeId=volumeId, @@ -6102,12 +9423,64 @@ def getVolumeData3dService( volumeId: Union[int, str], maxDim: int = 160, method: str = "binning", + mapper=None, ): - protocol, output = self._resolveOutputForVolumes(protocolId, outputName) + pgReader = self._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + result = pgReader.getVolumeArray(volumeId) + if result is not None: + volume, _props, _info = result + + if (method or "").lower() == "none": + volumeSmall = np.asarray(volume, dtype=np.float32) + else: + volumeSmall = self._downsampleVolumePreview( + np.asarray(volume, dtype=np.float32), + maxDim=maxDim, + method=method, + ) + + z, y, x = volumeSmall.shape + return { + "dims": [int(z), int(y), int(x)], + "values": volumeSmall.ravel(order="C").astype(np.float32).tolist(), + } + + logger.info( + "Skipping PostgreSQL volume data3d reader. projectId=%s protocolId=%s outputName=%s volumeId=%s reason=%s", + projectId, + protocolId, + outputName, + volumeId, + getattr(pgReader, "lastSkipReason", None), + ) + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Volume 3D data", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + volumeId=volumeId, + ) + + protocol, output = self._resolveOutputForVolumes( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) volumePath = self._getVolumePathFromOutput(output, volumeId) try: - vol, _props = readVolumeArray3d(volumePath) # Z, Y, X + vol, _props = readVolumeArray3d(volumePath) except HTTPException: raise except FileNotFoundError: @@ -6115,7 +9488,10 @@ def getVolumeData3dService( except Exception as e: raise HTTPException(status_code=500, detail=f"Cannot read volume file: {e}") - volSmall = self._downsampleVolumePreview(vol, maxDim=maxDim, method=method) + if (method or "").lower() == "none": + volSmall = np.asarray(vol, dtype=np.float32) + else: + volSmall = self._downsampleVolumePreview(vol, maxDim=maxDim, method=method) z, y, x = volSmall.shape return { @@ -6307,10 +9683,79 @@ def _downsampleVolumeForSurface( maxDim=maxDim, method=methodLower) - def getVolumeSurfaceMesh(self, protocolId, outputName, volumeId, level, - maxDim, method, maxTriangles, currentUser): + def getVolumeSurfaceMesh( + self, + projectId: int, + protocolId: int, + outputName: str, + volumeId: Union[int, str], + level, + maxDim, + method, + maxTriangles, + currentUser, + mapper=None, + ): + pgReader = self._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + result = pgReader.getVolumeArray(volumeId) + if result is not None: + volume, _props, _info = result + volumeSmall = self._downsampleVolumeForSurface( + np.asarray(volume, dtype=np.float32), + maxDim=maxDim, + method=method, + ) + + mesh = buildVolumeSurfaceMesh( + volumeSmall, + level=level, + maxTriangles=maxTriangles, + ) + + mesh["sourceDims"] = [int(volume.shape[0]), int(volume.shape[1]), int(volume.shape[2])] + mesh["maxDim"] = int(maxDim) + mesh["method"] = method + mesh["volumeId"] = str(volumeId) + mesh["outputName"] = outputName + + response = JSONResponse(mesh) + response.headers["X-Debug-Auth"] = "ok" + response.headers["X-Debug-UserId"] = str(getattr(currentUser, "id", currentUser.get("id", ""))) + response.headers["Vary"] = "Authorization" + return response - _protocol, output = self._resolveOutputForVolumes(protocolId, outputName) + logger.info( + "Skipping PostgreSQL volume surface reader. projectId=%s protocolId=%s outputName=%s volumeId=%s reason=%s", + projectId, + protocolId, + outputName, + volumeId, + getattr(pgReader, "lastSkipReason", None), + ) + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Volume surface mesh", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + volumeId=volumeId, + ) + + protocol, output = self._resolveOutputForVolumes( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) volumePath = self._getVolumePathFromOutput(output, volumeId) volume, _props = readVolumeArray3d(volumePath) @@ -6354,6 +9799,7 @@ def listOutputTiltSeriesService( projectId: int, protocolId: int, outputName: str, + mapper=None, ): """ List all tilt series in a SetOfTiltSeries-like output. @@ -6371,7 +9817,31 @@ def listOutputTiltSeriesService( ... ] """ - _, setOfTiltSeries = self._resolveOutputForTiltSeries(protocolId, outputName) + pgReader = self._getPostgresqlTiltSeriesReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + return pgReader.listTiltSeries() + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="TiltSeries", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="reader_not_available", + ) + + _, setOfTiltSeries = self._resolveOutputForTiltSeries( + protocolId=protocolId, + outputName=outputName, + projectId=projectId, + mapper=mapper, + ) seriesList: List[Dict[str, Any]] = [] for idx, ts in enumerate(setOfTiltSeries.iterItems(iterate=False)): @@ -6389,6 +9859,7 @@ def getTiltSeriesFramesService( protocolId: int, outputName: str, tiltSeriesId: Union[int, str], + mapper=None, ): """ Return metadata for all tilt images in a given tilt series. @@ -6415,7 +9886,34 @@ def getTiltSeriesFramesService( ] } """ - protocol, setOfTiltSeries = self._resolveOutputForTiltSeries(protocolId, outputName) + pgReader = self._getPostgresqlTiltSeriesReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.getTiltSeriesFrames(tiltSeriesId) + if payload is not None: + return payload + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="TiltSeries frames", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="frames_not_available" if pgReader is not None else "reader_not_available", + tiltSeriesId=tiltSeriesId, + ) + + protocol, setOfTiltSeries = self._resolveOutputForTiltSeries( + protocolId=protocolId, + outputName=outputName, + projectId=projectId, + mapper=mapper, + ) targetKey = str(tiltSeriesId) selectedSummary: Optional[Dict[str, Any]] = None selectedSeries = setOfTiltSeries.getItem('_tsId', targetKey) @@ -6519,6 +10017,7 @@ def createNewSetOfCtftomoSeriesService( outputName: str, exclusions: Dict[str, Any], restack: bool, + mapper=None, ) -> Dict[str, Any]: """ Create a new SetOfCTFTomoSeries applying per-series and per-tilt exclusions. @@ -6533,7 +10032,12 @@ def createNewSetOfCtftomoSeriesService( ... } """ - protocol, inputSet = self._resolveOutputForCtftomoSeries(protocolId, outputName) + protocol, inputSet = self._resolveOutputForCtftomoSeries( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) # Normalize exclusions keys to strings normalizedExclusions: Dict[str, Dict[str, Any]] = {} @@ -6580,20 +10084,40 @@ def createNewSetOfCtftomoSeriesService( "restack": bool(restack), "message": "No output was generated because it cannot be empty", } + postgresqlSync = None + postgresqlError = None + try: protocol._defineOutputs(**{newOutputName: outputSet}) protocol._store() + + postgresqlStore = self._storeGeneratedSetInPostgresql( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=newOutputName, + scipionSet=outputSet, + contextLabel="CTFTomo", + ) + postgresqlSync = postgresqlStore["postgresqlSync"] + postgresqlError = postgresqlStore["postgresqlError"] + except Exception: - logger.exception("Error attaching Ctftomo filtered set '%s' to protocol", newOutputName,) + logger.exception("Error attaching Ctftomo filtered set '%s' to protocol", newOutputName) logger.info( - "The new Ctftomo set (%s) has been created successfully with %d series", newOutputName, createdCount,) + "The new Ctftomo set (%s) has been created successfully with %d series", + newOutputName, + createdCount, + ) return { "status": 0, "outputName": newOutputName, "createdSeries": createdCount, "restack": bool(restack), + "postgresqlSync": postgresqlSync, + "postgresqlError": postgresqlError, } def _buildTiltSeriesPreviewCacheKey( @@ -6731,8 +10255,117 @@ def renderTiltSeriesImageService( applyTransform: bool = True, inline: bool = True, requestHeaders: Optional[Dict[str, str]] = None, + mapper=None, ): - protocol, setOfTiltSeries = self._resolveOutputForTiltSeries(protocolId, outputName) + + pgReader = self._getPostgresqlTiltSeriesReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + frame = pgReader.getTiltImageFrame(tiltSeriesId, index) + if frame is not None: + try: + imagePath, imageIndex = self._parseTiltSeriesFramePath( + frame.get("path"), + fallbackIndex=int(index), + ) + + cacheKey = self._buildTiltSeriesPreviewCacheKey( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + tiltSeriesId=tiltSeriesId, + index=imageIndex, + size=size, + fmt=fmt, + applyTransform=applyTransform, + inline=inline, + imagePath=imagePath, + ) + + cachedResponse = self._getTiltSeriesPreviewFromCache(cacheKey) + if cachedResponse is not None: + return cachedResponse + + rot = None + shifts = None + + if applyTransform: + rot = frame.get("rot") + shiftX = frame.get("shiftX") + shiftY = frame.get("shiftY") + if shiftX is not None and shiftY is not None: + shifts = (shiftX, shiftY) + + preview = OutputsPreview( + currentProject=self.currentProject, + protocol=None, + output=None, + requestHeaders=requestHeaders, + ) + + response = preview.renderImageFromFilePath( + imagePath, + size=size, + fmt=fmt, + index=imageIndex, + applyTransform=applyTransform, + inline=inline, + rot=rot, + shifts=shifts, + ) + + return self._storeTiltSeriesPreviewInCache(cacheKey, response) + + + except HTTPException: + logger.exception( + "Failed to render TiltSeries image from PostgreSQL. projectId=%s protocolId=%s outputName=%s tiltSeriesId=%s index=%s", + projectId, + protocolId, + outputName, + tiltSeriesId, + index, + ) + raise + + except Exception as e: + logger.exception( + "Failed to render TiltSeries image from PostgreSQL. projectId=%s protocolId=%s outputName=%s tiltSeriesId=%s index=%s", + projectId, + protocolId, + outputName, + tiltSeriesId, + index, + ) + + raise HTTPException( + status_code=500, + detail=f"Failed to render TiltSeries image from PostgreSQL metadata: {e}", + + ) + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="TiltSeries image", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + tiltSeriesId=tiltSeriesId, + index=index, + ) + + protocol, setOfTiltSeries = self._resolveOutputForTiltSeries( + protocolId=protocolId, + outputName=outputName, + projectId=projectId, + mapper=mapper, + ) ts = setOfTiltSeries.getItem('_tsId', tiltSeriesId) if ts is None: @@ -6808,6 +10441,7 @@ def renderTiltSeriesImagesBatchService( applyTransform: bool = True, inline: bool = True, requestHeaders: Optional[Dict[str, str]] = None, + mapper=None, ) -> Dict[str, Any]: # renderTiltSeriesImagesBatchService cleanIndices: List[int] = [] @@ -6844,6 +10478,7 @@ def renderTiltSeriesImagesBatchService( applyTransform=applyTransform, inline=inline, requestHeaders=requestHeaders, + mapper=mapper, ) body = getattr(response, "body", None) or b"" @@ -6892,6 +10527,7 @@ def createNewSetOfTiltSeriesService( outputName: str, exclusions: Dict[str, Any], restack: bool, + mapper=None ) -> Dict[str, Any]: """ Create a new SetOfTiltSeries applying per-tilt-series and per-tilt exclusions. @@ -6908,7 +10544,12 @@ def createNewSetOfTiltSeriesService( Returns a JSON-serializable payload for the web UI with basic metadata. """ # Resolve protocol and input set for the given output - protocol, inputSet = self._resolveOutputForTiltSeries(protocolId, outputName) + protocol, inputSet = self._resolveOutputForTiltSeries( + protocolId=protocolId, + outputName=outputName, + projectId=projectId, + mapper=mapper, + ) hasOddEven = inputSet.hasOddEven() # Normalize exclusions keys to strings (tsId can be int/str on the backend) @@ -7088,6 +10729,18 @@ def createNewSetOfTiltSeriesService( outputSet.write() protocol._defineOutputs(**{newOutputName: outputSet}) protocol._store() + + postgresqlStore = self._storeGeneratedSetInPostgresql( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=newOutputName, + scipionSet=outputSet, + contextLabel="TiltSeries", + ) + postgresqlSync = postgresqlStore["postgresqlSync"] + postgresqlError = postgresqlStore["postgresqlError"] + logger.info("The new set (%s) has been created successfully", newOutputName) return { @@ -7096,6 +10749,8 @@ def createNewSetOfTiltSeriesService( "createdTiltSeries": createdCount, "hasOddEven": bool(hasOddEven), "restack": bool(restack), + "postgresqlSync": postgresqlSync, + "postgresqlError": postgresqlError, } def _cloneTiltImage(self, ti, included): @@ -7109,11 +10764,73 @@ def _cloneTiltImage(self, ti, included): # ---------------------------------------------------------------------- # Analyze Results: Coordinates3D # ---------------------------------------------------------------------- + def _getPostgresqlCoords3dReaderIfAvailable( + self, + mapper, + projectId: int, + protocolId: int, + outputName: str, + ): + if mapper is None: + return None + + try: + from app.backend.viewers.postgresql_coords3d_reader import PostgresqlCoords3dReader + + readerProtocolId = self._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + reader = PostgresqlCoords3dReader( + db=mapper.db, + projectId=projectId, + protocolId=readerProtocolId, + outputName=outputName, + ) + + if reader.hasOutput(): + return reader + + except Exception: + logger.exception( + "Failed to initialize PostgreSQL Coordinates3D reader. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + ) + + return None + + def _resolveOutputForCoordinates3d( + self, + protocolId: int, + outputName: str, + mapper=None, + projectId: Optional[int] = None, + ): + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + output = getattr(protocol, outputName, None) + if output is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Output '{outputName}' not found in protocol", + ) + + return protocol, output + def listCoordinates3dTomogramsService( self, projectId: int, protocolId: int, outputName: str, + mapper=None, ): """ Return a list of tomograms referenced by the SetOfCoordinates3D output. @@ -7124,17 +10841,39 @@ def listCoordinates3dTomogramsService( ... ] """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + pgReader = self._getPostgresqlCoords3dReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.listTomograms() + if payload is not None: + self.tomoList = {} + return payload + + logger.info( + "Skipping PostgreSQL Coordinates3D tomograms reader. projectId=%s protocolId=%s outputName=%s reason=%s", + projectId, + protocolId, + outputName, + getattr(pgReader, "lastSkipReason", None), + ) - setOfCoordinates3D = getattr(protocol, outputName, None) - if setOfCoordinates3D is None: + if mapper is not None: raise HTTPException( status_code=404, - detail=f"Output '{outputName}' not found in protocol", + detail="Coordinates3D output is not available in PostgreSQL metadata", ) + + protocol, setOfCoordinates3D = self._resolveOutputForCoordinates3d( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) self.tomoList = {} tomogramList: List[Dict[str, Any]] = [] tomosIter = None @@ -7221,6 +10960,7 @@ def getCoordinates3dPointsService( protocolId: int, outputName: str, tomogramId: Union[int, str], + mapper=None, ): """ Return a flat list of 3D points for one tomogram. @@ -7244,18 +10984,43 @@ def getCoordinates3dPointsService( ... ] """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + pgReader = self._getPostgresqlCoords3dReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) - setOfCoordinates3D = getattr(protocol, outputName, None) - if setOfCoordinates3D is None: - raise HTTPException( - status_code=404, - detail=f"Output '{outputName}' not found in protocol", + if pgReader is not None: + payload = pgReader.getPoints(tomogramId) + if payload is not None: + return payload + + logger.info( + "Skipping PostgreSQL Coordinates3D points reader. projectId=%s protocolId=%s outputName=%s tomogramId=%s reason=%s", + projectId, + protocolId, + outputName, + tomogramId, + getattr(pgReader, "lastSkipReason", None), + ) + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Coordinates3D points", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", ) + protocol, setOfCoordinates3D = self._resolveOutputForCoordinates3d( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) + try: boxSize = float(setOfCoordinates3D.getBoxSize()) except Exception: @@ -7475,6 +11240,7 @@ def renderCoords3dTomogramSliceService( thumb: Optional[int] = 128, fast: bool = True, quality: int = 75, + mapper=None, ) -> Response: """ Render a 2D slice from a tomogram referenced by a SetOfCoordinates3D. @@ -7490,18 +11256,58 @@ def renderCoords3dTomogramSliceService( """ from PIL import Image as PILImage - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + pgReader = self._getPostgresqlCoords3dReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) - setOfCoordinates3D = getattr(protocol, outputName, None) - if setOfCoordinates3D is None: - raise HTTPException( - status_code=404, - detail=f"Output '{outputName}' not found in protocol", + if pgReader is not None: + tomogramInfo = pgReader.getTomogramFile(tomogramId) + if tomogramInfo is not None: + volumePath = tomogramInfo.get("fileName") + if volumePath and os.path.exists(volumePath): + return self._renderTomogramSliceFromPath( + volumePath=volumePath, + tomogramId=tomogramId, + sliceIndex=sliceIndex, + axis=axis, + colormap=colormap, + normalize=normalize, + scale=scale, + inline=inline, + fmt=fmt, + thumb=thumb, + fast=fast, + quality=quality, + ) + + logger.info( + "Skipping PostgreSQL Coordinates3D tomogram slice reader. projectId=%s protocolId=%s outputName=%s tomogramId=%s reason=%s", + projectId, + protocolId, + outputName, + tomogramId, + getattr(pgReader, "lastSkipReason", None), + ) + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Coordinates3D tomogram slice", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", ) + protocol, setOfCoordinates3D = self._resolveOutputForCoordinates3d( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) + try: if self.tomoList: tomogram = self.tomoList[tomogramId] @@ -7523,12 +11329,219 @@ def renderCoords3dTomogramSliceService( detail="Tomogram object has no getFileName()", ) - volumePath = getFileNameFn() - if not volumePath or not os.path.exists(volumePath): - raise HTTPException( - status_code=404, - detail="Tomogram file not found on disk", - ) + volumePath = getFileNameFn() + if not volumePath or not os.path.exists(volumePath): + raise HTTPException( + status_code=404, + detail="Tomogram file not found on disk", + ) + + axis = (axis or "z").lower() + if axis not in ("x", "y", "z"): + axis = "z" + + fmtLower = (fmt or "png").lower() + if fmtLower in ("jpg", "jpeg"): + pilFormat = "JPEG" + mediaType = "image/jpeg" + saveKw = {"quality": int(quality or 75)} + elif fmtLower == "webp": + pilFormat = "WEBP" + mediaType = "image/webp" + saveKw = {"quality": int(quality or 75)} + else: + pilFormat = "PNG" + mediaType = "image/png" + saveKw = {} + + usedColormap = colormap + gray: Optional[np.ndarray] = None + depth = 1 + + try: + requestedIndex = int(sliceIndex or 0) + except Exception: + requestedIndex = 0 + requestedIndex = max(0, requestedIndex) + + sliceUsed = requestedIndex + if axis == "z" and fast: + try: + reader = ImageReadersRegistry.open(volumePath) + + try: + images = reader.getImages() + if hasattr(images, "ndim") and images.ndim == 3: + zdim, ydim, xdim = int(images.shape[0]), int(images.shape[1]), int(images.shape[2]) + elif hasattr(images, "ndim") and images.ndim == 2: + zdim, ydim, xdim = 1, int(images.shape[0]), int(images.shape[1]) + else: + zdim, ydim, xdim = 1, 0, 0 + except Exception: + zdim, ydim, xdim = 1, 0, 0 + + depth = max(zdim, 1) + + k = requestedIndex + if zdim > 0: + k = max(0, min(k, zdim - 1)) + + try: + pilImg = reader.getImage(index=k, pilImage=True) + except Exception: + try: + pilImg = reader.getCentralImage(pilImage=True) + if zdim > 0: + k = max(0, min(zdim // 2, max(zdim - 1, 0))) + else: + k = 0 + except Exception: + pilImg = reader.getImage(index=0, pilImage=True) + k = 0 + + arr2d = self._coords3dPilTo2dTile(reader, pilImg) + if arr2d is None: + arrRaw = np.asarray(pilImg) + if arrRaw.ndim == 3: + arr2d = arrRaw.mean(axis=-1) + else: + arr2d = arrRaw.astype(np.float32, copy=False) + + gray = self._normalize2dSlice(arr2d, mode=normalize) + sliceUsed = k + except Exception: + gray = None + + if gray is None: + try: + vol3d, _props = readVolumeArray3d(str(volumePath)) # Z, Y, X + except HTTPException: + raise + except FileNotFoundError: + raise HTTPException( + status_code=404, + detail="Tomogram file not found on disk", + ) + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"Failed to read tomogram volume: {e}", + ) + + if vol3d.ndim != 3: + raise HTTPException( + status_code=500, + detail=f"Invalid tomogram volume shape {vol3d.shape}", + ) + + zdim, ydim, xdim = int(vol3d.shape[0]), int(vol3d.shape[1]), int(vol3d.shape[2]) + depth = max(zdim, 1) + + if axis == "z": + dim = zdim + elif axis == "y": + dim = ydim + else: + dim = xdim + + if dim <= 0: + raise HTTPException(status_code=500, detail="Empty tomogram volume") + + k = max(0, min(requestedIndex, dim - 1)) + + if axis == "z": + slice2d = vol3d[k, :, :] + elif axis == "y": + slice2d = vol3d[:, k, :] + else: + slice2d = vol3d[:, :, k] + + gray = self._normalize2dSlice(slice2d, mode=normalize) + sliceUsed = k + + if thumb is not None and thumb > 0: + pilTmp = PILImage.fromarray(gray.astype(np.uint8), mode="L") + pilTmp.thumbnail((thumb, thumb)) + gray = np.asarray(pilTmp) + + if gray.dtype != np.uint8: + gray = gray.astype(np.uint8, copy=False) + + imgArray = gray.astype(np.uint8, copy=False) + pilMode = "L" + + if usedColormap: + try: + import matplotlib.cm as cm + sliceNorm = imgArray.astype(np.float32) / 255.0 + cmapObj = cm.get_cmap(usedColormap) + rgba = cmapObj(sliceNorm) + rgb = (rgba[..., :3] * 255.0).clip(0, 255).astype(np.uint8) + imgArray = rgb + pilMode = "RGB" + except Exception: + usedColormap = None + imgArray = gray.astype(np.uint8, copy=False) + pilMode = "L" + + if scale is not None and scale != 1.0: + try: + pilScale = PILImage.fromarray(imgArray, mode=pilMode) + newW = max(1, int(round(pilScale.width * float(scale)))) + newH = max(1, int(round(pilScale.height * float(scale)))) + pilScale = pilScale.resize((newW, newH), resample=PILImage.Resampling.BILINEAR) + imgArray = np.asarray(pilScale, copy=False) + except Exception: + pass + + img = PILImage.fromarray(imgArray, mode=pilMode) + + buf = io.BytesIO() + img.save(buf, format=pilFormat, **saveKw) + + disp = "inline" if inline else "attachment" + filename = f"coords3d_{tomogramId}_axis-{axis}_slice-{sliceUsed}.{fmtLower}" + + headers = { + "Content-Disposition": f'{disp}; filename="{filename}"', + "Access-Control-Expose-Headers": ( + "Content-Disposition, " + "X-Preview-Mime, " + "X-Preview-Width, " + "X-Preview-Height, " + "X-Preview-Depth, " + "X-Preview-Colormap, " + "X-Preview-Format, " + "X-Preview-TomogramId" + ), + "X-Preview-Mime": mediaType, + "X-Preview-Width": str(img.width), + "X-Preview-Height": str(img.height), + "X-Preview-Depth": str(depth), + "X-Preview-Colormap": usedColormap or "", + "X-Preview-Format": pilFormat, + "X-Preview-TomogramId": str(tomogramId), + } + + return Response(content=buf.getvalue(), media_type=mediaType, headers=headers) + + def _renderTomogramSliceFromPath( + self, + volumePath: str, + tomogramId: Union[int, str], + sliceIndex: int, + axis: str = "z", + colormap: Optional[str] = None, + normalize: Optional[str] = "minmax", + scale: float = 1.0, + inline: bool = True, + fmt: str = "webp", + thumb: Optional[int] = 128, + fast: bool = True, + quality: int = 75, + ) -> Response: + + from PIL import Image as PILImage axis = (axis or "z").lower() if axis not in ("x", "y", "z"): @@ -7725,6 +11738,7 @@ def createCoords3dOutputFromPointsService( protocolId: int, outputName: str, payload: Any, + mapper=None ) -> Dict[str, Any]: tomograms = payload['tomograms'] @@ -7732,14 +11746,14 @@ def createCoords3dOutputFromPointsService( # --------------------------------- # 1. Obtaining protocol and origin # --------------------------------- - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(404, "Protocol not found") + protocol, srcSet = self._resolveOutputForCoordinates3d( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) - srcSet = getattr(protocol, outputName, None) - if srcSet is None: - raise HTTPException(404, f"Output '{outputName}' not found in protocol") + runtimeProtocolId = getattr(protocol, "getObjId", lambda: protocolId)() # ------------------------------- # 2. Ensure tomograms @@ -7747,8 +11761,9 @@ def createCoords3dOutputFromPointsService( if not self.tomoList: self.listCoordinates3dTomogramsService( projectId=projectId, - protocolId=protocolId, + protocolId=runtimeProtocolId, outputName=outputName, + mapper=None, ) # ----------------------------------- @@ -7799,7 +11814,7 @@ def createCoords3dOutputFromPointsService( replaced += 1 break # ------------------------------------ - # 7. Saving and registering the output + # 5. Saving and registering the output # ------------------------------------ try: dstSet.write() @@ -7811,6 +11826,17 @@ def createCoords3dOutputFromPointsService( except Exception as e: raise HTTPException(500, f"Failed to attach new coords3d output: {e}") + postgresqlStore = self._storeGeneratedSetInPostgresql( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outName, + scipionSet=dstSet, + contextLabel="Coordinates3D", + ) + postgresqlError = postgresqlStore["postgresqlError"] + postgresqlStored = mapper is not None and postgresqlError is None + return { "success": True, "outputName": outName, @@ -7819,6 +11845,8 @@ def createCoords3dOutputFromPointsService( "sourceOutputName": outputName, "replacedPoints": replaced, "copiedPoints": copied, + "postgresqlStored": postgresqlStored, + "postgresqlError": postgresqlError, }, } @@ -7830,28 +11858,25 @@ def getFscRowsService( projectId: int, protocolId: int, outputName: str, + mapper=None, ) -> Dict[str, Any]: """ Return FSC curves for a SetOfFSCs-like output. - - Response shape: - { - "threshold": 0.143, - "rows": [ - { - "label": "No mask", - "resolution": 3.21, - "x": [0.0, 0.01, 0.02, ...], - "y": [1.0, 0.98, 0.95, ...], - }, - ... - ], - } """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="FSC", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="reader_not_available", + ) + + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) output = getattr(protocol, outputName, None) if output is None: @@ -7977,7 +12002,13 @@ def getXY(fsc): # Internal helpers for metadata tables (STAR / SQLITE / etc.) # ====================================================================== - def _resolveOutputForMetadata(self, protocolId: int, outputName: str): + def _resolveOutputForMetadata( + self, + protocolId: int, + outputName: str, + mapper=None, + projectId: Optional[int] = None, + ): """ Resolve protocol and output object for metadata operations. @@ -7986,10 +12017,11 @@ def _resolveOutputForMetadata(self, protocolId: int, outputName: str): project-relative. - Raises 404 if the final path does not exist on disk. """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) if not hasattr(protocol, outputName): raise HTTPException( @@ -8067,20 +12099,148 @@ def _getMetadataObjectManager(self, metaPath: str) -> ObjectManager: objMgr.getTables() return objMgr - def _openMetadataTable(self, protocolId: int, outputName: str, tableName: str): + def _getPostgresqlDAOIfAvailable( + self, + projectId: int, + protocolId: int, + outputName: str, + mapper=None, + ): + from app.backend.viewers.postgresql_dao import PostgresqlDAO + + if mapper is None: + return None + + readerProtocolId = self._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + dao = PostgresqlDAO( + db=mapper.db, + projectId=projectId, + protocolId=readerProtocolId, + outputName=outputName, + ) + + if dao.hasOutput(): + return dao + + return None + + def _getMetadataObjectManagerForOutput( + self, + projectId: int, + protocolId: int, + outputName: str, + mapper=None, + ) -> ObjectManager: + pgDao = self._getPostgresqlDAOIfAvailable( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + ) + + if pgDao is not None: + objMgr = self._createObjectManager() + objMgr._fileName = "postgresql://project/%s/protocol/%s/output/%s" % ( + projectId, + protocolId, + outputName, + ) + objMgr._dao = pgDao + objMgr._tables = {} + objMgr.getTables() + return objMgr + + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Metadata", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="dao_not_available", + ) + + _, _, metaPath = self._resolveOutputForMetadata( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) + return self._getMetadataObjectManager(metaPath) + + def _openMetadataTable( + self, + projectId: int, + protocolId: int, + outputName: str, + tableName: str, + mapper=None, + ): """ Resolve (ObjectManager, Table) for a given output + tableName. + + PostgreSQL persisted outputs and SQLite/STAR outputs are both exposed + through ObjectManager. """ - _, _, metaPath = self._resolveOutputForMetadata(protocolId, outputName) - objMgr = self._getMetadataObjectManager(metaPath) + objMgr = self._getMetadataObjectManagerForOutput( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + ) + table = objMgr.getTable(tableName) if table is None: raise HTTPException( status_code=404, - detail=f"Metadata table '{tableName}' not found" + detail=f"Metadata table '{tableName}' not found", ) + return objMgr, table + def _isPropertiesMetadataTable(self, table) -> bool: + try: + tableName = str(table.getName() or "").strip() + except Exception: + tableName = "" + + try: + tableAlias = str(table.getAlias() or "").strip() + except Exception: + tableAlias = "" + + return tableName == "Properties" or tableAlias == "Properties" + + def _getMetadataTableActionNames(self, table) -> List[str]: + if self._isPropertiesMetadataTable(table): + return [] + + try: + tableActions = table.getActions() or [] + except Exception: + return [] + + actionNames: List[str] = [] + seen = set() + + for action in tableActions: + try: + actionName = str(action.getName() or "").strip() + except Exception: + actionName = "" + + if not actionName or actionName in seen: + continue + + seen.add(actionName) + actionNames.append(actionName) + + return actionNames + def _rendererTypeFromInstance(self, renderer) -> str: """ Map renderer class name to a simple type label for the API. @@ -8143,21 +12303,28 @@ def _convertCellForPage(self, renderer, rawValue, rowValues): def listOutputMetadataTablesService(self, projectId: int, protocolId: int, - outputName: str): + outputName: str, + mapper=None): """ List logical metadata tables associated with an output. """ with _metadataLock: - _, _, metaPath = self._resolveOutputForMetadata(protocolId, outputName) - objMgr = self._getMetadataObjectManager(metaPath) + objMgr = self._getMetadataObjectManagerForOutput( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + ) tables = objMgr.getTables() or {} items = [] + for name, table in tables.items(): try: rowCount = objMgr.getTableRowCount(name) or 0 except Exception: rowCount = 0 + hasColumnId = True try: hasColumnId = table.hasColumnId() @@ -8176,108 +12343,237 @@ def listOutputMetadataTablesService(self, projectId: int, def getMetadataTableSchemaService(self, projectId: int, protocolId: int, outputName: str, - tableName: str): + tableName: str, + mapper=None): """ Return logical schema for one metadata table: columns, renderers, flags. """ with _metadataLock: - objMgr, table = self._openMetadataTable(protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable( + projectId, + protocolId, + outputName, + tableName, + mapper=mapper, + ) visibleLabels = [] orderLabels = [] renderLabels = [] - actions = [] + actions = self._getMetadataTableActionNames(table) - if table.getName() == SQLITE_OBJECT_TABLE: - from pwem.viewers.viewers_data import RegistryViewerConfig - protocol = self.currentProject.getProtocol(int(protocolId)) - output = getattr(protocol, outputName) + try: + hasColumnId = table.hasColumnId() + except Exception: + hasColumnId = True - config = RegistryViewerConfig.getConfig(type(output)) or {} + columns = list(table.getColumns()) + columnNames = {str(col.getName()) for col in columns} - fileNameLabel = ' _filename' - stackLabel = ' stack' + additionalInfoColumns = [] + try: + dao = objMgr.getDAO() + getAdditionalInfoFn = getattr(dao, "getTableWithAdditionalInfo", None) - visibleLabelsStr = config.get(VISIBLE, '') - orderLabelsStr = config.get(ORDER, '') - renderLabelsStr = config.get(RENDER, '') + if callable(getAdditionalInfoFn): + additionalTable, additionalColumns = getAdditionalInfoFn() - orderLabelsStr = orderLabelsStr.replace(fileNameLabel, stackLabel, 1) - renderLabelsStr = renderLabelsStr.replace(fileNameLabel, stackLabel, 1) + if hasattr(additionalTable, "getName"): + additionalTableName = str(additionalTable.getName() or "") + else: + additionalTableName = "" if additionalTable is None else str(additionalTable) - if fileNameLabel in visibleLabelsStr and stackLabel not in renderLabelsStr: - renderLabelsStr += stackLabel - visibleLabelsStr += stackLabel + currentTableNames = { + str(tableName or ""), + str(table.getName() or ""), + str(table.getAlias() or ""), + } - visibleLabels = visibleLabelsStr.split() - orderLabels = orderLabelsStr.split() - renderLabels = renderLabelsStr.split() - for action in table.getActions(): - actions.append(action.getName()) + appliesToTable = not additionalTableName or additionalTableName in currentTableNames - try: - hasColumnId = table.hasColumnId() + if appliesToTable: + additionalInfoColumns = [ + str(columnName) + for columnName in (additionalColumns or []) + if str(columnName) in columnNames + ] except Exception: - hasColumnId = True + additionalInfoColumns = [] - columns = list(table.getColumns()) schema = { "name": tableName, "alias": table.getAlias(), "hasColumnId": bool(hasColumnId), "actions": actions, + "additionalInfoColumns": additionalInfoColumns, # "renderLabels": renderLabels, # "orderLabels": orderLabels, "columns": [], } - for idx, col in enumerate(columns): - try: - col.setIndex(idx) - except Exception: - pass + for idx, col in enumerate(columns): + try: + col.setIndex(idx) + except Exception: + pass + + renderer = col.getRenderer() + rendererType = self._rendererTypeFromInstance(renderer) + + decimals = None + if hasattr(renderer, "getDecimalsNumber"): + try: + decimals = renderer.getDecimalsNumber() + except Exception: + decimals = None + + hasTransformation = False + if hasattr(renderer, "hasTransformation"): + try: + hasTransformation = bool(renderer.hasTransformation()) + except Exception: + hasTransformation = False + + sortable = True + if hasattr(col, "isSorteable"): + try: + sortable = bool(col.isSorteable()) + except Exception: + sortable = True + + try: + visible = col.getName() in visibleLabels if visibleLabels else True + except Exception: + visible = True + + schema["columns"].append({ + "name": col.getName(), + "alias": col.getAlias() or col.getName(), + "index": idx, + "sortable": sortable, + "visible": visible, + "rendererType": rendererType, + "decimals": decimals, + "hasTransformation": hasTransformation, + }) + + return schema + + def _normalizeMetadataSelectionIds(self, ids: List[int]) -> List[int]: + normalizedIds: List[int] = [] + + for rowId in ids or []: + try: + normalizedIds.append(int(rowId)) + except Exception: + continue + + if not normalizedIds: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Missing ids", + ) + + return normalizedIds + + def _writeMetadataSelectionFile(self, ids: List[int]) -> str: + timeFormat = "%Y%m%d%H%M%S" + timestamp = datetime.now().strftime(timeFormat) + relativePath = "Logs/selection_%s.txt" % timestamp + + projectPath = None + try: + projectPath = self.currentProject.getPath() + except Exception: + projectPath = None + + if projectPath: + absolutePath = os.path.join(projectPath, relativePath) + else: + absolutePath = relativePath + + directoryPath = os.path.dirname(absolutePath) + if directoryPath: + os.makedirs(directoryPath, exist_ok=True) + + try: + with open(absolutePath, "w") as file: + for rowId in ids: + file.write(str(rowId) + " ") + + logger.debug("Metadata selection file was created: %s", absolutePath) + except Exception as e: + logger.exception("Error creating metadata selection file: %s", absolutePath) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error creating selection file: {e}", + ) + + return relativePath + + def _buildMetadataSelectionArgument(self, selectionPath: str, tableName: str) -> str: + selectionArg = selectionPath + "," + + if tableName != OBJECT_TABLE: + selectionArg += tableName.split("_Objects")[0] + + return selectionArg + + def _getMetadataActionAliasForTable(self, dao, table) -> str: + getActionAliasFn = getattr(dao, "_getActionAliasForTableName", None) + if callable(getActionAliasFn): + try: + actionAlias = str(getActionAliasFn(table.getName()) or "").strip() + if actionAlias: + return actionAlias + except Exception: + pass + + try: + return str(table.getAlias() or "").strip() + except Exception: + return "" + + def _resolveMetadataActionOutputClassName(self, dao, table, action: str) -> str: + actionName = str(action or "").strip() + if not actionName: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Missing action", + ) + + validActions = self._getMetadataTableActionNames(table) + if actionName not in validActions: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Unsupported action '{actionName}' for table '{table.getName()}'", + ) + + actionAlias = self._getMetadataActionAliasForTable(dao, table) - renderer = col.getRenderer() - rendererType = self._rendererTypeFromInstance(renderer) + if actionAlias == "Class2D" and actionName == "Averages": + return "SetOfAverages" - decimals = None - if hasattr(renderer, "getDecimalsNumber"): - try: - decimals = renderer.getDecimalsNumber() - except Exception: - decimals = None + if actionAlias == "Class3D" and actionName == "Volumes": + return "SetOfVolumes" - hasTransformation = False - if hasattr(renderer, "hasTransformation"): - try: - hasTransformation = bool(renderer.hasTransformation()) - except Exception: - hasTransformation = False + objectsType = getattr(dao, "_objectsType", {}) or {} - sortable = True - if hasattr(col, "isSorteable"): - try: - sortable = bool(col.isSorteable()) - except Exception: - sortable = True + if actionAlias == "Class2D": + objectsType.setdefault("Averages", "SetOfAverages") - try: - visible = col.getName() in visibleLabels if visibleLabels else True - except Exception: - visible = True + if actionAlias == "Class3D": + objectsType.setdefault("Volumes", "SetOfVolumes") - schema["columns"].append({ - "name": col.getName(), - "alias": col.getAlias() or col.getName(), - "index": idx, - "sortable": sortable, - "visible": visible, - "rendererType": rendererType, - "decimals": decimals, - "hasTransformation": hasTransformation, - }) + outputClassName = objectsType.get(actionName) - return schema + if outputClassName: + return str(outputClassName) + + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Could not resolve output class for action '{actionName}'", + ) def runMetadataTableActionService( self, @@ -8291,89 +12587,133 @@ def runMetadataTableActionService( currentUser: Any, mapper: Any, ) -> Any: - timeFormat = '%Y%m%d%H%M%S' - now = datetime.now() - timestamp = now.strftime(timeFormat) - path = 'Logs/selection_%s.txt' % timestamp - try: - with open(path, 'w') as file: - for rowId in ids: - file.write(str(rowId) + ' ') - file.close() - logger.debug(f"The file: {path} was created correctly.") - except Exception as e: - logger.error(f"Error creating the file: {e}") - path += "," # Always add a comma, it is expected by the user subset protocol - if tableName != OBJECT_TABLE: - path += tableName.split('_Objects')[0] + selectionIds = self._normalizeMetadataSelectionIds(ids) - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) if not hasattr(protocol, outputName): raise HTTPException( status_code=404, - detail=f"Output '{outputName}' not found in protocol" + detail=f"Output '{outputName}' not found in protocol", ) output = getattr(protocol, outputName) if output is None: raise HTTPException( status_code=404, - detail=f"Output '{outputName}' is None" + detail=f"Output '{outputName}' is None", ) - getFileNameFn = getattr(output, "getFileName", None) - if not callable(getFileNameFn): - raise HTTPException( - status_code=404, - detail="Output has no metadata file (getFileName not available)" + with _metadataLock: + objMgr, table = self._openMetadataTable( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + tableName=tableName, + mapper=mapper, ) - objMgr = self._getMetadataObjectManager(str(output.getFileName())) + dao = objMgr.getDAO() + outputClassName = self._resolveMetadataActionOutputClassName( + dao=dao, + table=table, + action=action, + ) + + selectionPath = self._writeMetadataSelectionFile(selectionIds) + selectionArg = self._buildMetadataSelectionArgument( + selectionPath=selectionPath, + tableName=tableName, + ) + + try: + batchProt = self.currentProject.newProtocol( + ProtUserSubSet, + inputObject=output, + sqliteFile=selectionArg, + outputClassName=outputClassName, + other="", + label=subsetName, + ) - dao = objMgr.getDAO() - table = objMgr.getTable(tableName) - dao.fillTable(table, objMgr) - if table.getAlias() == 'Class2D': - dao._objectsType['Averages'] = 'SetOfAverages' - elif table.getAlias() == 'Class3D': - dao._objectsType['Volumes'] = 'SetOfVolumes' - outputClassName = dao._objectsType[action] - - try: - batchProt = self.currentProject.newProtocol(ProtUserSubSet, - inputObject=output, - sqliteFile=path, - outputClassName=outputClassName, - other='', - label=subsetName) self.currentProject.launchProtocol(batchProt) - return True + + postgresqlSync = None + try: + postgresqlSync = self.syncProjectProtocolsAndDependencies( + mapper, + projectId, + refresh=True, + checkPid=True, + ) + except Exception as syncError: + logger.exception( + "Subset protocol was launched but PostgreSQL sync failed. projectId=%s protocolId=%s outputName=%s tableName=%s", + projectId, + protocolId, + outputName, + tableName, + ) + return { + "success": True, + "message": "Subset protocol was launched successfully, but PostgreSQL sync failed", + "postgresqlSync": None, + "postgresqlError": str(syncError), + } + + return { + "success": True, + "message": "Subset protocol was launched successfully", + "postgresqlSync": postgresqlSync, + "postgresqlError": None, + } + except Exception as e: - return False + logger.exception( + "Failed to launch metadata table action. projectId=%s protocolId=%s outputName=%s tableName=%s action=%s", + projectId, + protocolId, + outputName, + tableName, + action, + ) + return { + "success": False, + "errors": [str(e)], + } def getMetadataTablePageService( - self, - projectId: int, - protocolId: int, - outputName: str, - tableName: str, - page: int, - pageSize: int, - sortBy: str, - asc: bool, - selectionOnly: bool, + self, + projectId: int, + protocolId: int, + outputName: str, + tableName: str, + page: int, + pageSize: int, + sortBy: str, + asc: bool, + selectionOnly: bool, + mapper=None, ): """ Return one logical page of rows for a metadata table. Sorting is currently delegated to the underlying DAO default order. """ with _metadataLock: - objMgr, table = self._openMetadataTable(protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable( + projectId, + protocolId, + outputName, + tableName, + mapper=mapper, + ) columns = list(table.getColumns()) + table.setSortingColumn(sortBy) + table.setSortingAsc(asc) if selectionOnly: rows: list = [] @@ -8442,14 +12782,15 @@ def getMetadataTablePageService( } def exportMetadataTableService( - self, - projectId: int, - protocolId: int, - outputName: str, - tableName: str, - fmt: str, - selectionOnly: bool, - ids: Optional[List[int]], + self, + projectId: int, + protocolId: int, + outputName: str, + tableName: str, + fmt: str, + selectionOnly: bool, + ids: Optional[List[int]], + mapper=None, ) -> Response: """ Export metadata table as CSV or XLSX. @@ -8461,7 +12802,13 @@ def exportMetadataTableService( import csv with _metadataLock: - objMgr, table = self._openMetadataTable(protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable( + projectId, + protocolId, + outputName, + tableName, + mapper=mapper, + ) columns = list(table.getColumns()) colNames = [c.getName() for c in columns] @@ -8639,6 +12986,7 @@ def renderMetadataImageCellService( inline: bool, fmt: str, rowIndex: Optional[int] = None, + mapper=None, ) -> Response: """ Render one image cell from a metadata table using ImageRenderer. @@ -8652,18 +13000,25 @@ def renderMetadataImageCellService( from pathlib import Path as LocalPath # Resolve metadata root path for relative image paths - try: - _protocol, _output, metaPath = self._resolveOutputForMetadata(protocolId, outputName) - metaDir = LocalPath(metaPath).parent - except HTTPException: - # If the output / metadata file is really missing, that is a real error - raise - except Exception: - metaDir = None - - objMgr, table = self._openMetadataTable(protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable( + projectId, + protocolId, + outputName, + tableName, + mapper=mapper, + ) columns = list(table.getColumns()) + metaDir = None + objMgrFileName = str(getattr(objMgr, "_fileName", "") or "") + + if not objMgrFileName.startswith("postgresql://"): + try: + _protocol, _output, metaPath = self._resolveOutputForMetadata(protocolId, outputName) + metaDir = LocalPath(metaPath).parent + except Exception: + metaDir = None + # Resolve column index colIndex = table.getColumnIndexFromLabel(columnName) if colIndex < 0 or colIndex >= len(columns): @@ -8805,7 +13160,12 @@ def renderMetadataImageCellService( try: if self.currentProject is not None: projPath = LocalPath(self.currentProject.getPath()) - prot = self.currentProject.getProtocol(int(protocolId)) + + prot = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) protPath = LocalPath(prot.getPath()) except Exception: protPath = None @@ -8917,6 +13277,7 @@ def getMetadataTableWindowService( selectionOnly: bool, sortBy: str, asc: bool, + mapper=None, ): """ Return a window of rows for a metadata table using offset + limit. @@ -8931,7 +13292,13 @@ def getMetadataTableWindowService( - number of selected rows if selectionOnly == True """ with _metadataLock: - objMgr, table = self._openMetadataTable(protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable( + projectId, + protocolId, + outputName, + tableName, + mapper=mapper, + ) columns = list(table.getColumns()) table.setSortingColumn(sortBy) table.setSortingAsc(asc) @@ -9190,29 +13557,17 @@ def _unwrapScipionObject(self, obj: Any) -> Any: return obj def _getProtocolOutputObject( - self, - protocolId: int, - outputName: str, + self, + protocolId: int, + outputName: str, + mapper=None, + projectId: Optional[int] = None, ) -> Tuple[Any, Any]: - if self.currentProject is None: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="No current project loaded", - ) - - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception as e: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}. {e}", - ) - - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) outputObj = None @@ -9388,15 +13743,20 @@ def _resolveExternalViewerSetItemByPublicId( return None def listExternalViewers( - self, - protocolId: int, - outputName: str, - objectId: Optional[Union[str, int]] = None, - objectKind: Optional[str] = None, + self, + protocolId: int, + outputName: str, + objectId: Optional[Union[str, int]] = None, + objectKind: Optional[str] = None, + mapper=None, + projectId: Optional[int] = None, ) -> List[Dict[str, Any]]: + protocol, outputObj = self._getProtocolOutputObject( protocolId=protocolId, outputName=outputName, + mapper=mapper, + projectId=projectId, ) targetObj = self._resolveExternalViewerTargetObject( @@ -9529,17 +13889,22 @@ def _runExternalViewer(self, viewerClass: Any, protocol: Any, targetObj: Any): self._showExternalView(view) def launchExternalViewer( - self, - protocolId: int, - outputName: str, - viewerId: str, - objectId: Optional[Union[str, int]] = None, - objectKind: Optional[str] = None, - params: Optional[Dict[str, Any]] = None, + self, + protocolId: int, + outputName: str, + viewerId: str, + objectId: Optional[Union[str, int]] = None, + objectKind: Optional[str] = None, + params: Optional[Dict[str, Any]] = None, + mapper=None, + projectId: Optional[int] = None, ) -> Dict[str, Any]: + protocol, outputObj = self._getProtocolOutputObject( protocolId=protocolId, outputName=outputName, + mapper=mapper, + projectId=projectId, ) targetObj = self._resolveExternalViewerTargetObject( @@ -9723,33 +14088,72 @@ def deleteProjectTag( ) def listProtocolTags( - self, - mapper, - projectId: int, - protocolId: int, - currentUser: dict, + self, + mapper, + projectId: int, + protocolId: int, + currentUser: dict, ) -> Dict[str, Any]: # listProtocolTags + protocolDbId = self._resolvePostgresqlProtocolDbId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + if protocolDbId is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol not found in PostgreSQL: {protocolId}", + ) + try: - tagIds = mapper.getProtocolTagIds(projectId=projectId, protocolDbId=int(protocolId)) + tagIds = mapper.getProtocolTagIds( + projectId=projectId, + protocolDbId=protocolDbId, + ) except Exception as e: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to list protocol tags: {e}", ) - return {"tagIds": tagIds} + return { + "protocolId": str(protocolId), + "protocolDbId": protocolDbId, + "tagIds": tagIds, + } def setProtocolTags( - self, - mapper, - projectId: int, - protocolId: int, - tagIds: List[str], - currentUser: dict, + self, + mapper, + projectId: int, + protocolId: int, + tagIds: List[str], + currentUser: dict, ) -> Dict[str, Any]: # setProtocolTags + protocolDbId = self._resolvePostgresqlProtocolDbId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + if protocolDbId is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol not found in PostgreSQL: {protocolId}", + ) + try: + setByDbId = getattr(mapper, "setProtocolTagIdsByProtocolDbId", None) + if callable(setByDbId): + return setByDbId( + projectId=projectId, + protocolDbId=protocolDbId, + tagIds=tagIds or [], + ) + return mapper.setProtocolTagIds( projectId=projectId, protocolId=int(protocolId), diff --git a/app/backend/api/services/protocol_wizard_service.py b/app/backend/api/services/protocol_wizard_service.py index 45c129e2..946a5006 100644 --- a/app/backend/api/services/protocol_wizard_service.py +++ b/app/backend/api/services/protocol_wizard_service.py @@ -335,9 +335,11 @@ def _classifyWizardKind( return "unknown" def _applyFormValuesToProtocolInstance( - self, - protocol, - params: Dict[str, Any], + self, + protocol, + params: Dict[str, Any], + mapper=None, + projectId: Optional[int] = None, ) -> List[str]: if self.projectService is None: raise RuntimeError("projectService is required to apply wizard form values") @@ -390,22 +392,28 @@ def _applyFormValuesToProtocolInstance( cleaned = re.sub(r"[^A-Za-z0-9\s+\-*/=<>\!&|^%()\[\]{}_,.;:]", "", str(e)) errorList.append("**" + param.label.get() + "** " + cleaned) - errorList += self.projectService.applyParamsToProtocol(protocol, params) + errorList += self.projectService.applyParamsToProtocol( + mapper=mapper, + projectId=projectId, + protocol=protocol, + params=params, + ) return errorList def _buildWizardReadyProtocol( - self, - protocolId: Optional[int], - protocolClassName: str, - formValues: Dict[str, Any], + self, + protocolId: Optional[int], + protocolClassName: str, + formValues: Dict[str, Any], + mapper=None, + projectId: Optional[int] = None, ): if protocolId: - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Protocol not found", - ) + protocol = self.projectService._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) else: protClass = self.currentProject.getDomain().getProtocols().get(str(protocolClassName)) if protClass is None: @@ -418,7 +426,12 @@ def _buildWizardReadyProtocol( self.currentProject._fixProtParamsConfiguration(protocol) sanitizedFormValues = self._sanitizeWizardFormValues(formValues or {}) - errors = self._applyFormValuesToProtocolInstance(protocol, sanitizedFormValues) + errors = self._applyFormValuesToProtocolInstance( + protocol, + sanitizedFormValues, + mapper=mapper, + projectId=projectId, + ) if errors: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, @@ -495,6 +508,8 @@ def executeProtocolWizard( protocolId=getattr(payload, "protocolId", None), protocolClassName=str(getattr(payload, "protocolClassName", "")).strip(), formValues=getattr(payload, "formValues", {}) or {}, + mapper=mapper, + projectId=projectId, ) paramName = str(getattr(payload, "paramName", "")).strip() diff --git a/app/backend/database.py b/app/backend/database.py index 1e2b7f45..a207a324 100644 --- a/app/backend/database.py +++ b/app/backend/database.py @@ -66,8 +66,8 @@ def _requireEnv(key: str) -> str: Base = declarative_base() -def getMapper(): - # getPostgresqlMapper +def _createMapper(): + # createPostgresqlMapper from app.backend.mapper.postgresql import PostgresqlFlatMapper, PostgresqlDb dbName = _requireEnv("DATABASE_NAME") @@ -76,3 +76,20 @@ def getMapper(): db = PostgresqlDb(dbName=dbName, user=dbUser, password=dbPass) return PostgresqlFlatMapper(db) + + +def getMapper(): + # keepDirectMapperFactory + return _createMapper() + + +def getMapperDependency(): + # requestScopedMapperDependency + mapper = _createMapper() + try: + yield mapper + finally: + try: + mapper.db.close() + except Exception: + pass diff --git a/app/backend/main.py b/app/backend/main.py index 7ad0aa20..644087e4 100644 --- a/app/backend/main.py +++ b/app/backend/main.py @@ -39,7 +39,7 @@ from fastapi.middleware.cors import CORSMiddleware from app.backend.api.routers.project_router import router as projects -from app.backend.api.routers.protocol_router import router as protocols +# from app.backend.api.routers.protocol_router import router as protocols from app.backend.api.routers.plugin_router import router as plugins from app.backend.api.routers.auth_router import router as auth from app.backend.api.routers.user_router import router as users @@ -130,7 +130,7 @@ def _buildApiApp() -> FastAPI: # includeRouters apiApp.include_router(projects) - apiApp.include_router(protocols) + # apiApp.include_router(protocols) apiApp.include_router(plugins) apiApp.include_router(auth) apiApp.include_router(users) diff --git a/app/backend/mapper/__init__.py b/app/backend/mapper/__init__.py index 4cdb527e..9851601f 100644 --- a/app/backend/mapper/__init__.py +++ b/app/backend/mapper/__init__.py @@ -23,3 +23,5 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ****************************************************************************** +from app.backend.mapper.scipion_object_mapper import ScipionObjectPostgresqlMapper # noqa: F401 +from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper # noqa: F401 diff --git a/app/backend/mapper/postgresql.py b/app/backend/mapper/postgresql.py index f7236bf7..a6ced609 100644 --- a/app/backend/mapper/postgresql.py +++ b/app/backend/mapper/postgresql.py @@ -164,6 +164,44 @@ def initTables(self): """ ) + self.db.execute( + """ + CREATE TABLE IF NOT EXISTS protocol_input_refs ( + "projectId" INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + "protocolDbId" INTEGER NOT NULL, + "protocolId" TEXT NOT NULL, + "inputName" TEXT NOT NULL, + "itemIndex" INTEGER NOT NULL DEFAULT 0, + "parentProtocolDbId" INTEGER, + "parentProtocolId" TEXT, + "parentOutputName" TEXT, + "objectClassName" TEXT, + "objectId" TEXT, + "createdAt" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "updatedAt" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + PRIMARY KEY ("projectId", "protocolDbId", "inputName", "itemIndex"), + + FOREIGN KEY ("projectId", "protocolDbId") + REFERENCES protocols("projectId", id) + ON DELETE CASCADE, + + FOREIGN KEY ("projectId", "parentProtocolDbId") + REFERENCES protocols("projectId", id) + ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS idx_protocol_input_refs_protocol + ON protocol_input_refs("projectId", "protocolDbId"); + + CREATE INDEX IF NOT EXISTS idx_protocol_input_refs_parent + ON protocol_input_refs("projectId", "parentProtocolDbId", "parentOutputName"); + + CREATE INDEX IF NOT EXISTS idx_protocol_input_refs_parent_protocol_id + ON protocol_input_refs("projectId", "parentProtocolId", "parentOutputName"); + """ + ) + self.db.execute(""" CREATE TABLE IF NOT EXISTS protocol_steps ( id SERIAL PRIMARY KEY, @@ -356,35 +394,31 @@ def getProtocolTagIds(self, projectId: int, protocolDbId: int) -> List[str]: ) return [r["tagId"] for r in rows if r.get("tagId")] - def setProtocolTagIds(self, projectId: int, protocolId: int, tagIds: List[str]) -> dict: - # setProtocolTagIds + def setProtocolTagIdsByProtocolDbId(self, projectId: int, protocolDbId: int, tagIds: List[str]) -> dict: + # setProtocolTagIdsByProtocolDbId clean = sorted({str(t).strip() for t in (tagIds or []) if str(t).strip()}) - # validateProtocolBelongsToProject row = self.db.fetchOne( """ - SELECT "id" - FROM "protocols" - WHERE "protocolId" = %s + SELECT id, "protocolId" + FROM protocols + WHERE id = %s AND "projectId" = %s """, - (str(protocolId), projectId), + (int(protocolDbId), projectId), ) if not row: raise Exception("Protocol not found in project") - # deleteExistingAssignments - protocolDbId = row['id'] self.db.execute( """ DELETE FROM protocol_tag_assignments WHERE "protocolDbId" = %s """, - (protocolDbId,), + (int(protocolDbId),), ) if clean: - # bulkInsertWithUnnest self.db.execute( """ INSERT INTO protocol_tag_assignments ("protocolDbId", "tagId") @@ -392,10 +426,34 @@ def setProtocolTagIds(self, projectId: int, protocolId: int, tagIds: List[str]) FROM unnest(%s::text[]) AS x ON CONFLICT ("protocolDbId", "tagId") DO NOTHING """, - (protocolDbId, clean), + (int(protocolDbId), clean), ) - return {"protocolId": protocolDbId, "tagIds": clean} + return { + "protocolId": str(row["protocolId"]), + "protocolDbId": int(protocolDbId), + "tagIds": clean, + } + + def setProtocolTagIds(self, projectId: int, protocolId: int, tagIds: List[str]) -> dict: + # setProtocolTagIds + row = self.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE "protocolId" = %s + AND "projectId" = %s + """, + (str(protocolId), projectId), + ) + if not row: + raise Exception("Protocol not found in project") + + return self.setProtocolTagIdsByProtocolDbId( + projectId=projectId, + protocolDbId=int(row["id"]), + tagIds=tagIds, + ) def getProjectProtocolTagIdsByProtocolId(self, projectId: int, includeEmpty: bool = False) -> Dict[str, List[str]]: # getProjectProtocolTagIdsByProtocolId @@ -965,6 +1023,155 @@ def getProjectProtocolDbIdMap(self, projectId: int) -> Dict[str, int]: if row.get("protocolId") is not None and row.get("id") is not None } + def replaceProjectProtocolInputRefs( + self, + projectId: int, + refs: List[Dict[str, Any]], + ) -> int: + def toOptionalInt(value: Any) -> Optional[int]: + if value is None or value == "": + return None + + try: + return int(value) + except Exception: + try: + return int(float(value)) + except Exception: + return None + + self.db.execute( + """ + DELETE FROM protocol_input_refs + WHERE "projectId" = %s + """, + (projectId,), + ) + + cleanRefs: List[Dict[str, Any]] = [] + seen = set() + + for ref in refs or []: + protocolDbId = toOptionalInt(ref.get("protocolDbId")) + if protocolDbId is None or protocolDbId <= 0: + continue + + inputName = str(ref.get("inputName") or "").strip() + if not inputName: + continue + + itemIndex = toOptionalInt(ref.get("itemIndex")) + if itemIndex is None or itemIndex < 0: + itemIndex = 0 + + protocolId = str(ref.get("protocolId") or "").strip() + if not protocolId: + continue + + key = (protocolDbId, inputName, itemIndex) + if key in seen: + continue + + seen.add(key) + + parentProtocolDbId = toOptionalInt(ref.get("parentProtocolDbId")) + if parentProtocolDbId is not None and parentProtocolDbId <= 0: + parentProtocolDbId = None + + cleanRefs.append({ + "projectId": int(projectId), + "protocolDbId": protocolDbId, + "protocolId": protocolId, + "inputName": inputName, + "itemIndex": itemIndex, + "parentProtocolDbId": parentProtocolDbId, + "parentProtocolId": str(ref.get("parentProtocolId")).strip() + if ref.get("parentProtocolId") not in (None, "") else None, + "parentOutputName": str(ref.get("parentOutputName")).strip() + if ref.get("parentOutputName") not in (None, "") else None, + "objectClassName": str(ref.get("objectClassName")).strip() + if ref.get("objectClassName") not in (None, "") else None, + "objectId": str(ref.get("objectId")).strip() + if ref.get("objectId") not in (None, "") else None, + }) + + if not cleanRefs: + return 0 + + valuesSql = ",".join(["(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"] * len(cleanRefs)) + params: List[Any] = [] + + for ref in cleanRefs: + params.extend([ + ref["projectId"], + ref["protocolDbId"], + ref["protocolId"], + ref["inputName"], + ref["itemIndex"], + ref["parentProtocolDbId"], + ref["parentProtocolId"], + ref["parentOutputName"], + ref["objectClassName"], + ref["objectId"], + ]) + + self.db.execute( + f""" + INSERT INTO protocol_input_refs ( + "projectId", + "protocolDbId", + "protocolId", + "inputName", + "itemIndex", + "parentProtocolDbId", + "parentProtocolId", + "parentOutputName", + "objectClassName", + "objectId" + ) + VALUES {valuesSql} + ON CONFLICT ("projectId", "protocolDbId", "inputName", "itemIndex") + DO UPDATE SET + "protocolId" = EXCLUDED."protocolId", + "parentProtocolDbId" = EXCLUDED."parentProtocolDbId", + "parentProtocolId" = EXCLUDED."parentProtocolId", + "parentOutputName" = EXCLUDED."parentOutputName", + "objectClassName" = EXCLUDED."objectClassName", + "objectId" = EXCLUDED."objectId", + "updatedAt" = NOW() + """, + tuple(params), + ) + + return len(cleanRefs) + + def listProtocolInputRefs( + self, + projectId: int, + protocolDbId: Optional[int] = None, + ) -> List[Dict[str, Any]]: + if protocolDbId is None: + return self.db.fetchAll( + """ + SELECT * + FROM protocol_input_refs + WHERE "projectId" = %s + ORDER BY "protocolDbId", "inputName", "itemIndex" + """, + (projectId,), + ) + + return self.db.fetchAll( + """ + SELECT * + FROM protocol_input_refs + WHERE "projectId" = %s + AND "protocolDbId" = %s + ORDER BY "inputName", "itemIndex" + """, + (projectId, protocolDbId), + ) + def replaceProjectProtocolDependencies( self, projectId: int, @@ -1096,6 +1303,22 @@ def getProtocols(self, projectId: Optional[int] = None) -> List[Dict]: (projectId,), ) + def countProjectProtocols(self, projectId: int) -> int: + row = self.db.fetchOne( + """ + SELECT COUNT(*) AS count + FROM protocols + WHERE "projectId" = %s + """, + (projectId,), + ) + + if not row: + return 0 + + value = row.get("count") if isinstance(row, dict) else row[0] + return int(value or 0) + def updateProtocol(self, protocol: Dict[str, Any]) -> None: """Update protocol fields dynamically.""" updates = [] diff --git a/app/backend/mapper/scipion_object_mapper.py b/app/backend/mapper/scipion_object_mapper.py new file mode 100644 index 00000000..e392c1c3 --- /dev/null +++ b/app/backend/mapper/scipion_object_mapper.py @@ -0,0 +1,500 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +import json +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple + +import psycopg2.extras + + +class ScipionObjectPostgresqlMapper: + """Register and store Scipion data objects in PostgreSQL.""" + + def __init__(self, db): + self.db = db + + def registerObjectTypeFromObject( + self, + scipionObj: Any, + mapperKind: Optional[str] = None, + includeProperties: bool = True, + includeNestedProperties: bool = True, + classSchema: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + typeId = self.registerObjectType(scipionObj, mapperKind=mapperKind, classSchema=classSchema) + propertiesCount = 0 + if includeProperties: + propertiesCount = self.registerObjectTypeProperties( + typeId, + scipionObj, + includeNestedProperties=includeNestedProperties, + ) + return { + "typeId": typeId, + "className": self._getClassName(scipionObj), + "propertiesCount": propertiesCount, + } + + def registerObjectType( + self, + scipionObj: Any, + mapperKind: Optional[str] = None, + classSchema: Optional[Dict[str, Any]] = None, + ) -> int: + className = self._getClassName(scipionObj) + if not className: + raise ValueError("Cannot register a Scipion object type without className") + + cur = self.db.execute( + """ + INSERT INTO scipion_object_types ( + "className", "moduleName", "baseClassName", "mapperKind", "schema" + ) + VALUES (%s, %s, %s, %s, %s::jsonb) + ON CONFLICT ("className") + DO UPDATE SET + "moduleName" = EXCLUDED."moduleName", + "baseClassName" = EXCLUDED."baseClassName", + "mapperKind" = EXCLUDED."mapperKind", + "schema" = scipion_object_types."schema" || EXCLUDED."schema", + "updatedAt" = NOW() + RETURNING id + """, + ( + className, + self._getModuleName(scipionObj), + self._getBaseClassName(scipionObj), + mapperKind or self._guessMapperKind(scipionObj), + self._jsonParam(classSchema or {}), + ), + ) + return int(cur.fetchone()["id"]) + + def registerObjectTypeProperties( + self, + typeId: int, + scipionObj: Any, + includeNestedProperties: bool = True, + ) -> int: + properties = list(self._iterProperties(scipionObj, includeNestedProperties=includeNestedProperties)) + if not properties: + return 0 + + with self.db.transaction(): + for prop in properties: + self.db.execute( + """ + INSERT INTO scipion_object_type_properties ( + "typeId", "propertyPath", "className", "valueKind", + "isPointer", "isNested", "schema" + ) + VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb) + ON CONFLICT ("typeId", "propertyPath") + DO UPDATE SET + "className" = EXCLUDED."className", + "valueKind" = EXCLUDED."valueKind", + "isPointer" = EXCLUDED."isPointer", + "isNested" = EXCLUDED."isNested", + "schema" = scipion_object_type_properties."schema" || EXCLUDED."schema", + "updatedAt" = NOW() + """, + ( + typeId, + prop["propertyPath"], + prop["className"], + prop["valueKind"], + prop["isPointer"], + prop["isNested"], + self._jsonParam(prop.get("schema") or {}), + ), + commit=False, + ) + return len(properties) + + def storeObjectTree( + self, + projectId: int, + protocolDbId: int, + outputName: str, + scipionObj: Any, + registerType: bool = True, + includeNestedProperties: bool = True, + ) -> Dict[str, Any]: + if not projectId: + raise ValueError("projectId is required") + if not protocolDbId: + raise ValueError("protocolDbId is required") + if not outputName: + raise ValueError("outputName is required") + + if registerType: + self.registerObjectTypeFromObject(scipionObj, includeNestedProperties=includeNestedProperties) + + storedPaths: List[str] = [] + with self.db.transaction(): + rootObjectId = self._storeObjectNode( + projectId=projectId, + protocolDbId=protocolDbId, + scipionObj=scipionObj, + name=outputName, + path=outputName, + parentObjectId=None, + storedPaths=storedPaths, + includeNestedProperties=includeNestedProperties, + visited=set(), + ) + + return { + "rootObjectId": rootObjectId, + "projectId": projectId, + "protocolDbId": protocolDbId, + "outputName": outputName, + "storedObjectsCount": len(storedPaths), + "storedPaths": storedPaths, + } + + def getStoredObjectTree(self, projectId: int, protocolDbId: int, outputName: str) -> List[Dict[str, Any]]: + rootPath = str(outputName) + return self.db.fetchAll( + """ + SELECT id, "projectId", "protocolDbId", "scipionObjId", "parentObjectId", + name, path, "className", value, label, comment, creation, + metadata, "createdAt", "updatedAt" + FROM scipion_objects + WHERE "projectId" = %s + AND "protocolDbId" = %s + AND (path = %s OR path LIKE %s) + ORDER BY path ASC + """, + (projectId, protocolDbId, rootPath, f"{rootPath}.%"), + ) + + def listProtocolStoredObjects(self, projectId: int, protocolDbId: int) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT id, "projectId", "protocolDbId", "scipionObjId", "parentObjectId", + name, path, "className", value, label, comment, creation, + metadata, "createdAt", "updatedAt" + FROM scipion_objects + WHERE "projectId" = %s + AND "protocolDbId" = %s + ORDER BY path ASC + """, + (projectId, protocolDbId), + ) + + def getObjectType(self, className: str) -> Optional[Dict[str, Any]]: + return self.db.fetchOne( + """ + SELECT id, "className", "moduleName", "baseClassName", "mapperKind", "schema", "createdAt", "updatedAt" + FROM scipion_object_types + WHERE "className" = %s + """, + (className,), + ) + + def listObjectTypeProperties(self, className: str) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT p.id, p."typeId", p."propertyPath", p."className", p."valueKind", + p."isPointer", p."isNested", p."schema", p."createdAt", p."updatedAt" + FROM scipion_object_type_properties p + JOIN scipion_object_types t + ON t.id = p."typeId" + WHERE t."className" = %s + ORDER BY p."propertyPath" ASC + """, + (className,), + ) + + def _storeObjectNode( + self, + projectId: int, + protocolDbId: int, + scipionObj: Any, + name: str, + path: str, + parentObjectId: Optional[int], + storedPaths: List[str], + includeNestedProperties: bool, + visited: Set[int], + ) -> int: + objIdentity = id(scipionObj) + if objIdentity in visited: + return parentObjectId or 0 + visited.add(objIdentity) + + attributes = self._getAttributesToStore(scipionObj) + metadata = { + "moduleName": self._getModuleName(scipionObj), + "baseClassName": self._getBaseClassName(scipionObj), + "isPointer": self._isPointer(scipionObj), + "isNested": bool(attributes), + "hasSourceObjId": self._getSourceObjId(scipionObj) is not None, + } + + cur = self.db.execute( + """ + INSERT INTO scipion_objects ( + "projectId", "protocolDbId", "scipionObjId", "parentObjectId", + name, path, "className", value, label, comment, creation, metadata + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb) + ON CONFLICT ON CONSTRAINT ux_scipion_objects_project_protocol_path + DO UPDATE SET + "scipionObjId" = EXCLUDED."scipionObjId", + "parentObjectId" = EXCLUDED."parentObjectId", + name = EXCLUDED.name, + "className" = EXCLUDED."className", + value = EXCLUDED.value, + label = EXCLUDED.label, + comment = EXCLUDED.comment, + creation = EXCLUDED.creation, + metadata = scipion_objects.metadata || EXCLUDED.metadata, + "updatedAt" = NOW() + RETURNING id + """, + ( + projectId, + protocolDbId, + self._getScipionObjId(scipionObj, path), + parentObjectId, + name, + path, + self._getClassName(scipionObj), + self._getObjectValueText(scipionObj), + self._getObjectLabel(scipionObj), + self._getObjectComment(scipionObj), + self._getObjectCreation(scipionObj), + self._jsonParam(metadata), + ), + commit=False, + ) + objectId = int(cur.fetchone()["id"]) + storedPaths.append(path) + + if includeNestedProperties: + for attrName, attrValue in attributes: + self._storeObjectNode( + projectId=projectId, + protocolDbId=protocolDbId, + scipionObj=attrValue, + name=attrName, + path=f"{path}.{attrName}", + parentObjectId=objectId, + storedPaths=storedPaths, + includeNestedProperties=includeNestedProperties, + visited=visited, + ) + + return objectId + + def _iterProperties( + self, + scipionObj: Any, + prefix: str = "", + includeNestedProperties: bool = True, + visited: Optional[Set[int]] = None, + ) -> Iterable[Dict[str, Any]]: + visited = visited or set() + objIdentity = id(scipionObj) + if objIdentity in visited: + return + visited.add(objIdentity) + + for attrName, attrValue in self._getAttributesToStore(scipionObj): + propertyPath = f"{prefix}.{attrName}" if prefix else str(attrName) + childAttributes = self._getAttributesToStore(attrValue) + isPointer = self._isPointer(attrValue) + isNested = bool(childAttributes) + yield { + "propertyPath": propertyPath, + "className": self._getClassName(attrValue), + "valueKind": self._getValueKind(attrValue, isPointer=isPointer, isNested=isNested), + "isPointer": isPointer, + "isNested": isNested, + "schema": { + "moduleName": self._getModuleName(attrValue), + "baseClassName": self._getBaseClassName(attrValue), + }, + } + if includeNestedProperties and isNested: + yield from self._iterProperties( + attrValue, + prefix=propertyPath, + includeNestedProperties=includeNestedProperties, + visited=visited, + ) + + def _getAttributesToStore(self, scipionObj: Any) -> List[Tuple[str, Any]]: + getter = getattr(scipionObj, "getAttributesToStore", None) + if not callable(getter): + return [] + try: + return [(str(name), value) for name, value in getter()] + except Exception: + return [] + + def _getClassName(self, scipionObj: Any) -> Optional[str]: + getter = getattr(scipionObj, "getClassName", None) + if callable(getter): + try: + className = getter() + if className: + return str(className) + except Exception: + pass + if scipionObj is None: + return None + return scipionObj.__class__.__name__ + + def _getModuleName(self, scipionObj: Any) -> Optional[str]: + if scipionObj is None: + return None + moduleName = getattr(scipionObj.__class__, "__module__", None) + return str(moduleName) if moduleName else None + + def _getBaseClassName(self, scipionObj: Any) -> Optional[str]: + if scipionObj is None: + return None + bases = getattr(scipionObj.__class__, "__bases__", None) or [] + return bases[0].__name__ if bases else None + + def _guessMapperKind(self, scipionObj: Any) -> str: + className = self._getClassName(scipionObj) or "" + if self._isPointer(scipionObj): + return "pointer" + if className.startswith("SetOf") or "SetOf" in className: + return "flat_set" + if self._getAttributesToStore(scipionObj): + return "tree" + return "scalar" + + def _getValueKind(self, scipionObj: Any, isPointer: bool, isNested: bool) -> str: + if isPointer: + return "pointer" + if isNested: + return "object" + return self._getClassName(scipionObj) or "scalar" + + def _getSourceObjId(self, scipionObj: Any) -> Optional[int]: + for getterName in ("getObjId", "getId"): + getter = getattr(scipionObj, getterName, None) + if not callable(getter): + continue + try: + value = getter() + except Exception: + continue + if value is None: + continue + try: + return int(value) + except Exception: + continue + return None + + def _getScipionObjId(self, scipionObj: Any, path: str) -> Optional[int]: + return self._getSourceObjId(scipionObj) + + def _getObjectValueText(self, scipionObj: Any) -> Optional[str]: + if self._isPointer(scipionObj): + pointedObj = self._getPointerValue(scipionObj) + pointedId = self._getSourceObjId(pointedObj) + if pointedId is not None: + return str(pointedId) + if pointedObj is not None: + return str(pointedObj) + + value = None + for methodName in ("getObjValue", "get"): + getter = getattr(scipionObj, methodName, None) + if not callable(getter): + continue + try: + value = getter() + break + except Exception: + continue + + if value is None: + return None + if isinstance(value, (dict, list, tuple)): + return json.dumps(value, ensure_ascii=False) + return str(value) + + def _getPointerValue(self, scipionObj: Any) -> Any: + hasValue = getattr(scipionObj, "hasValue", None) + if callable(hasValue): + try: + if not hasValue(): + return None + except Exception: + return None + getter = getattr(scipionObj, "get", None) + if not callable(getter): + return None + try: + return getter() + except Exception: + return None + + def _getObjectLabel(self, scipionObj: Any) -> Optional[str]: + return self._getOptionalObjectText(scipionObj, "getObjLabel", "_objLabel") + + def _getObjectComment(self, scipionObj: Any) -> Optional[str]: + return self._getOptionalObjectText(scipionObj, "getObjComment", "_objComment") + + def _getObjectCreation(self, scipionObj: Any) -> Any: + getter = getattr(scipionObj, "getObjCreation", None) + if callable(getter): + try: + return getter() + except Exception: + pass + return getattr(scipionObj, "_objCreation", None) + + def _getOptionalObjectText(self, scipionObj: Any, getterName: str, attributeName: str) -> Optional[str]: + getter = getattr(scipionObj, getterName, None) + if callable(getter): + try: + value = getter() + return str(value) if value else None + except Exception: + pass + value = getattr(scipionObj, attributeName, None) + return str(value) if value else None + + def _isPointer(self, scipionObj: Any) -> bool: + checker = getattr(scipionObj, "isPointer", None) + if not callable(checker): + return False + try: + return bool(checker()) + except Exception: + return False + + def _jsonParam(self, value: Dict[str, Any]) -> Any: + return psycopg2.extras.Json(value or {}, dumps=json.dumps) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py new file mode 100644 index 00000000..719c4c88 --- /dev/null +++ b/app/backend/mapper/scipion_set_mapper.py @@ -0,0 +1,1614 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +import json +import os +from datetime import datetime, timezone +from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple +import logging + +import psycopg2.extras + +from app.backend.mapper.scipion_object_mapper import ScipionObjectPostgresqlMapper +logger = logging.getLogger(__name__) + +try: + from tomo.constants import BOTTOM_LEFT_CORNER +except Exception: + BOTTOM_LEFT_CORNER = None + + +SELF_LABEL = "self" +NESTED_LOGICAL_TABLES_VERSION = 14 + + +class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): + """Store Scipion SetOf... objects in PostgreSQL using a flat JSONB layout.""" + + def storeSet( + self, + projectId: int, + protocolDbId: int, + outputName: str, + scipionSet: Any, + registerType: bool = True, + batchSize: int = 1000, + ) -> Dict[str, Any]: + if not projectId: + raise ValueError("projectId is required") + if not protocolDbId: + raise ValueError("protocolDbId is required") + if not outputName: + raise ValueError("outputName is required") + if batchSize <= 0: + raise ValueError("batchSize must be greater than zero") + + protocolDbId = self._resolveProtocolDbId(projectId, protocolDbId) + syncTimestamp = datetime.now(timezone.utc).isoformat() + itemsCountHint = self._getSetItemsCountHint(scipionSet) + maxItemIdHint = self._getSetMaxItemIdHint(scipionSet) + sourceMTime = self._getSetSourceMTime(scipionSet) + existingSet = self._getExistingSet(projectId, protocolDbId, outputName) + + if existingSet is not None: + existingSetId = int(existingSet["id"]) + existingProperties = self._normalizeProperties(existingSet.get("properties")) + if ( + self.hasStoredSetTables(existingSetId) + and self._shouldSkipSetSync(existingProperties, itemsCountHint, maxItemIdHint, sourceMTime) + ): + skippedProperties = dict(existingProperties) + skippedProperties["lastCheckedAt"] = syncTimestamp + skippedProperties["lastSkipReason"] = "unchanged_signature" + skippedProperties["skippedLastSync"] = True + skippedProperties["incremental"] = True + skippedProperties["nestedTablesVersion"] = NESTED_LOGICAL_TABLES_VERSION + if sourceMTime is not None: + skippedProperties["sourceMTime"] = sourceMTime + + with self.db.transaction(): + self._updateSetProperties(int(existingSet["id"]), skippedProperties) + self._upsertSetProperties(int(existingSet["id"]), skippedProperties) + + return { + "setId": int(existingSet["id"]), + "rootObjectId": existingSet.get("objectId"), + "projectId": projectId, + "protocolDbId": protocolDbId, + "outputName": outputName, + "setClassName": existingSet.get("setClassName"), + "itemClassName": existingSet.get("itemClassName"), + "columnsCount": self._toOptionalInt(existingProperties.get("columnsCount")), + "itemsCount": self._toOptionalInt(existingProperties.get("itemsCount")), + "maxItemId": self._toOptionalInt(existingProperties.get("maxItemId")), + "lastSyncAt": existingProperties.get("lastSyncAt"), + "lastCheckedAt": syncTimestamp, + "skipped": True, + } + + if registerType: + self.registerObjectTypeFromObject( + scipionSet, + mapperKind="flat_set", + includeProperties=False, + classSchema={"storage": "flat_set"}, + ) + + itemIterator = iter(self._iterSetItems(scipionSet)) + firstItem = self._nextOrNone(itemIterator) + itemSchema = self._getItemSchema(firstItem) if firstItem is not None else {} + itemClassName = self._getItemClassName(firstItem, itemSchema) + columns = self._getSetColumns(itemSchema) + initialProperties = self._getSetProperties(scipionSet) + initialProperties["nestedTablesVersion"] = NESTED_LOGICAL_TABLES_VERSION + + storedPaths: List[str] = [] + with self.db.transaction(): + rootObjectId = self._storeObjectNode( + projectId=projectId, + protocolDbId=protocolDbId, + scipionObj=scipionSet, + name=outputName, + path=outputName, + parentObjectId=None, + storedPaths=storedPaths, + includeNestedProperties=False, + visited=set(), + ) + + setId = self._upsertSet( + projectId=projectId, + protocolDbId=protocolDbId, + objectId=rootObjectId, + outputName=outputName, + setClassName=self._getClassName(scipionSet) or scipionSet.__class__.__name__, + itemClassName=itemClassName, + properties=initialProperties, + ) + self._upsertSetColumns(setId, columns) + + rootTableId = self._upsertSetTable( + setId=setId, + name="objects", + alias=self._getClassName(scipionSet) or outputName, + tableKind="root", + parentTableId=None, + parentItemId=None, + itemClassName=itemClassName, + properties={ + "source": "postgresql", + "legacySetTable": True, + }, + ) + self._upsertSetTableColumns(rootTableId, columns) + + itemsCount = 0 + maxItemId = None + if firstItem is not None: + itemsCount, maxItemId = self._upsertSetItems( + setId=setId, + tableId=rootTableId, + firstItem=firstItem, + remainingItems=itemIterator, + batchSize=batchSize, + scipionSet=scipionSet, + ) + + finalProperties = dict(initialProperties) + finalProperties["columnsCount"] = len(columns) + finalProperties["itemsCount"] = itemsCount + finalProperties["maxItemId"] = maxItemId + finalProperties["lastSyncAt"] = syncTimestamp + finalProperties["lastCheckedAt"] = syncTimestamp + finalProperties["lastSkipReason"] = None + finalProperties["skippedLastSync"] = False + finalProperties["incremental"] = True + finalProperties["nestedTablesVersion"] = NESTED_LOGICAL_TABLES_VERSION + if sourceMTime is not None: + finalProperties["sourceMTime"] = sourceMTime + self._updateSetProperties(setId, finalProperties) + self._upsertSetProperties(setId, finalProperties) + + return { + "setId": setId, + "rootObjectId": rootObjectId, + "projectId": projectId, + "protocolDbId": protocolDbId, + "outputName": outputName, + "setClassName": self._getClassName(scipionSet), + "itemClassName": itemClassName, + "columnsCount": len(columns), + "itemsCount": itemsCount, + "maxItemId": maxItemId, + "lastSyncAt": syncTimestamp, + "lastCheckedAt": syncTimestamp, + "skipped": False, + } + + def getStoredSet( + self, + projectId: int, + protocolDbId: int, + outputName: str, + limit: Optional[int] = None, + offset: int = 0, + ) -> Optional[Dict[str, Any]]: + protocolDbId = self._resolveProtocolDbId(projectId, protocolDbId) + + storedSet = self.db.fetchOne( + """ + SELECT id, "projectId", "protocolDbId", "objectId", "outputName", + "setClassName", "itemClassName", properties, "createdAt", "updatedAt" + FROM scipion_sets + WHERE "projectId" = %s + AND "protocolDbId" = %s + AND "outputName" = %s + """, + (projectId, protocolDbId, outputName), + ) + if storedSet is None: + return None + + storedSet["columns"] = self.getStoredSetColumns(storedSet["id"]) + storedSet["setProperties"] = self.getStoredSetProperties(storedSet["id"]) + storedSet["items"] = self.getStoredSetItems(storedSet["id"], limit=limit, offset=offset) + return storedSet + + def listProtocolStoredSets(self, projectId: int, protocolDbId: int) -> List[Dict[str, Any]]: + protocolDbId = self._resolveProtocolDbId(projectId, protocolDbId) + + return self.db.fetchAll( + """ + SELECT id, "projectId", "protocolDbId", "objectId", "outputName", + "setClassName", "itemClassName", properties, "createdAt", "updatedAt" + FROM scipion_sets + WHERE "projectId" = %s + AND "protocolDbId" = %s + ORDER BY "outputName" ASC + """, + (projectId, protocolDbId), + ) + + def getStoredSetColumns(self, setId: int) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT id, "setId", "labelProperty", "columnName", "className", + "valueType", position, indexed + FROM scipion_set_columns + WHERE "setId" = %s + ORDER BY position ASC + """, + (setId,), + ) + + def getStoredSetProperties(self, setId: int) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT id, "setId", key, value + FROM scipion_set_properties + WHERE "setId" = %s + ORDER BY key ASC + """, + (setId,), + ) + + def getStoredSetItems(self, setId: int, limit: Optional[int] = None, offset: int = 0) -> List[Dict[str, Any]]: + if limit is None: + return self.db.fetchAll( + """ + SELECT id, "setId", "scipionItemId", enabled, label, comment, + creation, "values", "createdAt", "updatedAt" + FROM scipion_set_items + WHERE "setId" = %s + ORDER BY "scipionItemId" ASC + """, + (setId,), + ) + + return self.db.fetchAll( + """ + SELECT id, "setId", "scipionItemId", enabled, label, comment, + creation, "values", "createdAt", "updatedAt" + FROM scipion_set_items + WHERE "setId" = %s + ORDER BY "scipionItemId" ASC + LIMIT %s OFFSET %s + """, + (setId, limit, offset), + ) + + def _resolveProtocolDbId(self, projectId: int, protocolDbId: int) -> int: + byDatabaseId = self.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE id = %s + AND "projectId" = %s + """, + (protocolDbId, projectId), + ) + if byDatabaseId is not None: + return int(byDatabaseId["id"]) + + byScipionId = self.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE "projectId" = %s + AND "protocolId" = %s + """, + (projectId, str(protocolDbId)), + ) + if byScipionId is not None: + return int(byScipionId["id"]) + + raise ValueError( + "Protocol %s was not found in PostgreSQL protocols table for project %s" + % (protocolDbId, projectId) + ) + + def _getExistingSet(self, projectId: int, protocolDbId: int, outputName: str) -> Optional[Dict[str, Any]]: + return self.db.fetchOne( + """ + SELECT id, "objectId", "setClassName", "itemClassName", properties + FROM scipion_sets + WHERE "projectId" = %s + AND "protocolDbId" = %s + AND "outputName" = %s + """, + (projectId, protocolDbId, outputName), + ) + + def _upsertSet( + self, + projectId: int, + protocolDbId: int, + objectId: int, + outputName: str, + setClassName: str, + itemClassName: str, + properties: Dict[str, Any], + ) -> int: + cur = self.db.execute( + """ + INSERT INTO scipion_sets ( + "projectId", "protocolDbId", "objectId", "outputName", + "setClassName", "itemClassName", properties + ) + VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb) + ON CONFLICT ON CONSTRAINT ux_scipion_sets_project_protocol_output + DO UPDATE SET + "objectId" = EXCLUDED."objectId", + "setClassName" = EXCLUDED."setClassName", + "itemClassName" = EXCLUDED."itemClassName", + properties = EXCLUDED.properties, + "updatedAt" = NOW() + RETURNING id + """, + ( + projectId, + protocolDbId, + objectId, + outputName, + setClassName, + itemClassName, + self._jsonParam(properties), + ), + commit=False, + ) + return int(cur.fetchone()["id"]) + + def _upsertSetColumns(self, setId: int, columns: List[Dict[str, Any]]) -> None: + for column in columns: + self.db.execute( + """ + INSERT INTO scipion_set_columns ( + "setId", "labelProperty", "columnName", "className", "valueType", position, indexed + ) + VALUES (%s, %s, %s, %s, %s, %s, %s) + ON CONFLICT ON CONSTRAINT ux_scipion_set_columns_set_label + DO UPDATE SET + "columnName" = EXCLUDED."columnName", + "className" = EXCLUDED."className", + "valueType" = EXCLUDED."valueType", + position = EXCLUDED.position, + indexed = EXCLUDED.indexed + """, + ( + setId, + column["labelProperty"], + column["columnName"], + column["className"], + column["valueType"], + column["position"], + column["indexed"], + ), + commit=False, + ) + + def _upsertSetProperties(self, setId: int, properties: Dict[str, Any]) -> None: + for key, value in sorted(properties.items()): + self.db.execute( + """ + INSERT INTO scipion_set_properties ("setId", key, value) + VALUES (%s, %s, %s) + ON CONFLICT ON CONSTRAINT ux_scipion_set_properties_set_key + DO UPDATE SET + value = EXCLUDED.value + """, + (setId, str(key), self._stringifyPropertyValue(value)), + commit=False, + ) + + def _upsertSetItems( + self, + setId: int, + tableId: Optional[int], + firstItem: Any, + remainingItems: Iterator[Any], + batchSize: int, + scipionSet: Optional[Any] = None, + ) -> Tuple[int, Optional[int]]: + rows: List[Tuple[Any, ...]] = [] + tableRows: List[Tuple[Any, ...]] = [] + itemsCount = 0 + maxItemId: Optional[int] = None + + for item in self._chainFirst(firstItem, remainingItems): + itemId = self._getSourceObjId(item) + if itemId is None: + raise ValueError("Cannot store a Scipion set item without getObjId()/getId()") + + maxItemId = itemId if maxItemId is None else max(maxItemId, itemId) + itemValues = self._getItemValues(item, scipionSet=scipionSet) + + rows.append( + ( + setId, + itemId, + self._getItemEnabled(item), + self._getObjectLabel(item), + self._getObjectComment(item), + self._getObjectCreation(item), + self._jsonParam(itemValues), + ) + ) + + if tableId is not None: + tableRows.append( + ( + tableId, + itemId, + None, + self._getItemEnabled(item), + self._getObjectLabel(item), + self._getObjectComment(item), + self._getObjectCreation(item), + self._jsonParam(itemValues), + ) + ) + + self._upsertNestedLogicalTablesForItem( + setId=setId, + parentTableId=tableId, + parentItem=item, + parentItemId=itemId, + batchSize=batchSize, + ) + + itemsCount += 1 + + if len(rows) >= batchSize: + self._flushSetItems(rows) + rows = [] + + if tableRows: + self._flushSetTableItems(tableRows) + tableRows = [] + + if rows: + self._flushSetItems(rows) + + if tableRows: + self._flushSetTableItems(tableRows) + + return itemsCount, maxItemId + + + def _upsertNestedLogicalTablesForItem( + self, + setId: int, + parentTableId: int, + parentItem: Any, + parentItemId: int, + batchSize: int, + ) -> None: + """ + Persist child logical tables for complex Scipion set items. + + Supported examples: + - Class2D/Class3D items expose class members. + - TiltSeries items expose tilt images. + - Any item exposing iterItems() can expose a child logical table. + """ + if not self._hasNestedLogicalItems(parentItem): + return + + childIterator = iter(self._iterNestedItems(parentItem)) + firstChild = self._nextOrNone(childIterator) + if firstChild is None: + return + + childSchema = self._getItemSchema(firstChild) + childColumns = self._getSetColumns(childSchema) + childItemClassName = self._getItemClassName(firstChild, childSchema) + + tableName = self._getNestedLogicalTableName(parentItem, parentItemId) + tableAlias = self._getNestedLogicalTableAlias(tableName, childItemClassName) + + childTableId = self._upsertSetTable( + setId=setId, + name=tableName, + alias=tableAlias, + tableKind="child", + parentTableId=parentTableId, + parentItemId=parentItemId, + itemClassName=childItemClassName, + properties={ + "source": "postgresql", + "parentItemId": parentItemId, + "parentClassName": self._getClassName(parentItem), + }, + ) + + self._upsertSetTableColumns(childTableId, childColumns) + + self._upsertLogicalTableItems( + tableId=childTableId, + parentItemId=parentItemId, + firstItem=firstChild, + remainingItems=childIterator, + batchSize=batchSize, + ) + + def _hasNestedLogicalItems(self, item: Any) -> bool: + if item is None: + return False + + iterItems = getattr(item, "iterItems", None) + return callable(iterItems) + + def _iterNestedItems(self, item: Any) -> Iterable[Any]: + iterItems = getattr(item, "iterItems", None) + if callable(iterItems): + try: + return iterItems(iterate=False) + except TypeError: + return iterItems() + + return iter(()) + + def _getNestedLogicalTableName(self, parentItem: Any, parentItemId: int) -> str: + className = self._getClassName(parentItem) or parentItem.__class__.__name__ + + if str(className).startswith("Class"): + return self._getNestedClassTableName(parentItemId) + + stableName = self._getNestedItemStableName( + parentItem=parentItem, + parentItemId=parentItemId, + className=className, + ) + return "%s_Objects" % self._sanitizeLogicalTableNamePart(stableName) + + def _getNestedItemStableName( + self, + parentItem: Any, + parentItemId: int, + className: str, + ) -> str: + for getterName in ("getTsId", "getTomoId", "getObjLabel", "getName"): + value = self._callOptionalGetter(parentItem, getterName) + valueText = str(value or "").strip() + if valueText: + return valueText + + try: + return "%s%03d" % (className or "Item", int(parentItemId)) + except Exception: + return "%s%s" % (className or "Item", str(parentItemId)) + + def _sanitizeLogicalTableNamePart(self, value: Any) -> str: + text = str(value or "").strip() + chars = [] + previousWasUnderscore = False + + for char in text: + if char.isalnum(): + chars.append(char) + previousWasUnderscore = False + continue + + if char == "_" and not previousWasUnderscore: + chars.append("_") + previousWasUnderscore = True + continue + + if not previousWasUnderscore: + chars.append("_") + previousWasUnderscore = True + + cleanText = "".join(chars).strip("_") + return cleanText or "Item" + + def _getNestedClassTableName(self, parentItemId: int) -> str: + try: + return "Class%03d_Objects" % int(parentItemId) + except Exception: + return "Class%s_Objects" % str(parentItemId) + + def _getNestedLogicalTableAlias(self, tableName: str, childItemClassName: str) -> str: + cleanClassName = str(childItemClassName or "Objects").strip() or "Objects" + if tableName.endswith("_Objects"): + return "%s_%s" % (tableName[: -len("_Objects")], cleanClassName) + return "%s_%s" % (tableName, cleanClassName) + + def _upsertLogicalTableItems( + self, + tableId: int, + parentItemId: Optional[int], + firstItem: Any, + remainingItems: Iterator[Any], + batchSize: int, + ) -> int: + rows: List[Tuple[Any, ...]] = [] + itemsCount = 0 + + for item in self._chainFirst(firstItem, remainingItems): + itemId = self._getSourceObjId(item) + if itemId is None: + continue + + rows.append( + ( + tableId, + itemId, + parentItemId, + self._getItemEnabled(item), + self._getObjectLabel(item), + self._getObjectComment(item), + self._getObjectCreation(item), + self._jsonParam(self._getItemValues(item)), + ) + ) + itemsCount += 1 + + if len(rows) >= batchSize: + self._flushSetTableItems(rows) + rows = [] + + if rows: + self._flushSetTableItems(rows) + + return itemsCount + + def hasStoredSetTables(self, setId: int) -> bool: + row = self.db.fetchOne( + """ + SELECT id + FROM scipion_set_tables + WHERE "setId" = %s + LIMIT 1 + """, + (setId,), + ) + return row is not None + + def listStoredSetTables(self, setId: int) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT id, "setId", name, alias, "tableKind", "parentTableId", + "parentItemId", "itemClassName", properties, "createdAt", "updatedAt" + FROM scipion_set_tables + WHERE "setId" = %s + ORDER BY + CASE WHEN "tableKind" = 'root' THEN 0 ELSE 1 END, + name ASC + """, + (setId,), + ) + + def getStoredSetTableColumns(self, tableId: int) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT id, "tableId", "labelProperty", "columnName", "className", + "valueType", position, indexed, properties + FROM scipion_set_table_columns + WHERE "tableId" = %s + ORDER BY position ASC + """, + (tableId,), + ) + + def getStoredSetTableItems( + self, + tableId: int, + limit: Optional[int] = None, + offset: int = 0, + ) -> List[Dict[str, Any]]: + if limit is None: + return self.db.fetchAll( + """ + SELECT id, "tableId", "scipionItemId", "parentItemId", enabled, + label, comment, creation, "values", "createdAt", "updatedAt" + FROM scipion_set_table_items + WHERE "tableId" = %s + ORDER BY "scipionItemId" ASC + """, + (tableId,), + ) + + return self.db.fetchAll( + """ + SELECT id, "tableId", "scipionItemId", "parentItemId", enabled, + label, comment, creation, "values", "createdAt", "updatedAt" + FROM scipion_set_table_items + WHERE "tableId" = %s + ORDER BY "scipionItemId" ASC + LIMIT %s OFFSET %s + """, + (tableId, limit, offset), + ) + + def _upsertSetTable( + self, + setId: int, + name: str, + alias: Optional[str], + tableKind: str, + parentTableId: Optional[int], + parentItemId: Optional[int], + itemClassName: Optional[str], + properties: Optional[Dict[str, Any]] = None, + ) -> int: + cur = self.db.execute( + """ + INSERT INTO scipion_set_tables ( + "setId", name, alias, "tableKind", "parentTableId", + "parentItemId", "itemClassName", properties + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb) + ON CONFLICT ON CONSTRAINT ux_scipion_set_tables_set_name + DO UPDATE SET + alias = EXCLUDED.alias, + "tableKind" = EXCLUDED."tableKind", + "parentTableId" = EXCLUDED."parentTableId", + "parentItemId" = EXCLUDED."parentItemId", + "itemClassName" = EXCLUDED."itemClassName", + properties = EXCLUDED.properties, + "updatedAt" = NOW() + RETURNING id + """, + ( + setId, + name, + alias, + tableKind, + parentTableId, + parentItemId, + itemClassName, + self._jsonParam(properties or {}), + ), + commit=False, + ) + return int(cur.fetchone()["id"]) + + def _upsertSetTableColumns(self, tableId: int, columns: List[Dict[str, Any]]) -> None: + for column in columns: + self.db.execute( + """ + INSERT INTO scipion_set_table_columns ( + "tableId", "labelProperty", "columnName", "className", + "valueType", position, indexed, properties + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb) + ON CONFLICT ON CONSTRAINT ux_scipion_set_table_columns_table_label + DO UPDATE SET + "columnName" = EXCLUDED."columnName", + "className" = EXCLUDED."className", + "valueType" = EXCLUDED."valueType", + position = EXCLUDED.position, + indexed = EXCLUDED.indexed, + properties = EXCLUDED.properties + """, + ( + tableId, + column["labelProperty"], + column["columnName"], + column["className"], + column["valueType"], + column["position"], + column["indexed"], + self._jsonParam(column.get("properties") or {}), + ), + commit=False, + ) + + def _flushSetTableItems(self, rows: List[Tuple[Any, ...]]) -> None: + psycopg2.extras.execute_values( + self.db.cursor, + """ + INSERT INTO scipion_set_table_items ( + "tableId", "scipionItemId", "parentItemId", enabled, + label, comment, creation, "values" + ) + VALUES %s + ON CONFLICT ON CONSTRAINT ux_scipion_set_table_items_table_item + DO UPDATE SET + "parentItemId" = EXCLUDED."parentItemId", + enabled = EXCLUDED.enabled, + label = EXCLUDED.label, + comment = EXCLUDED.comment, + creation = EXCLUDED.creation, + "values" = EXCLUDED."values", + "updatedAt" = NOW() + """, + rows, + template="(%s, %s, %s, %s, %s, %s, %s, %s::jsonb)", + page_size=len(rows), + ) + + def _flushSetItems(self, rows: List[Tuple[Any, ...]]) -> None: + psycopg2.extras.execute_values( + self.db.cursor, + """ + INSERT INTO scipion_set_items ( + "setId", "scipionItemId", enabled, label, comment, creation, "values" + ) + VALUES %s + ON CONFLICT ON CONSTRAINT ux_scipion_set_items_set_item + DO UPDATE SET + enabled = EXCLUDED.enabled, + label = EXCLUDED.label, + comment = EXCLUDED.comment, + creation = EXCLUDED.creation, + "values" = EXCLUDED."values", + "updatedAt" = NOW() + """, + rows, + template="(%s, %s, %s, %s, %s, %s, %s::jsonb)", + page_size=len(rows), + ) + + def _updateSetProperties(self, setId: int, properties: Dict[str, Any]) -> None: + self.db.execute( + """ + UPDATE scipion_sets + SET properties = %s::jsonb, + "updatedAt" = NOW() + WHERE id = %s + """, + (self._jsonParam(properties), setId), + commit=False, + ) + + def _shouldSkipSetSync( + self, + existingProperties: Dict[str, Any], + itemsCountHint: Optional[int], + maxItemIdHint: Optional[int], + sourceMTime: Optional[float], + ) -> bool: + if not existingProperties or not existingProperties.get("incremental"): + return False + + storedNestedTablesVersion = self._toOptionalInt( + existingProperties.get("nestedTablesVersion") + ) + if storedNestedTablesVersion != NESTED_LOGICAL_TABLES_VERSION: + return False + + if itemsCountHint is None: + return False + + storedItemsCount = self._toOptionalInt(existingProperties.get("itemsCount")) + if storedItemsCount != itemsCountHint: + return False + + storedMaxItemId = self._toOptionalInt(existingProperties.get("maxItemId")) + if maxItemIdHint is not None: + if storedMaxItemId is None: + return False + if storedMaxItemId != maxItemIdHint: + return False + + storedSourceMTime = self._toOptionalFloat(existingProperties.get("sourceMTime")) + if sourceMTime is not None: + if storedSourceMTime is None: + return False + if abs(storedSourceMTime - sourceMTime) > 0.000001: + return False + + return True + + def _getSetItemsCountHint(self, scipionSet: Any) -> Optional[int]: + for methodName in ("getSize", "count"): + getter = getattr(scipionSet, methodName, None) + if not callable(getter): + continue + try: + value = getter() + except Exception: + continue + countValue = self._toOptionalInt(value) + if countValue is not None: + return countValue + + try: + return int(len(scipionSet)) + except Exception: + return None + + def _getSetMaxItemIdHint(self, scipionSet: Any) -> Optional[int]: + for methodName in ("getMaxId", "maxId", "getLastId"): + getter = getattr(scipionSet, methodName, None) + if not callable(getter): + continue + try: + value = getter() + except Exception: + continue + maxValue = self._toOptionalInt(value) + if maxValue is not None: + return maxValue + + getter = getattr(scipionSet, "getLastItem", None) + if callable(getter): + try: + return self._getSourceObjId(getter()) + except Exception: + return None + + return None + + def _getSetSourceMTime(self, scipionSet: Any) -> Optional[float]: + fileName = self._callOptionalGetter(scipionSet, "getFileName") + if not fileName: + return None + + try: + filePath = str(fileName) + if not os.path.exists(filePath): + return None + return float(os.path.getmtime(filePath)) + except Exception: + return None + + def _normalizeProperties(self, properties: Any) -> Dict[str, Any]: + if isinstance(properties, dict): + return dict(properties) + if isinstance(properties, str): + try: + parsed = json.loads(properties) + return dict(parsed) if isinstance(parsed, dict) else {} + except Exception: + return {} + return {} + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except Exception: + return None + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + try: + return float(value) + except Exception: + return None + + def _iterSetItems(self, scipionSet: Any) -> Iterable[Any]: + iterItems = getattr(scipionSet, "iterItems", None) + if callable(iterItems): + return iterItems() + + try: + return iter(scipionSet) + except TypeError: + raise ValueError("scipionSet must provide iterItems() or be iterable") + + def _getItemSchema(self, item: Any) -> Dict[str, Any]: + return self._getObjDict(item, includeClass=True) + + def _getItemValues(self, item: Any, scipionSet: Optional[Any] = None) -> Dict[str, Any]: + rawValues = self._getObjDict(item, includeClass=False) + + values = { + str(label): self._toJsonValue(value) + for label, value in (rawValues or {}).items() + if str(label) != SELF_LABEL + } + + self._addRelationIdentityValues( + item=item, + values=values, + ) + + classSize = self._getClassItemSize(item) + if classSize is not None and "_size" not in values: + values["_size"] = classSize + + self._addCoordinate3dBottomLeftCoordinates( + item=item, + values=values, + scipionSet=scipionSet, + ) + + return values + + def _addRelationIdentityValues( + self, + item: Any, + values: Dict[str, Any], + ) -> None: + tsId = self._getFirstGetterValue( + item, + ("getTsId", "getTiltSeriesId"), + ) + if tsId is not None and not values.get("_tsId"): + values["_tsId"] = self._toJsonValue(tsId) + + tomoId = self._getFirstGetterValue( + item, + ("getTomoId",), + ) + if tomoId is not None and not values.get("_tomoId"): + values["_tomoId"] = self._toJsonValue(tomoId) + + def _getFirstGetterValue( + self, + item: Any, + getterNames: Tuple[str, ...], + ) -> Optional[Any]: + for getterName in getterNames: + getter = getattr(item, getterName, None) + if not callable(getter): + continue + + try: + value = getter() + except Exception: + continue + + getterValue = getattr(value, "get", None) + if callable(getterValue): + try: + value = getterValue() + except Exception: + continue + + if value is None: + continue + + text = str(value).strip() + if text: + return value + + return None + + def _getClassItemSize(self, item: Any) -> Optional[int]: + className = self._getClassName(item) or item.__class__.__name__ + if not str(className or "").startswith("Class"): + return None + + for methodName in ("getSize", "getObjSize", "count"): + getter = getattr(item, methodName, None) + if not callable(getter): + continue + + try: + value = getter() + except Exception: + continue + + sizeValue = self._toOptionalInt(value) + if sizeValue is not None: + return sizeValue + + try: + return int(len(item)) + except Exception: + return None + + def _addCoordinate3dBottomLeftCoordinates( + self, + item: Any, + values: Dict[str, Any], + scipionSet: Optional[Any] = None, + ) -> None: + coords = self._getCoordinate3dBottomLeftCoordinates( + item=item, + values=values, + scipionSet=scipionSet, + ) + if coords is None: + return + + x, y, z = coords + + if "_x" in values and "rawX" not in values: + values["rawX"] = values.get("_x") + if "_y" in values and "rawY" not in values: + values["rawY"] = values.get("_y") + if "_z" in values and "rawZ" not in values: + values["rawZ"] = values.get("_z") + + values["bottomLeftX"] = x + values["bottomLeftY"] = y + values["bottomLeftZ"] = z + values["coordinateConvention"] = "BOTTOM_LEFT_CORNER" + + def _getCoordinate3dBottomLeftCoordinates( + self, + item: Any, + values: Optional[Dict[str, Any]] = None, + scipionSet: Optional[Any] = None, + ) -> Optional[Tuple[float, float, float]]: + if BOTTOM_LEFT_CORNER is None: + return None + + self._attachCoordinate3dTomogram( + item=item, + values=values or {}, + scipionSet=scipionSet, + ) + + return self._readCoordinate3dBottomLeftCoordinates(item) + + def _readCoordinate3dBottomLeftCoordinates(self, item: Any) -> Optional[Tuple[float, float, float]]: + getX = getattr(item, "getX", None) + getY = getattr(item, "getY", None) + getZ = getattr(item, "getZ", None) + + if not callable(getX) or not callable(getY) or not callable(getZ): + return None + + try: + return ( + float(getX(BOTTOM_LEFT_CORNER)), + float(getY(BOTTOM_LEFT_CORNER)), + float(getZ(BOTTOM_LEFT_CORNER)), + ) + except Exception: + return None + + def _attachCoordinate3dTomogram( + self, + item: Any, + values: Dict[str, Any], + scipionSet: Optional[Any], + ) -> bool: + if scipionSet is None: + return False + + tomogram = self._resolveCoordinate3dTomogram( + item=item, + values=values, + scipionSet=scipionSet, + ) + if tomogram is None: + return False + + setVolume = getattr(item, "setVolume", None) + if callable(setVolume): + try: + setVolume(tomogram) + return True + except Exception: + pass + + return False + + def _resolveCoordinate3dTomogram( + self, + item: Any, + values: Dict[str, Any], + scipionSet: Any, + ) -> Optional[Any]: + candidateKeys = self._getCoordinate3dTomogramCandidateKeys(item, values) + if not candidateKeys: + return None + + getTomogram = getattr(scipionSet, "_getTomogram", None) + if callable(getTomogram): + for key in candidateKeys: + for candidate in self._expandTomogramLookupKey(key): + try: + tomogram = getTomogram(candidate) + except Exception: + tomogram = None + + if tomogram is not None: + return tomogram + + for tomogram in self._iterLinkedTomograms(scipionSet): + tomogramKeys = self._getTomogramObjectMatchKeys(tomogram) + if tomogramKeys.intersection(candidateKeys): + return tomogram + + return None + + def _getCoordinate3dTomogramCandidateKeys( + self, + item: Any, + values: Dict[str, Any], + ) -> set: + candidates = [] + + for keyName in ( + "_tomoId", + "_volId", + "_volumeId", + "tomoId", + "tomogramId", + "volId", + "volumeId", + "tsId", + "tiltSeriesId", + ): + value = self._getValueByNormalizedKey(values, keyName) + text = self._toMatchText(value) + if text: + candidates.append(text) + + for getterName in ("getTomoId", "getVolId", "getVolumeId", "getTsId"): + value = self._callOptionalGetter(item, getterName) + text = self._toMatchText(value) + if text: + candidates.append(text) + + return { + str(value) + for value in candidates + if value is not None and str(value).strip() + } + + def _getTomogramObjectMatchKeys(self, tomogram: Any) -> set: + candidates = [] + + for getterName in ("getObjId", "getTsId", "getTomoId", "getNameId", "getObjLabel"): + value = self._callOptionalGetter(tomogram, getterName) + text = self._toMatchText(value) + if text: + candidates.append(text) + + return { + str(value) + for value in candidates + if value is not None and str(value).strip() + } + + def _expandTomogramLookupKey(self, key: Any) -> List[Any]: + values = [key] + + intValue = self._toOptionalInt(key) + if intValue is not None: + values.append(intValue) + + return values + + def _getValueByNormalizedKey(self, values: Dict[str, Any], keyName: str) -> Any: + targetKey = self._normalizeMatchKey(keyName) + + for key, value in values.items(): + if self._normalizeMatchKey(key) == targetKey: + return value + + return None + + def _normalizeMatchKey(self, value: Any) -> str: + return str(value).replace("_", "").replace(".", "").replace("-", "").lower() + + def _toMatchText(self, value: Any) -> Optional[str]: + if value is None: + return None + + getter = getattr(value, "get", None) + if callable(getter): + try: + value = getter() + except Exception: + return None + + text = str(value).strip() + return text or None + + def _callCoordinateGetter(self, item: Any, getterName: str) -> Optional[float]: + getter = getattr(item, getterName, None) + if not callable(getter): + return None + + try: + return float(getter(BOTTOM_LEFT_CORNER)) + except Exception: + return None + + def _getObjDict(self, scipionObj: Any, includeClass: bool) -> Dict[str, Any]: + getter = getattr(scipionObj, "getObjDict", None) + if not callable(getter): + return {} + + try: + return dict(getter(includeClass=includeClass) or {}) + except TypeError: + try: + return dict(getter(includeClass) or {}) + except TypeError: + if includeClass: + return {} + return dict(getter() or {}) + except Exception: + return {} + + def _getSetColumns(self, itemSchema: Dict[str, Any]) -> List[Dict[str, Any]]: + columns = [] + position = 0 + + for label, rawValue in itemSchema.items(): + labelProperty = str(label) + if labelProperty == SELF_LABEL: + continue + + className = self._getSchemaClassName(rawValue) + columns.append( + { + "labelProperty": labelProperty, + "columnName": "c%02d" % position, + "className": className, + "valueType": self._getColumnValueType(className), + "position": position, + "indexed": False, + } + ) + position += 1 + + if self._schemaIsClassItem(itemSchema) and not any( + column.get("labelProperty") == "_size" + for column in columns + ): + columns.append( + { + "labelProperty": "_size", + "columnName": "c%02d" % position, + "className": "Integer", + "valueType": "integer", + "position": position, + "indexed": True, + } + ) + + return columns + + def _getItemClassName(self, item: Any, itemSchema: Dict[str, Any]) -> str: + selfSchema = itemSchema.get(SELF_LABEL) + schemaClassName = self._getSchemaClassName(selfSchema) + if schemaClassName: + return schemaClassName + return self._getClassName(item) or item.__class__.__name__ if item is not None else "Unknown" + + def _schemaIsClassItem(self, itemSchema: Dict[str, Any]) -> bool: + selfSchema = itemSchema.get(SELF_LABEL) + className = self._getSchemaClassName(selfSchema) + return str(className or "").startswith("Class") + + def _getSchemaClassName(self, schemaValue: Any) -> Optional[str]: + if isinstance(schemaValue, (tuple, list)) and schemaValue: + return str(schemaValue[0]) if schemaValue[0] else None + if isinstance(schemaValue, dict): + className = schemaValue.get("className") or schemaValue.get("class_name") + return str(className) if className else None + return None + + def _getColumnValueType(self, className: Optional[str]) -> Optional[str]: + if className in ("Integer", "Long", "Boolean"): + return "integer" + if className in ("Float", "Decimal"): + return "float" + if className in ("String", "CsvList", "Pointer", "PointerList"): + return "text" + return className + + def _getSetProperties(self, scipionSet: Any) -> Dict[str, Any]: + properties: Dict[str, Any] = { + "className": self._getClassName(scipionSet), + "moduleName": self._getModuleName(scipionSet), + "baseClassName": self._getBaseClassName(scipionSet), + "scipionObjId": self._getSourceObjId(scipionSet), + } + + fileName = self._callOptionalGetter(scipionSet, "getFileName") + if fileName is not None: + properties["fileName"] = self._toJsonValue(fileName) + + streamState = self._callOptionalGetter(scipionSet, "getStreamState") + if streamState is not None: + properties["streamState"] = self._toJsonValue(streamState) + + linkedTomograms = self._getLinkedTomogramsSummary(scipionSet) + if linkedTomograms: + properties["linkedTomograms"] = linkedTomograms + + + for attrName, attrValue in self._getAttributesToStore(scipionSet): + if self._getAttributesToStore(attrValue): + continue + properties[str(attrName)] = self._toJsonValue(self._getObjectValueText(attrValue)) + + return {key: value for key, value in properties.items() if value is not None} + + def _getLinkedTomogramsSummary(self, scipionSet: Any) -> List[Dict[str, Any]]: + tomograms = [] + + for index, tomogram in enumerate(self._iterLinkedTomograms(scipionSet)): + item = self._buildLinkedTomogramSummary(tomogram, index) + if item is not None: + tomograms.append(item) + + return tomograms + + def _iterLinkedTomograms(self, scipionSet: Any) -> Iterable[Any]: + for methodName in ("iterTomograms", "iterVolumes"): + iteratorGetter = getattr(scipionSet, methodName, None) + if not callable(iteratorGetter): + continue + + try: + return iteratorGetter() + except Exception: + continue + + getTomograms = getattr(scipionSet, "getTomograms", None) + if callable(getTomograms): + try: + tomograms = getTomograms() + iterItems = getattr(tomograms, "iterItems", None) + if callable(iterItems): + try: + return iterItems(iterate=False) + except TypeError: + return iterItems() + return iter(tomograms) + except Exception: + pass + + return iter(()) + + def _buildLinkedTomogramSummary(self, tomogram: Any, index: int) -> Optional[Dict[str, Any]]: + objectId = self._callOptionalGetter(tomogram, "getObjId") + tsId = self._callOptionalGetter(tomogram, "getTsId") + tomoId = self._callOptionalGetter(tomogram, "getTomoId") + + stableId = tsId or tomoId or objectId or index + if stableId is None: + return None + + name = None + for methodName in ("getObjLabel", "getNameId", "getFileName"): + value = self._callOptionalGetter(tomogram, methodName) + if value: + name = value + break + + if not name: + name = stableId + + dims = self._normalizeLinkedTomogramDims( + self._callOptionalGetter(tomogram, "getDim") + ) + + samplingRate = self._toOptionalFloat( + self._callOptionalGetter(tomogram, "getSamplingRate") + ) + + item: Dict[str, Any] = { + "id": str(stableId), + "tomoId": str(stableId), + "label": str(stableId), + "name": str(name), + } + + if objectId is not None: + item["objectId"] = str(objectId) + item["volumeId"] = str(objectId) + + if tsId is not None: + item["tsId"] = str(tsId) + item["tiltSeriesId"] = str(tsId) + + fileName = self._callOptionalGetter(tomogram, "getFileName") + if fileName: + item["fileName"] = str(fileName) + + if dims is not None: + item["dims"] = dims + + if samplingRate is not None: + item["voxelSize"] = [samplingRate, samplingRate, samplingRate] + + return item + + def _normalizeLinkedTomogramDims(self, dims: Any) -> Optional[List[int]]: + if dims is None: + return None + + try: + values = list(dims) + except Exception: + return None + + if len(values) < 3: + return None + + out = [] + for value in values[:3]: + intValue = self._toOptionalInt(value) + if intValue is None or intValue <= 0: + return None + out.append(intValue) + + return out + + def _getItemEnabled(self, item: Any) -> bool: + isEnabled = getattr(item, "isEnabled", None) + if not callable(isEnabled): + return True + try: + return bool(isEnabled()) + except Exception: + return True + + def _callOptionalGetter(self, scipionObj: Any, getterName: str) -> Any: + getter = getattr(scipionObj, getterName, None) + if not callable(getter): + return None + try: + return getter() + except Exception: + return None + + def _toJsonValue(self, value: Any) -> Any: + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if isinstance(value, (list, tuple)): + return [self._toJsonValue(item) for item in value] + if isinstance(value, dict): + return {str(key): self._toJsonValue(item) for key, item in value.items()} + + isoformat = getattr(value, "isoformat", None) + if callable(isoformat): + try: + return isoformat() + except Exception: + pass + + getter = getattr(value, "getObjValue", None) + if callable(getter): + try: + return self._toJsonValue(getter()) + except Exception: + pass + + getter = getattr(value, "get", None) + if callable(getter): + try: + return self._toJsonValue(getter()) + except Exception: + pass + + return str(value) + + def _stringifyPropertyValue(self, value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, (dict, list, tuple)): + return json.dumps(self._toJsonValue(value), ensure_ascii=False) + return str(value) + + def _nextOrNone(self, iterator: Iterator[Any]) -> Any: + try: + return next(iterator) + except StopIteration: + return None + + def _chainFirst(self, firstItem: Any, remainingItems: Iterator[Any]) -> Iterator[Any]: + yield firstItem + for item in remainingItems: + yield item \ No newline at end of file diff --git a/app/backend/models/__init__.py b/app/backend/models/__init__.py index 4abd50b4..8ae235f9 100644 --- a/app/backend/models/__init__.py +++ b/app/backend/models/__init__.py @@ -28,3 +28,13 @@ from app.backend.models.protocol_model import Protocol # noqa: F401 from app.backend.models.settings_models import * # noqa: F401 from app.backend.models.protocol_step_model import ProtocolStep # noqa: F401 +from app.backend.models.scipion_object_model import ( # noqa: F401 + ScipionObject, + ScipionObjectRelation, + ScipionObjectType, + ScipionObjectTypeProperty, + ScipionSet, + ScipionSetColumn, + ScipionSetItem, + ScipionSetProperty, +) diff --git a/app/backend/models/scipion_object_model.py b/app/backend/models/scipion_object_model.py new file mode 100644 index 00000000..ed10ab6b --- /dev/null +++ b/app/backend/models/scipion_object_model.py @@ -0,0 +1,216 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from sqlalchemy import Boolean, CheckConstraint, Column, DateTime, ForeignKey, Index, Integer, Text, UniqueConstraint, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import relationship +from sqlalchemy.sql import func + +from app.backend.database import Base + + +class ScipionObjectType(Base): + __tablename__ = "scipion_object_types" + + id = Column(Integer, primary_key=True, index=True) + className = Column(Text, nullable=False, unique=True) + moduleName = Column(Text, nullable=True) + baseClassName = Column(Text, nullable=True) + mapperKind = Column(Text, nullable=False, default="tree", server_default="tree") + classSchema = Column("schema", JSONB, nullable=False, server_default=text("'{}'::jsonb")) + createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) + + properties = relationship("ScipionObjectTypeProperty", back_populates="type", cascade="all, delete-orphan") + + __table_args__ = ( + CheckConstraint('"mapperKind" IN (\'tree\', \'flat_set\', \'scalar\', \'pointer\')', name="ck_scipion_object_types_mapper_kind"), + Index("idx_scipion_object_types_className", "className"), + Index("idx_scipion_object_types_mapperKind", "mapperKind"), + Index("idx_scipion_object_types_schema_gin", "schema", postgresql_using="gin"), + ) + + +class ScipionObjectTypeProperty(Base): + __tablename__ = "scipion_object_type_properties" + + id = Column(Integer, primary_key=True, index=True) + typeId = Column(Integer, ForeignKey("scipion_object_types.id", ondelete="CASCADE"), nullable=False) + propertyPath = Column(Text, nullable=False) + className = Column(Text, nullable=True) + valueKind = Column(Text, nullable=True) + isPointer = Column(Boolean, nullable=False, default=False, server_default="false") + isNested = Column(Boolean, nullable=False, default=False, server_default="false") + propertySchema = Column("schema", JSONB, nullable=False, server_default=text("'{}'::jsonb")) + createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) + + type = relationship("ScipionObjectType", back_populates="properties") + + __table_args__ = ( + UniqueConstraint("typeId", "propertyPath", name="ux_scipion_object_type_properties_type_path"), + Index("idx_scipion_object_type_properties_path", "propertyPath"), + Index("idx_scipion_object_type_properties_schema_gin", "schema", postgresql_using="gin"), + ) + + +class ScipionObject(Base): + __tablename__ = "scipion_objects" + + id = Column(Integer, primary_key=True, index=True) + projectId = Column(Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + protocolDbId = Column(Integer, ForeignKey("protocols.id", ondelete="CASCADE"), nullable=True) + scipionObjId = Column(Integer, nullable=True) + parentObjectId = Column(Integer, ForeignKey("scipion_objects.id", ondelete="CASCADE"), nullable=True) + name = Column(Text, nullable=True) + path = Column(Text, nullable=False) + className = Column(Text, nullable=False) + value = Column(Text, nullable=True) + label = Column(Text, nullable=True) + comment = Column(Text, nullable=True) + creation = Column(DateTime(timezone=True), nullable=True) + objectMetadata = Column("metadata", JSONB, nullable=False, server_default=text("'{}'::jsonb")) + createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) + + parent = relationship("ScipionObject", remote_side=[id], back_populates="children") + children = relationship("ScipionObject", back_populates="parent", cascade="all, delete-orphan") + + __table_args__ = ( + UniqueConstraint("projectId", "protocolDbId", "scipionObjId", name="ux_scipion_objects_project_protocol_obj"), + UniqueConstraint("projectId", "protocolDbId", "path", name="ux_scipion_objects_project_protocol_path"), + Index("idx_scipion_objects_project_class", "projectId", "className"), + Index("idx_scipion_objects_project_protocol", "projectId", "protocolDbId"), + Index("idx_scipion_objects_parent", "parentObjectId"), + Index("idx_scipion_objects_metadata_gin", "metadata", postgresql_using="gin"), + ) + + +class ScipionObjectRelation(Base): + __tablename__ = "scipion_object_relations" + + id = Column(Integer, primary_key=True, index=True) + projectId = Column(Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + creatorObjectId = Column(Integer, ForeignKey("scipion_objects.id", ondelete="CASCADE"), nullable=False) + parentObjectId = Column(Integer, ForeignKey("scipion_objects.id", ondelete="CASCADE"), nullable=False) + childObjectId = Column(Integer, ForeignKey("scipion_objects.id", ondelete="CASCADE"), nullable=False) + name = Column(Text, nullable=False) + parentExtended = Column(Text, nullable=True) + childExtended = Column(Text, nullable=True) + relationMetadata = Column("metadata", JSONB, nullable=False, server_default=text("'{}'::jsonb")) + creation = Column(DateTime(timezone=True), nullable=True) + createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + + __table_args__ = ( + Index("idx_scipion_object_relations_project_name", "projectId", "name"), + Index("idx_scipion_object_relations_parent", "parentObjectId"), + Index("idx_scipion_object_relations_child", "childObjectId"), + Index("idx_scipion_object_relations_metadata_gin", "metadata", postgresql_using="gin"), + ) + + +class ScipionSet(Base): + __tablename__ = "scipion_sets" + + id = Column(Integer, primary_key=True, index=True) + projectId = Column(Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + protocolDbId = Column(Integer, ForeignKey("protocols.id", ondelete="CASCADE"), nullable=True) + objectId = Column(Integer, ForeignKey("scipion_objects.id", ondelete="SET NULL"), nullable=True) + outputName = Column(Text, nullable=False) + setClassName = Column(Text, nullable=False) + itemClassName = Column(Text, nullable=False) + properties = Column(JSONB, nullable=False, server_default=text("'{}'::jsonb")) + createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) + + columns = relationship("ScipionSetColumn", back_populates="set", cascade="all, delete-orphan") + setProperties = relationship("ScipionSetProperty", back_populates="set", cascade="all, delete-orphan") + items = relationship("ScipionSetItem", back_populates="set", cascade="all, delete-orphan") + + __table_args__ = ( + UniqueConstraint("projectId", "protocolDbId", "outputName", name="ux_scipion_sets_project_protocol_output"), + Index("idx_scipion_sets_project_protocol", "projectId", "protocolDbId"), + Index("idx_scipion_sets_properties_gin", "properties", postgresql_using="gin"), + ) + + +class ScipionSetColumn(Base): + __tablename__ = "scipion_set_columns" + + id = Column(Integer, primary_key=True, index=True) + setId = Column(Integer, ForeignKey("scipion_sets.id", ondelete="CASCADE"), nullable=False) + labelProperty = Column(Text, nullable=False) + columnName = Column(Text, nullable=False) + className = Column(Text, nullable=True) + valueType = Column(Text, nullable=True) + position = Column(Integer, nullable=False) + indexed = Column(Boolean, nullable=False, default=False, server_default="false") + + set = relationship("ScipionSet", back_populates="columns") + + __table_args__ = ( + UniqueConstraint("setId", "labelProperty", name="ux_scipion_set_columns_set_label"), + UniqueConstraint("setId", "columnName", name="ux_scipion_set_columns_set_column"), + Index("idx_scipion_set_columns_label", "labelProperty"), + ) + + +class ScipionSetProperty(Base): + __tablename__ = "scipion_set_properties" + + id = Column(Integer, primary_key=True, index=True) + setId = Column(Integer, ForeignKey("scipion_sets.id", ondelete="CASCADE"), nullable=False) + key = Column(Text, nullable=False) + value = Column(Text, nullable=True) + + set = relationship("ScipionSet", back_populates="setProperties") + + __table_args__ = ( + UniqueConstraint("setId", "key", name="ux_scipion_set_properties_set_key"), + Index("idx_scipion_set_properties_key", "key"), + ) + + +class ScipionSetItem(Base): + __tablename__ = "scipion_set_items" + + id = Column(Integer, primary_key=True, index=True) + setId = Column(Integer, ForeignKey("scipion_sets.id", ondelete="CASCADE"), nullable=False) + scipionItemId = Column(Integer, nullable=False) + enabled = Column(Boolean, nullable=False, default=True, server_default="true") + label = Column(Text, nullable=True) + comment = Column(Text, nullable=True) + creation = Column(DateTime(timezone=True), nullable=True) + values = Column(JSONB, nullable=False, server_default=text("'{}'::jsonb")) + createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) + + set = relationship("ScipionSet", back_populates="items") + + __table_args__ = ( + UniqueConstraint("setId", "scipionItemId", name="ux_scipion_set_items_set_item"), + Index("idx_scipion_set_items_set", "setId"), + Index("idx_scipion_set_items_values_gin", "values", postgresql_using="gin"), + ) diff --git a/app/backend/utils/file_handlers.py b/app/backend/utils/file_handlers.py index f1c02af7..02ad7eec 100644 --- a/app/backend/utils/file_handlers.py +++ b/app/backend/utils/file_handlers.py @@ -68,6 +68,9 @@ def getProtocolPath(self, protocolId): """ Return the protocol browser paths. + protocolId must be a Scipion runtime protocol id. PostgreSQL protocol + ids must be resolved by ProjectService before calling FileHandlers. + Returns: - rootAbs: absolute root boundary (project folder inferred) - startPath: relative to rootAbs ("" means rootAbs) diff --git a/app/backend/utils/thumbnail_service.py b/app/backend/utils/thumbnail_service.py index 1b803b0b..fa3eca82 100644 --- a/app/backend/utils/thumbnail_service.py +++ b/app/backend/utils/thumbnail_service.py @@ -176,6 +176,12 @@ def buildProtocolThumbnail( size: int = 360, outputName: Optional[str] = None, ) -> Dict[str, Any]: + """ + Build a thumbnail for a Scipion runtime protocol id. + + PostgreSQL protocol ids must be resolved by ProjectService before + calling ThumbnailService. + """ try: protocol = self.currentProject.getProtocol(int(protocolId)) except Exception: diff --git a/app/backend/viewers/__init__.py b/app/backend/viewers/__init__.py new file mode 100644 index 00000000..4cdb527e --- /dev/null +++ b/app/backend/viewers/__init__.py @@ -0,0 +1,25 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** diff --git a/app/backend/viewers/postgresql_coords2d_reader.py b/app/backend/viewers/postgresql_coords2d_reader.py new file mode 100644 index 00000000..2f3339ba --- /dev/null +++ b/app/backend/viewers/postgresql_coords2d_reader.py @@ -0,0 +1,366 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from typing import Any, Dict, List, Optional, Set + +from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + +class PostgresqlCoords2dReader: + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + self.setMapper = ScipionSetPostgresqlMapper(db) + self._storedSet = None + self.lastSkipReason = None + + def hasOutput(self) -> bool: + storedSet = self._getStoredSet() + return storedSet is not None and self._isCoords2dStoredSet(storedSet) + + def listMicrographs(self) -> Optional[Dict[str, Any]]: + self.lastSkipReason = None + + storedSet = self._getStoredSet() + if storedSet is None: + self.lastSkipReason = "stored_set_not_found" + return None + + if not self._isCoords2dStoredSet(storedSet): + self.lastSkipReason = "stored_set_is_not_coordinates2d" + return None + + countsByMicId: Dict[str, int] = {} + micrographsById: Dict[str, Dict[str, Any]] = {} + + for item in storedSet.get("items") or []: + values = item.get("values") or {} + micId = self._extractMicId(item, values) + + if not micId: + continue + + countsByMicId[micId] = countsByMicId.get(micId, 0) + 1 + + if micId not in micrographsById: + micrographsById[micId] = self._buildMicrographSummary( + micId=micId, + item=item, + values=values, + ) + + if not micrographsById: + self.lastSkipReason = "micrographs_not_resolved_from_coordinates" + return None + + micrographs: List[Dict[str, Any]] = [] + for index, micId in enumerate(sorted(micrographsById.keys(), key=self._micrographSortKey), start=1): + summary = dict(micrographsById[micId]) + summary["index"] = index + summary["particles"] = int(countsByMicId.get(micId, 0)) + micrographs.append(summary) + + boxSize = self._extractBoxSize(storedSet) + totalPicks = sum(int(item.get("particles") or 0) for item in micrographs) + + return { + "micrographs": micrographs, + "totalMicrographs": len(micrographs), + "totalPicks": totalPicks, + "boxSize": boxSize, + } + + def listCoordinatesForMicrograph(self, micId: Any) -> Optional[Dict[str, Any]]: + self.lastSkipReason = None + + storedSet = self._getStoredSet() + if storedSet is None: + self.lastSkipReason = "stored_set_not_found" + return None + + if not self._isCoords2dStoredSet(storedSet): + self.lastSkipReason = "stored_set_is_not_coordinates2d" + return None + + targetMicId = str(micId) + coordinates: List[Dict[str, Any]] = [] + + for index, item in enumerate(storedSet.get("items") or []): + values = item.get("values") or {} + itemMicId = self._extractMicId(item, values) + + if str(itemMicId) != targetMicId: + continue + + point = self._buildCoordinatePoint( + item=item, + values=values, + micId=targetMicId, + fallbackIndex=index, + ) + + if point is not None: + coordinates.append(point) + + if not coordinates: + self.lastSkipReason = "coordinates_not_found_for_micrograph micId=%s" % targetMicId + return None + + return {"coordinates": coordinates} + + def _getStoredSet(self) -> Optional[Dict[str, Any]]: + if self._storedSet is None: + self._storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=self.protocolId, + outputName=self.outputName, + ) + return self._storedSet + + def _isCoords2dStoredSet(self, storedSet: Dict[str, Any]) -> bool: + classText = ("%s %s" % ( + storedSet.get("setClassName") or "", + storedSet.get("itemClassName") or "", + )).replace(" ", "").lower() + + if "coordinate" not in classText: + return False + + if "coordinates3d" in classText or "coordinate3d" in classText: + return False + + if "tomogram" in classText or "tomo" in classText: + return False + + return True + + def _buildMicrographSummary( + self, + micId: str, + item: Dict[str, Any], + values: Dict[str, Any], + ) -> Dict[str, Any]: + label = ( + self._firstValueBySuffix(values, ["micname", "micrographname", "filename", "filepath"]) + or item.get("label") + or "Micrograph %s" % micId + ) + + fileName = self._firstValueBySuffix( + values, + ["filename", "filepath", "micfilename", "micrographfilename", "path"], + ) + + width = self._toOptionalInt( + self._firstValueBySuffix(values, ["width", "xdim", "dimx"]) + ) + height = self._toOptionalInt( + self._firstValueBySuffix(values, ["height", "ydim", "dimy"]) + ) + + return { + "id": str(micId), + "fileName": str(fileName) if fileName else "", + "label": str(label), + "particles": 0, + "updated": False, + "width": width, + "height": height, + "locationIndex": self._toOptionalInt( + self._firstValueBySuffix(values, ["locationindex", "imageindex", "index"]) + ), + "thumbnailUrl": None, + } + + def _buildCoordinatePoint( + self, + item: Dict[str, Any], + values: Dict[str, Any], + micId: str, + fallbackIndex: int, + ) -> Optional[Dict[str, Any]]: + x = self._extractCoordinateValue(values, ["_x", "x", "coordx", "coordinatex", "positionx"]) + y = self._extractCoordinateValue(values, ["_y", "y", "coordy", "coordinatey", "positiony"]) + + if x is None or y is None: + return None + + objId = item.get("scipionItemId") + if objId is None: + objId = item.get("id") + if objId is None: + objId = "%s:%s" % (micId, fallbackIndex) + + return { + "id": objId, + "micId": str(micId), + "x": x, + "y": y, + "score": self._toOptionalFloat( + self._firstValueBySuffix(values, ["score", "weight"]) + ), + "classLabel": self._optionalString( + self._firstValueBySuffix(values, ["classid", "classlabel", "objlabel"]) + ), + } + + def _extractMicId(self, item: Dict[str, Any], values: Dict[str, Any]) -> Optional[str]: + value = self._firstValue( + values, + [ + "_micId", + "micId", + "micrographId", + "micrograph.id", + "micrograph._objId", + "_mic._objId", + "_micrograph._objId", + ], + ) + + if value is None: + value = self._firstValueBySuffix( + values, + ["micid", "micrographid", "micobjid", "micrographobjid"], + ) + + if value is None: + value = self._findNestedValueBySuffix( + values, + ["micid", "micrographid", "objid"], + ) + + if value is None: + return None + + return str(value) + + def _extractBoxSize(self, storedSet: Dict[str, Any]) -> Optional[int]: + properties = storedSet.get("properties") or {} + setProperties = storedSet.get("setProperties") or [] + + value = None + if isinstance(properties, dict): + value = self._firstValueBySuffix(properties, ["boxsize", "box"]) + + if value is None: + for row in setProperties: + key = str((row or {}).get("key") or "").replace("_", "").lower() + if key.endswith("boxsize") or key == "box": + value = (row or {}).get("value") + break + + return self._toOptionalInt(value) + + def _extractCoordinateValue(self, values: Dict[str, Any], names: List[str]) -> Optional[float]: + value = self._firstValue(values, names) + if value is None: + value = self._firstValueBySuffix(values, names) + return self._toOptionalFloat(value) + + def _firstValue(self, values: Dict[str, Any], keys: List[str]) -> Any: + for key in keys: + if key in values: + return values.get(key) + return None + + def _firstValueBySuffix(self, values: Dict[str, Any], suffixes: List[str]) -> Any: + normalizedSuffixes = [ + str(suffix).replace("_", "").replace(".", "").lower() + for suffix in suffixes + ] + + for key, value in (values or {}).items(): + normalizedKey = str(key).replace("_", "").replace(".", "").lower() + for suffix in normalizedSuffixes: + if normalizedKey.endswith(suffix): + return value + + return None + + def _findNestedValueBySuffix(self, value: Any, suffixes: List[str]) -> Any: + normalizedSuffixes = [ + str(suffix).replace("_", "").replace(".", "").lower() + for suffix in suffixes + ] + + def walk(node: Any, path: str = "") -> Any: + if isinstance(node, dict): + for key, child in node.items(): + nextPath = "%s.%s" % (path, key) if path else str(key) + normalizedPath = nextPath.replace("_", "").replace(".", "").lower() + for suffix in normalizedSuffixes: + if normalizedPath.endswith(suffix): + return child + + found = walk(child, nextPath) + if found is not None: + return found + + if isinstance(node, list): + for index, child in enumerate(node): + found = walk(child, "%s.%s" % (path, index)) + if found is not None: + return found + + return None + + return walk(value) + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + + try: + return float(value) + except Exception: + return None + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + + try: + return int(value) + except Exception: + try: + return int(float(value)) + except Exception: + return None + + def _optionalString(self, value: Any) -> Optional[str]: + if value is None: + return None + + text = str(value).strip() + return text or None + + def _micrographSortKey(self, micId: str): + try: + return 0, int(micId) + except Exception: + return 1, str(micId).lower() \ No newline at end of file diff --git a/app/backend/viewers/postgresql_coords3d_reader.py b/app/backend/viewers/postgresql_coords3d_reader.py new file mode 100644 index 00000000..5424390f --- /dev/null +++ b/app/backend/viewers/postgresql_coords3d_reader.py @@ -0,0 +1,1061 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +import json +import os +import re +from typing import Any, Dict, List, Optional, Set + +from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + +class PostgresqlCoords3dReader: + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + self.setMapper = ScipionSetPostgresqlMapper(db) + self._storedSet = None + self.lastSkipReason = None + + def hasOutput(self) -> bool: + return self._getStoredSet() is not None + + def listTomograms(self) -> Optional[List[Dict[str, Any]]]: + self.lastSkipReason = None + + storedSet = self._getStoredSet() + if storedSet is None: + self.lastSkipReason = "stored_set_not_found" + return None + + countsByKey = self._countCoordinatesByTomogramKey(storedSet) + coordinateKeys = set(countsByKey.keys()) + + linkedTomograms = self._getLinkedTomogramsFromProperties(storedSet) + payload = self._buildTomogramPayloadFromLinkedTomograms( + linkedTomograms=linkedTomograms, + countsByKey=countsByKey, + ) + + if payload is not None: + return payload + + payload = self._buildTomogramPayloadFromProjectSets( + coordinateKeys=coordinateKeys, + countsByKey=countsByKey, + ) + + if payload is not None: + return payload + + if not linkedTomograms: + self.lastSkipReason = ( + "linked_tomograms_not_found " + "and_project_tomograms_not_resolved " + "coordinateKeys=%s" + ) % sorted(coordinateKeys) + else: + self.lastSkipReason = ( + "linked_tomograms_invalid " + "and_project_tomograms_not_resolved " + "coordinateKeys=%s" + ) % sorted(coordinateKeys) + + return None + + def getPoints(self, tomogramId: Any) -> Optional[List[Dict[str, Any]]]: + self.lastSkipReason = None + + storedSet = self._getStoredSet() + if storedSet is None: + self.lastSkipReason = "stored_set_not_found" + return None + + targetKeys = self._resolveTomogramTargetKeys(storedSet, tomogramId) + if not targetKeys: + self.lastSkipReason = "tomogram_target_keys_not_resolved tomogramId=%s" % str(tomogramId) + return None + + boxSize = self._extractBoxSize(storedSet) + points: List[Dict[str, Any]] = [] + + for item in storedSet.get("items") or []: + values = item.get("values") or {} + coordinateKeys = set(self._extractCoordinateTomogramKeys(values)) + + if not coordinateKeys.intersection(targetKeys): + continue + + point = self._buildCoordinatePoint( + item=item, + values=values, + tomogramId=tomogramId, + boxSize=boxSize, + ) + + if point is not None: + points.append(point) + + if not points: + self.lastSkipReason = ( + "coordinates_not_found_for_tomogram " + "tomogramId=%s targetKeys=%s" + ) % (str(tomogramId), sorted(targetKeys)) + return None + + return points + + def _getStoredSet(self) -> Optional[Dict[str, Any]]: + if self._storedSet is None: + self._storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=self.protocolId, + outputName=self.outputName, + ) + return self._storedSet + + def _buildTomogramPayloadFromLinkedTomograms( + self, + linkedTomograms: List[Dict[str, Any]], + countsByKey: Dict[str, int], + ) -> Optional[List[Dict[str, Any]]]: + if not linkedTomograms: + return None + + result = [] + for item in linkedTomograms: + normalized = self._normalizeLinkedTomogramItem(item) + if normalized is None: + continue + + count = self._findTomogramCount(normalized, countsByKey) + if count is not None: + normalized["nCoords"] = count + normalized["count"] = count + + result.append(normalized) + + if not result: + return None + + if not self._hasTomogramViewerContract(result): + return None + + return result + + def _buildTomogramPayloadFromProjectSets( + self, + coordinateKeys: Set[str], + countsByKey: Dict[str, int], + ) -> Optional[List[Dict[str, Any]]]: + if not coordinateKeys: + return None + + rows = self._getProjectTomogramRows() + if not rows: + return None + + resultById: Dict[str, Dict[str, Any]] = {} + + for row in rows: + summary = self._buildTomogramSummaryFromStoredItem(row) + if summary is None: + continue + + matchKeys = self._getTomogramMatchKeys(summary) + if not matchKeys.intersection(coordinateKeys): + continue + + count = self._findTomogramCount(summary, countsByKey) + if count is not None: + summary["nCoords"] = count + summary["count"] = count + + resultKey = str(summary.get("id")) + existing = resultById.get(resultKey) + + if existing is None: + resultById[resultKey] = summary + continue + + resultById[resultKey] = self._mergeTomogramSummaries(existing, summary) + + result = list(resultById.values()) + if not result: + return None + + if not self._hasTomogramViewerContract(result): + return None + + return result + + def _getProjectTomogramRows(self) -> List[Dict[str, Any]]: + try: + return self.db.fetchAll( + """ + SELECT + s.id AS "setId", + s."projectId", + s."protocolDbId", + s."outputName", + s."setClassName", + s."itemClassName", + s.properties AS "setProperties", + i.id AS "itemRowId", + i."scipionItemId", + i.enabled, + i.label, + i.comment, + i.creation, + i."values", + i."createdAt", + i."updatedAt" + FROM scipion_sets s + JOIN scipion_set_items i + ON i."setId" = s.id + WHERE s."projectId" = %s + AND ( + LOWER(COALESCE(s."setClassName", '')) LIKE '%%tomogram%%' + OR LOWER(COALESCE(s."itemClassName", '')) LIKE '%%tomogram%%' + OR LOWER(COALESCE(s."setClassName", '')) LIKE '%%volume%%' + OR LOWER(COALESCE(s."itemClassName", '')) LIKE '%%volume%%' + ) + ORDER BY + CASE + WHEN LOWER(COALESCE(s."itemClassName", '')) LIKE '%%tomogram%%' THEN 0 + WHEN LOWER(COALESCE(s."setClassName", '')) LIKE '%%tomogram%%' THEN 1 + ELSE 2 + END, + s."protocolDbId" ASC, + s."outputName" ASC, + i."scipionItemId" ASC + """, + (self.projectId,), + ) + except Exception: + return [] + + def _buildTomogramSummaryFromStoredItem(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]: + values = row.get("values") or {} + objectId = row.get("scipionItemId") + objectIdText = self._toTextCandidate(objectId) + + tsId = self._firstValueBySuffix( + values, + ["tsid", "tiltseriesid", "tilt_series_id"], + ) + + tomoId = self._firstValueBySuffix( + values, + ["tomoid", "tomogramid", "tomo_id", "tomogram_id"], + ) + + nameId = self._firstValueBySuffix( + values, + ["nameid", "name_id"], + ) + + labelValue = self._firstValueBySuffix( + values, + ["objlabel", "label"], + ) + + stableId = ( + self._toTextCandidate(tsId) + or self._toTextCandidate(tomoId) + or self._toTextCandidate(labelValue) + or self._toTextCandidate(nameId) + or objectIdText + ) + + if not stableId: + return None + + dims = self._extractDims(values) + if dims is None: + return None + + name = ( + self._toTextCandidate(nameId) + or self._toTextCandidate(labelValue) + or self._extractFileBasename(values) + or stableId + ) + + summary: Dict[str, Any] = { + "id": stableId, + "tomoId": stableId, + "label": str(tsId or stableId), + "name": str(name), + "dims": dims, + } + + voxelSize = self._extractVoxelSize(values) + if voxelSize is not None: + summary["voxelSize"] = voxelSize + + fileName = self._extractTomogramFile(values) + if fileName: + summary["fileName"] = str(fileName) + + if objectIdText: + summary["objectId"] = objectIdText + summary["volumeId"] = objectIdText + + if tsId is not None: + summary["tsId"] = str(tsId) + summary["tiltSeriesId"] = str(tsId) + + if tomoId is not None: + summary["sourceTomoId"] = str(tomoId) + + summary["sourceOutputName"] = str(row.get("outputName") or "") + summary["sourceProtocolId"] = str(row.get("protocolDbId") or "") + + return summary + + def _mergeTomogramSummaries( + self, + current: Dict[str, Any], + candidate: Dict[str, Any], + ) -> Dict[str, Any]: + currentScore = self._getTomogramSummaryScore(current) + candidateScore = self._getTomogramSummaryScore(candidate) + + if candidateScore > currentScore: + base = dict(candidate) + for key in ("nCoords", "count"): + if key in current and key not in base: + base[key] = current[key] + return base + + base = dict(current) + for key, value in candidate.items(): + if value is not None and key not in base: + base[key] = value + return base + + def _getTomogramSummaryScore(self, item: Dict[str, Any]) -> int: + score = 0 + + if item.get("dims"): + score += 10 + + if item.get("voxelSize"): + score += 5 + + if item.get("fileName"): + score += 3 + + classText = " ".join( + [ + str(item.get("sourceOutputName") or ""), + str(item.get("sourceProtocolId") or ""), + ] + ).lower() + + if "tomogram" in classText: + score += 2 + + return score + + def _hasTomogramViewerContract(self, items: List[Dict[str, Any]]) -> bool: + for item in items: + dims = item.get("dims") + if not isinstance(dims, list) or len(dims) < 3: + return False + + for value in dims[:3]: + intValue = self._toOptionalInt(value) + if intValue is None or intValue <= 0: + return False + + return True + + def _getLinkedTomogramsFromProperties(self, storedSet: Dict[str, Any]) -> List[Dict[str, Any]]: + properties = self._normalizeJsonObject(storedSet.get("properties")) + + linkedTomograms = properties.get("linkedTomograms") + if isinstance(linkedTomograms, list): + return [ + item + for item in linkedTomograms + if isinstance(item, dict) + ] + + setProperties = storedSet.get("setProperties") or [] + for item in setProperties: + if str(item.get("key")) != "linkedTomograms": + continue + + parsed = self._parseJsonValue(item.get("value")) + if isinstance(parsed, list): + return [ + entry + for entry in parsed + if isinstance(entry, dict) + ] + + return [] + + def _normalizeLinkedTomogramItem(self, item: Dict[str, Any]) -> Optional[Dict[str, Any]]: + tomoId = item.get("tomoId") or item.get("id") or item.get("tsId") or item.get("label") + if tomoId is None: + return None + + dims = self._normalizeDims(item.get("dims")) + if dims is None: + return None + + normalized: Dict[str, Any] = { + "id": str(tomoId), + "tomoId": str(tomoId), + "label": str(item.get("label") or tomoId), + "name": str(item.get("name") or item.get("label") or tomoId), + "dims": dims, + } + + voxelSize = self._normalizeVoxelSize(item.get("voxelSize")) + if voxelSize is not None: + normalized["voxelSize"] = voxelSize + + for key in ("objectId", "volumeId", "tsId", "tiltSeriesId", "fileName"): + value = item.get(key) + if value is not None: + normalized[key] = str(value) + + return normalized + + def _countCoordinatesByTomogramKey(self, storedSet: Dict[str, Any]) -> Dict[str, int]: + counts: Dict[str, int] = {} + + for item in storedSet.get("items") or []: + values = item.get("values") or {} + keys = self._extractCoordinateTomogramKeys(values) + + for key in keys: + counts[key] = counts.get(key, 0) + 1 + + return counts + + def _extractCoordinateTomogramKeys(self, values: Dict[str, Any]) -> List[str]: + candidates = [] + + for suffixes in ( + ["volid", "volumeid", "volumeobjid", "volume_obj_id"], + ["tomoid", "tomogramid", "tomo_id", "tomogram_id"], + ["tsid", "tiltseriesid", "tilt_series_id"], + ["volname", "volumename", "tomoname", "tomogramname"], + ): + value = self._firstValueBySuffix(values, suffixes) + text = self._toTextCandidate(value) + if text: + candidates.append(text) + + for key, value in values.items(): + normalizedKey = self._normalizeKey(key) + if normalizedKey not in { + "volid", + "volumeid", + "volumeobjid", + "tomoid", + "tomogramid", + "tsid", + "tiltseriesid", + "volname", + "volumename", + "tomoname", + "tomogramname", + }: + continue + + text = self._toTextCandidate(value) + if text: + candidates.append(text) + + return self._uniqueStrings(candidates) + + def _findTomogramCount(self, item: Dict[str, Any], countsByKey: Dict[str, int]) -> Optional[int]: + candidates = [ + item.get("id"), + item.get("tomoId"), + item.get("label"), + item.get("name"), + item.get("tsId"), + item.get("tiltSeriesId"), + item.get("objectId"), + item.get("volumeId"), + item.get("sourceTomoId"), + ] + + for candidate in candidates: + if candidate is None: + continue + + key = str(candidate) + if key in countsByKey: + return countsByKey[key] + + return None + + def _getTomogramMatchKeys(self, item: Dict[str, Any]) -> Set[str]: + keys = set() + + for key in ( + "id", + "tomoId", + "label", + "name", + "tsId", + "tiltSeriesId", + "objectId", + "volumeId", + "sourceTomoId", + ): + value = item.get(key) + if value is not None and str(value).strip(): + keys.add(str(value).strip()) + + return keys + + def _extractTomogramId(self, values: Dict[str, Any]) -> Optional[Any]: + value = self._firstValueBySuffix( + values, + [ + "tomoid", + "tomogramid", + "tomoName", + "tomogramName", + "volumeid", + "volid", + "volname", + "tsid", + "tiltseriesid", + ], + ) + return value + + def _extractTomogramLabel(self, values: Dict[str, Any], fallback: str) -> str: + label = self._firstValueBySuffix( + values, + [ + "nameid", + "tomoname", + "tomogramname", + "volumename", + "volname", + "objlabel", + "label", + "name", + ], + ) + + if label: + return str(label) + + fileName = self._extractTomogramFile(values) + if fileName: + return str(fileName).split("/")[-1] + + return str(fallback) + + def _extractTomogramFile(self, values: Dict[str, Any]) -> Optional[Any]: + return self._firstValueBySuffix( + values, + [ + "filename", + "fileName", + "filepath", + "filePath", + "volumefile", + "tomogramfile", + "location", + ], + ) + + def _extractFileBasename(self, values: Dict[str, Any]) -> Optional[str]: + fileName = self._extractTomogramFile(values) + if not fileName: + return None + + try: + return os.path.basename(str(fileName)) + except Exception: + return str(fileName) + + def _extractDims(self, values: Dict[str, Any]) -> Optional[List[int]]: + raw = self._firstValueBySuffix( + values, + [ + "dim", + "dims", + "dimensions", + "volumedim", + "tomogramdim", + "getdim", + ], + ) + + return self._normalizeDims(raw) + + def _extractVoxelSize(self, values: Dict[str, Any]) -> Optional[List[float]]: + raw = self._firstValueBySuffix( + values, + [ + "voxelsize", + "voxel_size", + "samplingrate", + "samplingRate", + "pixelSize", + "pixel_size", + ], + ) + + return self._normalizeVoxelSize(raw) + + def _normalizeDims(self, value: Any) -> Optional[List[int]]: + parsed = self._parseNumberList(value) + if parsed is None or len(parsed) < 3: + return None + + dims = [] + for item in parsed[:3]: + intValue = self._toOptionalInt(item) + if intValue is None or intValue <= 0: + return None + dims.append(intValue) + + return dims + + def _normalizeVoxelSize(self, value: Any) -> Optional[List[float]]: + parsed = self._parseNumberList(value) + if parsed is None: + floatValue = self._toOptionalFloat(value) + if floatValue is None: + return None + return [floatValue, floatValue, floatValue] + + if len(parsed) == 1: + floatValue = self._toOptionalFloat(parsed[0]) + if floatValue is None: + return None + return [floatValue, floatValue, floatValue] + + if len(parsed) >= 3: + voxelSize = [] + for item in parsed[:3]: + floatValue = self._toOptionalFloat(item) + if floatValue is None: + return None + voxelSize.append(floatValue) + return voxelSize + + return None + + def _parseNumberList(self, value: Any) -> Optional[List[Any]]: + if value is None or value == "": + return None + + parsed = self._parseJsonValue(value) + if isinstance(parsed, (list, tuple)): + return list(parsed) + + if isinstance(value, (list, tuple)): + return list(value) + + text = str(value).strip() + if not text: + return None + + numbers = re.findall(r"[-+]?\d*\.?\d+(?:[eE][-+]?\d+)?", text) + if not numbers: + return None + + return numbers + + def _firstValue(self, values: Dict[str, Any], keys: List[str]) -> Any: + for key in keys: + if key in values: + return values.get(key) + return None + + def _firstValueBySuffix(self, values: Dict[str, Any], suffixes: List[str]) -> Any: + normalizedSuffixes = [ + self._normalizeKey(suffix) + for suffix in suffixes + ] + + for key, value in values.items(): + if value is None: + continue + + normalizedKey = self._normalizeKey(key) + for suffix in normalizedSuffixes: + if normalizedKey.endswith(suffix): + return value + + return None + + def _normalizeKey(self, value: Any) -> str: + return str(value).replace("_", "").replace(".", "").replace("-", "").lower() + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except Exception: + try: + return int(float(value)) + except Exception: + return None + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + try: + return float(value) + except Exception: + return None + + def _toTextCandidate(self, value: Any) -> Optional[str]: + if value is None: + return None + + if isinstance(value, dict): + for key in ( + "id", + "objId", + "objectId", + "volumeId", + "volId", + "tomoId", + "tomogramId", + "tsId", + "name", + "label", + ): + if key in value: + text = self._toTextCandidate(value.get(key)) + if text: + return text + return None + + if isinstance(value, (list, tuple)): + if len(value) == 1: + return self._toTextCandidate(value[0]) + return None + + text = str(value).strip() + return text or None + + def _uniqueStrings(self, values: List[str]) -> List[str]: + seen = set() + out = [] + + for value in values: + text = str(value).strip() + if not text or text in seen: + continue + + seen.add(text) + out.append(text) + + return out + + def _parseJsonValue(self, value: Any) -> Any: + if isinstance(value, (dict, list, tuple)): + return value + + if not isinstance(value, str): + return value + + text = value.strip() + if not text: + return value + + if not ( + text.startswith("{") + or text.startswith("[") + or text.startswith('"') + ): + return value + + try: + return json.loads(text) + except Exception: + return value + + def _normalizeJsonObject(self, value: Any) -> Dict[str, Any]: + parsed = self._parseJsonValue(value) + if isinstance(parsed, dict): + return parsed + return {} + + def _resolveTomogramTargetKeys( + self, + storedSet: Dict[str, Any], + tomogramId: Any, + ) -> Set[str]: + requested = self._toTextCandidate(tomogramId) + if not requested: + return set() + + targetKeys = {requested} + + linkedTomograms = self._getLinkedTomogramsFromProperties(storedSet) + for item in linkedTomograms: + normalized = self._normalizeLinkedTomogramItem(item) + if normalized is None: + continue + + matchKeys = self._getTomogramMatchKeys(normalized) + if requested in matchKeys: + targetKeys.update(matchKeys) + + countsByKey = self._countCoordinatesByTomogramKey(storedSet) + coordinateKeys = set(countsByKey.keys()) + + payload = self._buildTomogramPayloadFromProjectSets( + coordinateKeys=coordinateKeys, + countsByKey=countsByKey, + ) + + for item in payload or []: + matchKeys = self._getTomogramMatchKeys(item) + if requested in matchKeys: + targetKeys.update(matchKeys) + + return { + str(value) + for value in targetKeys + if value is not None and str(value).strip() + } + + def _buildCoordinatePoint( + self, + item: Dict[str, Any], + values: Dict[str, Any], + tomogramId: Any, + boxSize: Optional[float], + ) -> Optional[Dict[str, Any]]: + x = self._getCoordinateValue(values, "x") + y = self._getCoordinateValue(values, "y") + z = self._getCoordinateValue(values, "z") + + if x is None or y is None or z is None: + return None + + point: Dict[str, Any] = { + "x": float(x), + "y": float(y), + "z": float(z), + "tomoId": tomogramId, + } + + objId = item.get("scipionItemId") + if objId is not None: + point["id"] = objId + + classId = self._extractPointClassId(values) + if classId is not None: + point["classId"] = classId + + score = self._extractPointScore(values) + if score is not None: + point["score"] = score + + label = item.get("label") or self._firstValueBySuffix(values, ["objlabel", "label"]) + if label not in (None, ""): + point["label"] = str(label) + + matrix = self._extractPointMatrix(values) + if matrix is not None: + point["matrix"] = matrix + else: + point["matrix"] = [] + + if boxSize is not None: + point["radius"] = float(boxSize) + + return point + + def _getCoordinateValue(self, values: Dict[str, Any], axis: str) -> Optional[float]: + normalizedAxis = self._normalizeKey(axis) + + preferredKeys = { + "x": "bottomleftx", + "y": "bottomlefty", + "z": "bottomleftz", + } + + preferredKey = preferredKeys.get(normalizedAxis) + if preferredKey is None: + return None + + for key, value in values.items(): + if self._normalizeKey(key) == preferredKey: + return self._toOptionalFloat(value) + + return None + + def _extractPointClassId(self, values: Dict[str, Any]) -> Optional[Any]: + return self._firstValueBySuffix( + values, + [ + "classid", + "class", + "groupid", + "group", + ], + ) + + def _extractPointScore(self, values: Dict[str, Any]) -> Optional[float]: + raw = self._firstValueBySuffix( + values, + [ + "score", + "weight", + "prob", + "probability", + "confidence", + ], + ) + + return self._toOptionalFloat(raw) + + def _extractPointMatrix(self, values: Dict[str, Any]) -> Optional[Any]: + raw = self._firstValueBySuffix( + values, + [ + "matrix", + "transform", + "transformmatrix", + "transformationmatrix", + ], + ) + + if raw is None: + return None + + parsed = self._parseJsonValue(raw) + if isinstance(parsed, list): + return parsed + + return None + + def _extractBoxSize(self, storedSet: Dict[str, Any]) -> Optional[float]: + properties = self._normalizeJsonObject(storedSet.get("properties")) + + raw = self._firstValueBySuffix( + properties, + [ + "boxsize", + "box_size", + "radius", + ], + ) + + value = self._toOptionalFloat(raw) + if value is not None: + return value + + for item in storedSet.get("setProperties") or []: + key = item.get("key") + if self._normalizeKey(key) not in {"boxsize", "radius"}: + continue + + value = self._toOptionalFloat(item.get("value")) + if value is not None: + return value + + return None + + def getTomogramFile(self, tomogramId: Any) -> Optional[Dict[str, Any]]: + self.lastSkipReason = None + + storedSet = self._getStoredSet() + if storedSet is None: + self.lastSkipReason = "stored_set_not_found" + return None + + requested = self._toTextCandidate(tomogramId) + if not requested: + self.lastSkipReason = "empty_tomogram_id" + return None + + linkedTomograms = self._getLinkedTomogramsFromProperties(storedSet) + for item in linkedTomograms: + normalized = self._normalizeLinkedTomogramItem(item) + if normalized is None: + continue + + matchKeys = self._getTomogramMatchKeys(normalized) + if requested not in matchKeys: + continue + + fileName = normalized.get("fileName") + if not fileName: + self.lastSkipReason = "tomogram_file_not_found_in_linked_metadata tomogramId=%s" % requested + return None + + return { + "id": normalized.get("id"), + "tomoId": normalized.get("tomoId"), + "label": normalized.get("label"), + "name": normalized.get("name"), + "fileName": str(fileName), + "dims": normalized.get("dims"), + "voxelSize": normalized.get("voxelSize"), + } + + payload = self._buildTomogramPayloadFromProjectSets( + coordinateKeys={requested}, + countsByKey={}, + ) + + for item in payload or []: + matchKeys = self._getTomogramMatchKeys(item) + if requested not in matchKeys: + continue + + fileName = item.get("fileName") + if not fileName: + continue + + return { + "id": item.get("id"), + "tomoId": item.get("tomoId"), + "label": item.get("label"), + "name": item.get("name"), + "fileName": str(fileName), + "dims": item.get("dims"), + "voxelSize": item.get("voxelSize"), + } + + self.lastSkipReason = "tomogram_file_not_resolved tomogramId=%s" % requested + return None diff --git a/app/backend/viewers/postgresql_coords3d_tomogram_volume_reader.py b/app/backend/viewers/postgresql_coords3d_tomogram_volume_reader.py new file mode 100644 index 00000000..ef5bc2d0 --- /dev/null +++ b/app/backend/viewers/postgresql_coords3d_tomogram_volume_reader.py @@ -0,0 +1,469 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np + +from app.backend.utils.volume_utils import readVolumeArray3d +from app.backend.viewers.postgresql_coords3d_reader import PostgresqlCoords3dReader + + +class PostgresqlCoords3dTomogramVolumeReader: + """Expose tomograms resolved from a SetOfCoordinates3D as volume-like items.""" + + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = int(projectId) + self.protocolId = int(protocolId) + self.outputName = str(outputName) + self.coordsReader = PostgresqlCoords3dReader( + db=db, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + self.lastSkipReason = None + self._volumes = None + + def hasOutput(self) -> bool: + return bool(self.listVolumes()) + + def listVolumes(self) -> Optional[List[Dict[str, Any]]]: + self.lastSkipReason = None + + if self._volumes is not None: + return self._volumes + + tomograms = self.coordsReader.listTomograms() + if not tomograms: + self.lastSkipReason = getattr(self.coordsReader, "lastSkipReason", None) or "coords3d_tomograms_not_found" + return None + + volumes = [] + + for index, tomogram in enumerate(tomograms): + volume = self._buildVolumeFromTomogram(tomogram, index) + if volume is not None: + volumes.append(volume) + + if not volumes: + self.lastSkipReason = "coords3d_tomograms_without_volume_files" + return None + + self._volumes = volumes + return self._volumes + + def getVolumeInfo(self, volumeId: Union[int, str]) -> Optional[Dict[str, Any]]: + self.lastSkipReason = None + + volumes = self.listVolumes() + if not volumes: + self.lastSkipReason = self.lastSkipReason or "volume_list_empty" + return None + + volume = self._findVolume(volumeId, volumes) + if volume is None: + self.lastSkipReason = "volume_not_found volumeId=%s" % str(volumeId) + return None + + info = dict(volume) + volumeFile = self.getVolumeFile(volume.get("id")) + if volumeFile is None: + return info + + info.update(volumeFile) + self._ensureVolumeInfoFromFile(info) + return info + + def getVolumeFile(self, volumeId: Union[int, str]) -> Optional[Dict[str, Any]]: + volumes = self.listVolumes() + if not volumes: + self.lastSkipReason = self.lastSkipReason or "volume_list_empty" + return None + + volume = self._findVolume(volumeId, volumes) + if volume is None: + self.lastSkipReason = "volume_not_found volumeId=%s" % str(volumeId) + return None + + info = dict(volume) + fileName = info.get("fileName") or info.get("path") + if not fileName: + self.lastSkipReason = "volume_file_not_found volumeId=%s" % str(volumeId) + return None + + resolvedPath = self._resolveExistingPath(fileName) + if resolvedPath is None: + self.lastSkipReason = "volume_file_missing fileName=%s" % str(fileName) + return None + + info["fileName"] = resolvedPath + info["path"] = resolvedPath + return info + + def getVolumeArray( + self, + volumeId: Union[int, str], + ) -> Optional[Tuple[np.ndarray, Dict[str, Any], Dict[str, Any]]]: + info = self.getVolumeFile(volumeId) + if info is None: + return None + + volumePath = info.get("fileName") or info.get("path") + if not volumePath: + self.lastSkipReason = "volume_file_not_found volumeId=%s" % str(volumeId) + return None + + try: + array, props = readVolumeArray3d(str(volumePath)) + except Exception as exc: + self.lastSkipReason = "volume_read_failed volumeId=%s error=%s" % ( + str(volumeId), + str(exc), + ) + return None + + props = props if isinstance(props, dict) else {} + return array, props, info + + def getHistogram( + self, + volumeId: Union[int, str], + bins: int = 128, + ) -> Optional[Dict[str, Any]]: + result = self.getVolumeArray(volumeId) + if result is None: + return None + + array, _props, _info = result + cleanArray = np.asarray(array, dtype=np.float32) + cleanArray = cleanArray[np.isfinite(cleanArray)] + + if cleanArray.size == 0: + return { + "binEdges": [], + "counts": [], + } + + counts, binEdges = np.histogram(cleanArray, bins=max(4, int(bins or 128))) + + return { + "binEdges": [float(value) for value in binEdges.tolist()], + "counts": [int(value) for value in counts.tolist()], + } + + def _buildVolumeFromTomogram( + self, + tomogram: Dict[str, Any], + index: int, + ) -> Optional[Dict[str, Any]]: + fileName, locationIndex = self._extractTomogramFile(tomogram) + if not fileName: + return None + + volumeId = ( + tomogram.get("volumeId") + or tomogram.get("objectId") + or tomogram.get("id") + or tomogram.get("tomoId") + or tomogram.get("label") + or index + ) + + tomoId = tomogram.get("tomoId") or tomogram.get("id") or volumeId + label = tomogram.get("label") or tomogram.get("name") or tomoId or Path(str(fileName)).name + + volume: Dict[str, Any] = { + "id": str(volumeId), + "index": int(index), + "name": str(label), + "label": str(label), + "relPath": str(label), + "tomoId": str(tomoId), + "fileName": str(fileName), + "path": str(fileName), + "source": "coordinates3d", + } + + if locationIndex is not None: + volume["locationIndex"] = locationIndex + + for key in ( + "objectId", + "scipionItemId", + "tsId", + "tiltSeriesId", + "nCoords", + "count", + "sourceTomoId", + ): + value = tomogram.get(key) + if value is not None: + volume[key] = value + + dims = self._normalizeDims(tomogram.get("dims")) + if dims is not None: + volume["dims"] = dims + + voxelSize = self._normalizeVoxelSize(tomogram.get("voxelSize")) + if voxelSize is not None: + volume["voxelSize"] = voxelSize + volume["pixelSize"] = voxelSize[0] + volume["samplingRate"] = voxelSize[0] + + return volume + + def _extractTomogramFile(self, tomogram: Dict[str, Any]) -> Tuple[Optional[str], Optional[int]]: + for key in ( + "fileName", + "filename", + "path", + "tomogramFile", + "tomogramFileName", + "tomoFile", + "tomoFileName", + "volumeFile", + "volumeFileName", + "location", + ): + value = tomogram.get(key) + if value is None: + continue + + fileName, locationIndex = self._parseLocation(value) + if fileName: + return fileName, locationIndex + + return None, None + + def _parseLocation(self, raw: Any) -> Tuple[Optional[str], Optional[int]]: + if isinstance(raw, dict): + pathValue = None + for key in ("fileName", "filename", "path", "location", "stack"): + if key in raw: + pathValue = raw.get(key) + break + + indexValue = None + for key in ("index", "locationIndex", "slice", "itemIndex"): + if key in raw: + indexValue = raw.get(key) + break + + fileName, embeddedIndex = self._parseLocation(pathValue) + locationIndex = self._toOptionalInt(indexValue) + if locationIndex is None: + locationIndex = embeddedIndex + + return fileName, locationIndex + + if isinstance(raw, (list, tuple)): + if len(raw) >= 2: + return self._toText(raw[1]), self._toOptionalInt(raw[0]) + if len(raw) == 1: + return self._parseLocation(raw[0]) + return None, None + + text = self._toText(raw) + if not text: + return None, None + + locationIndex = None + fileName = text + + if "@" in text: + indexText, pathText = text.split("@", 1) + parsedIndex = self._toOptionalInt(indexText) + if parsedIndex is not None: + locationIndex = parsedIndex + fileName = pathText + + return fileName, locationIndex + + def _findVolume( + self, + volumeId: Union[int, str], + volumes: List[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + requested = self._toText(volumeId) + if requested is None: + return None + + for volume in volumes: + for key in ("id", "index", "tomoId", "volumeId", "objectId", "scipionItemId", "name", "label"): + if self._toText(volume.get(key)) == requested: + return volume + + requestedInt = self._toOptionalInt(requested) + if requestedInt is not None and 0 <= requestedInt < len(volumes): + return volumes[requestedInt] + + return None + + def _ensureVolumeInfoFromFile(self, volume: Dict[str, Any]) -> None: + fileName = volume.get("fileName") or volume.get("path") + resolvedPath = self._resolveExistingPath(fileName) + if resolvedPath is None: + return + + volume["fileName"] = resolvedPath + volume["path"] = resolvedPath + + try: + array, props = readVolumeArray3d(resolvedPath) + except Exception: + return + + if getattr(array, "ndim", None) == 3: + zDim, yDim, xDim = array.shape + volume["dims"] = [int(zDim), int(yDim), int(xDim)] + volume["xyzDims"] = [int(xDim), int(yDim), int(zDim)] + + props = props if isinstance(props, dict) else {} + + if "samplingRate" not in volume: + samplingRate = self._extractSamplingRate(props) + if samplingRate is not None: + volume["samplingRate"] = samplingRate + volume["pixelSize"] = samplingRate + volume["voxelSize"] = [samplingRate, samplingRate, samplingRate] + + try: + finite = np.asarray(array, dtype=np.float32) + finite = finite[np.isfinite(finite)] + if finite.size: + volume["min"] = float(np.min(finite)) + volume["max"] = float(np.max(finite)) + volume["mean"] = float(np.mean(finite)) + except Exception: + pass + + def _resolveExistingPath(self, fileName: Any) -> Optional[str]: + text = self._toText(fileName) + if not text: + return None + + path = Path(text).expanduser() + candidates = [] + + if path.is_absolute(): + candidates.append(path) + else: + candidates.append(path) + candidates.append(Path.cwd() / path) + + for candidate in candidates: + try: + resolved = candidate.resolve() + if resolved.exists(): + return str(resolved) + except Exception: + continue + + return None + + def _normalizeDims(self, value: Any) -> Optional[List[int]]: + if isinstance(value, dict): + value = [ + value.get("x") or value.get("X") or value.get("nx"), + value.get("y") or value.get("Y") or value.get("ny"), + value.get("z") or value.get("Z") or value.get("nz"), + ] + + if isinstance(value, (list, tuple)) and len(value) >= 3: + dims = [] + for item in value[:3]: + intValue = self._toOptionalInt(item) + if intValue is None or intValue <= 0: + return None + dims.append(intValue) + return dims + + return None + + def _normalizeVoxelSize(self, value: Any) -> Optional[List[float]]: + if value is None: + return None + + if isinstance(value, dict): + for key in ("x", "X", "samplingRate", "pixelSize", "voxelSize", "apix"): + number = self._toOptionalFloat(value.get(key)) + if number is not None: + return [number, number, number] + return None + + if isinstance(value, (list, tuple)): + if len(value) == 1: + number = self._toOptionalFloat(value[0]) + return [number, number, number] if number is not None else None + + if len(value) >= 3: + voxelSize = [] + for item in value[:3]: + number = self._toOptionalFloat(item) + if number is None: + return None + voxelSize.append(number) + return voxelSize + + number = self._toOptionalFloat(value) + return [number, number, number] if number is not None else None + + def _extractSamplingRate(self, values: Dict[str, Any]) -> Optional[float]: + for key in ("samplingRate", "pixelSize", "voxelSize", "apix"): + number = self._toOptionalFloat(values.get(key)) + if number is not None: + return number + return None + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + + try: + return int(value) + except Exception: + try: + return int(float(value)) + except Exception: + return None + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + + try: + return float(value) + except Exception: + return None + + def _toText(self, value: Any) -> Optional[str]: + if value is None: + return None + + text = str(value).strip() + return text or None \ No newline at end of file diff --git a/app/backend/viewers/postgresql_ctftomo_reader.py b/app/backend/viewers/postgresql_ctftomo_reader.py new file mode 100644 index 00000000..2250f2fb --- /dev/null +++ b/app/backend/viewers/postgresql_ctftomo_reader.py @@ -0,0 +1,639 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from typing import Any, Dict, List, Optional + +from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper +from app.backend.viewers.postgresql_tiltseries_reader import PostgresqlTiltSeriesReader + + +class PostgresqlCtftomoReader: + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + self.setMapper = ScipionSetPostgresqlMapper(db) + self._storedSet = None + self._logicalTables = None + self._associatedTiltSeriesFramesBySeriesId = {} + + def hasOutput(self) -> bool: + return self._getStoredSet() is not None + + def listCtftomoSeries(self) -> List[Dict[str, Any]]: + storedSet = self._getStoredSet() + if storedSet is None: + return [] + + result = [] + for index, item in enumerate(storedSet.get("items") or []): + summary = self._buildCtftomoSeriesSummary(item, index) + result.append(summary) + + return result + + def getCtftomoSeriesViews(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: + seriesItem = self._findCtftomoSeriesItem(tiltSeriesId) + if seriesItem is None: + return None + + summary = self._buildCtftomoSeriesSummary(seriesItem, 0) + childTable = self._findChildTableForParentItem(seriesItem.get("scipionItemId")) + + associatedTiltFrames = self._getAssociatedTiltSeriesFrames( + summary.get("tiltSeriesId") or tiltSeriesId + ) + + frames: List[Dict[str, Any]] = [] + if childTable is not None: + childItems = self.setMapper.getStoredSetTableItems(int(childTable["id"])) + for index, item in enumerate(childItems): + frame = self._buildCtftomoMeasurementFrame(item, index) + tiltFrame = self._findAssociatedTiltSeriesFrame( + ctfFrame=frame, + position=index, + tiltFrames=associatedTiltFrames, + ) + self._mergeTiltSeriesFrameIntoCtftomoFrame(frame, tiltFrame) + frames.append(frame) + + summary["frames"] = frames + summary["tiltSeriesId"] = summary.get("tiltSeriesId") or str(tiltSeriesId) + summary["ctfSeriesId"] = ( + summary.get("ctfSeriesId") + or summary.get("tiltSeriesId") + or str(tiltSeriesId) + ) + summary["nViews"] = len(frames) + + return summary + + def _getStoredSet(self) -> Optional[Dict[str, Any]]: + if self._storedSet is None: + self._storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=self.protocolId, + outputName=self.outputName, + ) + return self._storedSet + + def _getLogicalTables(self) -> List[Dict[str, Any]]: + if self._logicalTables is None: + storedSet = self._getStoredSet() + if storedSet is None: + self._logicalTables = [] + else: + self._logicalTables = self.setMapper.listStoredSetTables( + int(storedSet["id"]) + ) + return self._logicalTables + + def _getAssociatedTiltSeriesFrames(self, tiltSeriesId: Any) -> List[Dict[str, Any]]: + seriesKey = str(tiltSeriesId) + if seriesKey in self._associatedTiltSeriesFramesBySeriesId: + return self._associatedTiltSeriesFramesBySeriesId[seriesKey] + + frames = self._loadAssociatedTiltSeriesFrames(tiltSeriesId) + self._associatedTiltSeriesFramesBySeriesId[seriesKey] = frames + return frames + + def _loadAssociatedTiltSeriesFrames(self, tiltSeriesId: Any) -> List[Dict[str, Any]]: + storedSet = self._getStoredSet() + rootProtocolDbId = storedSet.get("protocolDbId") if storedSet else self.protocolId + + tiltStoredSet = self._findRegularTiltSeriesStoredSetForProtocol(rootProtocolDbId) + if tiltStoredSet is None: + return [] + + protocolDbId = tiltStoredSet.get("protocolDbId") + outputName = tiltStoredSet.get("outputName") + + if protocolDbId is None or not outputName: + return [] + + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + + payload = reader.getTiltSeriesFrames(tiltSeriesId) + if not payload: + return [] + + return payload.get("frames") or [] + + def _findRegularTiltSeriesStoredSetForProtocol( + self, + protocolDbId: Any, + visited: Optional[set] = None, + ) -> Optional[Dict[str, Any]]: + if protocolDbId is None: + return None + + if visited is None: + visited = set() + + protocolKey = str(protocolDbId) + if protocolKey in visited: + return None + + visited.add(protocolKey) + + sameProtocolStoredSet = self._findRegularTiltSeriesStoredSetInProtocol(protocolDbId) + if sameProtocolStoredSet is not None: + return sameProtocolStoredSet + + inputRefs = self._listProtocolInputRefs(protocolDbId) + + for inputRef in inputRefs: + if self._getInputRefKind(inputRef) != "tiltSeries": + continue + + storedSet = self._getStoredSetFromInputRef(inputRef) + if self._isRegularTiltSeriesStoredSet(storedSet): + return storedSet + + for inputRef in inputRefs: + if self._getInputRefKind(inputRef) != "ctf": + continue + + parentProtocolDbId = inputRef.get("parentProtocolDbId") + storedSet = self._findRegularTiltSeriesStoredSetForProtocol( + parentProtocolDbId, + visited=visited, + ) + + if storedSet is not None: + return storedSet + + return None + + def _findRegularTiltSeriesStoredSetInProtocol( + self, + protocolDbId: Any, + ) -> Optional[Dict[str, Any]]: + try: + storedSets = self.setMapper.listProtocolStoredSets( + projectId=self.projectId, + protocolDbId=int(protocolDbId), + ) + except Exception: + return None + + for storedSet in storedSets or []: + storedSetDict = dict(storedSet) + if self._isRegularTiltSeriesStoredSet(storedSetDict): + return storedSetDict + + return None + + def _listProtocolInputRefs(self, protocolDbId: Any) -> List[Dict[str, Any]]: + if protocolDbId is None: + return [] + + try: + rows = self.db.fetchAll( + """ + SELECT + "projectId", + "protocolDbId", + "protocolId", + "inputName", + "itemIndex", + "parentProtocolDbId", + "parentProtocolId", + "parentOutputName", + "objectClassName", + "objectId" + FROM protocol_input_refs + WHERE "projectId" = %s + AND "protocolDbId" = %s + ORDER BY "inputName", "itemIndex" + """, + (self.projectId, int(protocolDbId)), + ) + except Exception: + return [] + + return [dict(row) for row in rows or []] + + def _getInputRefKind(self, inputRef: Dict[str, Any]) -> Optional[str]: + text = self._normalizeClassText(inputRef.get("objectClassName")) + + if "ctftomo" in text: + return "ctf" + + if self._isTiltSeriesMClassText(text): + return "tiltSeriesM" + + if self._isRegularTiltSeriesClassText(text): + return "tiltSeries" + + return None + + def _isRegularTiltSeriesStoredSet(self, storedSet: Optional[Dict[str, Any]]) -> bool: + if storedSet is None: + return False + + classText = self._normalizeClassText( + "%s %s" % ( + storedSet.get("setClassName") or "", + storedSet.get("itemClassName") or "", + ) + ) + + return self._isRegularTiltSeriesClassText(classText) + + def _isRegularTiltSeriesClassText(self, text: Any) -> bool: + classText = self._normalizeClassText(text) + + if self._isTiltSeriesMClassText(classText): + return False + + if "ctftomo" in classText: + return False + + return "tiltseries" in classText + + def _isTiltSeriesMClassText(self, text: Any) -> bool: + classText = self._normalizeClassText(text) + + return ( + "setoftiltseriesm" in classText + or "tiltseriesm" in classText + or "tiltimagem" in classText + or "movie" in classText + or "movies" in classText + ) + + def _normalizeClassText(self, value: Any) -> str: + return str(value or "").replace(" ", "").replace("_", "").replace(".", "").lower() + + def _getStoredSetFromInputRef(self, inputRef: Dict[str, Any]) -> Optional[Dict[str, Any]]: + parentProtocolDbId = inputRef.get("parentProtocolDbId") + if parentProtocolDbId is None: + return None + + for outputName in self._expandInputRefOutputNames(inputRef.get("parentOutputName")): + storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=parentProtocolDbId, + outputName=outputName, + limit=None, + offset=0, + ) + + if storedSet is not None: + return storedSet + + return None + + def _expandInputRefOutputNames(self, outputName: Any) -> List[str]: + outputNameText = str(outputName or "").strip() + if not outputNameText: + return [] + + outputNames = [outputNameText] + + if "." in outputNameText: + outputNames.append(outputNameText.split(".", 1)[0]) + + result = [] + seen = set() + + for item in outputNames: + if item in seen: + continue + + seen.add(item) + result.append(item) + + return result + + def _findAssociatedTiltSeriesFrame( + self, + ctfFrame: Dict[str, Any], + position: int, + tiltFrames: List[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + if not tiltFrames: + return None + + orderKey = self._toTextKey(ctfFrame.get("order")) + if orderKey: + for tiltFrame in tiltFrames: + if self._toTextKey(tiltFrame.get("order")) == orderKey: + return tiltFrame + + if 0 <= position < len(tiltFrames): + return tiltFrames[position] + + return None + + def _mergeTiltSeriesFrameIntoCtftomoFrame( + self, + ctfFrame: Dict[str, Any], + tiltFrame: Optional[Dict[str, Any]], + ) -> None: + if tiltFrame is None: + return + + for key in ("tiltAngle", "dose", "order"): + if ctfFrame.get(key) is None and tiltFrame.get(key) is not None: + ctfFrame[key] = tiltFrame.get(key) + + def _toTextKey(self, value: Any) -> Optional[str]: + if value is None: + return None + + text = str(value).strip() + return text or None + + def _buildCtftomoSeriesSummary(self, item: Dict[str, Any], index: int) -> Dict[str, Any]: + values = item.get("values") or {} + itemId = item.get("scipionItemId") + + tiltSeriesId = self._firstValue( + values, + ["_tsId", "tsId", "tiltSeriesId", "id"], + ) + if tiltSeriesId is None: + tiltSeriesId = item.get("label") or itemId or index + + label = self._firstValueBySuffix( + values, + ["objlabel", "label", "name"], + ) + if label is None: + label = str(item.get("label") or "CTFTomoSeries %s" % str(tiltSeriesId)) + + summary: Dict[str, Any] = { + "ctfSeriesId": str(tiltSeriesId), + "tiltSeriesId": str(tiltSeriesId), + "label": str(label), + "index": index, + } + + childTable = self._findChildTableForParentItem(itemId) + if childTable is not None: + childItems = self.setMapper.getStoredSetTableItems(int(childTable["id"])) + summary["nViews"] = len(childItems) + + dims = self._firstValueBySuffix(values, ["dim", "dims", "dimensions", "getDim"]) + if dims is not None: + summary["dims"] = dims + + pixelSize = self._firstValueBySuffix( + values, + ["samplingrate", "pixelSize", "pixel_size"], + ) + if pixelSize is not None: + summary["pixelSize"] = self._toOptionalFloat(pixelSize) + + tiltAxisAngle = self._firstValueBySuffix(values, ["tiltaxisangle"]) + if tiltAxisAngle is not None: + summary["tiltAxisAngle"] = self._toOptionalFloat(tiltAxisAngle) + + return summary + + def _findCtftomoSeriesItem(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: + storedSet = self._getStoredSet() + if storedSet is None: + return None + + targetKey = str(tiltSeriesId).strip() + if not targetKey: + return None + + for index, item in enumerate(storedSet.get("items") or []): + if targetKey in self._getCtftomoSeriesItemMatchKeys(item, index): + return item + + return None + + def _getCtftomoSeriesItemMatchKeys(self, item: Dict[str, Any], index: int) -> set: + values = item.get("values") or {} + + candidates = [ + self._getTiltSeriesIdFromItem(item, index), + self._firstValue(values, ["_tsId", "tsId", "tiltSeriesId", "ctfSeriesId", "id"]), + self._firstValueBySuffix(values, ["tsid", "tiltseriesid", "ctfseriesid"]), + self._firstValueBySuffix(values, ["objlabel", "label", "name"]), + item.get("label"), + item.get("scipionItemId"), + item.get("id"), + index, + ] + + keys = set() + for candidate in candidates: + if candidate is None: + continue + + text = str(candidate).strip() + if text: + keys.add(text) + + return keys + + def _getTiltSeriesIdFromItem(self, item: Dict[str, Any], index: int) -> Any: + values = item.get("values") or {} + tiltSeriesId = self._firstValue( + values, + ["_tsId", "tsId", "tiltSeriesId", "id"], + ) + + if tiltSeriesId is not None: + return tiltSeriesId + + return item.get("label") or item.get("scipionItemId") or index + + def _buildCtftomoMeasurementFrame(self, item: Dict[str, Any], position: int) -> Dict[str, Any]: + values = item.get("values") or {} + viewId = item.get("scipionItemId") or position + + frame: Dict[str, Any] = { + "viewId": viewId, + "index": position, + "viewIndex": position, + } + + tiltAngle = self._firstValueBySuffix(values, ["tiltangle"]) + tiltAngleFloat = self._toOptionalFloat(tiltAngle) + if tiltAngleFloat is not None: + frame["tiltAngle"] = tiltAngleFloat + + dose = self._firstValueBySuffix(values, ["accumdose", "dose"]) + doseFloat = self._toOptionalFloat(dose) + if doseFloat is not None: + frame["dose"] = doseFloat + + defocusU = self._firstValueBySuffix(values, ["defocusu"]) + defocusUFloat = self._toOptionalFloat(defocusU) + if defocusUFloat is not None: + frame["defocusU"] = defocusUFloat + + defocusV = self._firstValueBySuffix(values, ["defocusv"]) + defocusVFloat = self._toOptionalFloat(defocusV) + if defocusVFloat is not None: + frame["defocusV"] = defocusVFloat + + if defocusUFloat is not None and defocusVFloat is not None: + frame["astigmatism"] = defocusUFloat - defocusVFloat + + defocusAngle = self._firstValueBySuffix(values, ["defocusangle"]) + defocusAngleFloat = self._toOptionalFloat(defocusAngle) + if defocusAngleFloat is not None: + frame["defocusAngle"] = defocusAngleFloat + + resolution = self._firstValueBySuffix(values, ["resolution"]) + resolutionFloat = self._toOptionalFloat(resolution) + if resolutionFloat is not None: + frame["resolution"] = resolutionFloat + + phaseShift = self._firstValueBySuffix(values, ["phaseshift"]) + phaseShiftFloat = self._toOptionalFloat(phaseShift) + if phaseShiftFloat is not None: + frame["phaseShift"] = phaseShiftFloat + + acquisitionOrder = self._firstValueBySuffix( + values, + ["acquisitionorder", "acqorder", "order"], + ) + acquisitionOrderInt = self._toOptionalInt(acquisitionOrder) + if acquisitionOrderInt is not None: + frame["order"] = acquisitionOrderInt + + psdFile = self._firstValueBySuffix( + values, + ["psdfile", "psdfilename", "psdpath"], + ) + if psdFile: + frame["psdFile"] = str(psdFile) + + enabled = self._firstValueBySuffix(values, ["enabled", "isenabled"]) + enabledBool = self._toOptionalBool(enabled) + frame["excluded"] = False if enabledBool is None else not enabledBool + + return frame + + def _findChildTableForParentItem(self, parentItemId: Any) -> Optional[Dict[str, Any]]: + if parentItemId is None: + return None + + for table in self._getLogicalTables(): + if table.get("tableKind") != "child": + continue + if str(table.get("parentItemId")) == str(parentItemId): + return table + + return None + + def _firstValue(self, values: Dict[str, Any], keys: List[str]) -> Any: + for key in keys: + if key in values: + return values.get(key) + return None + + def _firstValueBySuffix(self, values: Dict[str, Any], suffixes: List[str]) -> Any: + normalizedSuffixes = [ + str(suffix).replace("_", "").replace(".", "").lower() + for suffix in suffixes + ] + + for key, value in values.items(): + normalizedKey = str(key).replace("_", "").replace(".", "").lower() + for suffix in normalizedSuffixes: + if normalizedKey.endswith(suffix): + return value + + return None + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + try: + return float(value) + except Exception: + return None + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except Exception: + try: + return int(float(value)) + except Exception: + return None + + def _toOptionalBool(self, value: Any) -> Optional[bool]: + if value is None or value == "": + return None + + if isinstance(value, bool): + return value + + if isinstance(value, (int, float)): + return bool(value) + + text = str(value).strip().lower() + if text in ("1", "true", "yes", "y", "on", "enabled"): + return True + if text in ("0", "false", "no", "n", "off", "disabled"): + return False + + return None + + def _hasCtftomoViewerContract(self, payload: Dict[str, Any]) -> bool: + frames = payload.get("frames") or [] + if not frames: + return False + + if not payload.get("dims"): + return False + + requiredNumericKeys = ( + "tiltAngle", + "defocusU", + "defocusV", + "defocusAngle", + "resolution", + "order", + ) + + for frame in frames: + for key in requiredNumericKeys: + if self._toOptionalFloat(frame.get(key)) is None: + return False + + if not frame.get("psdFile"): + return False + + return True \ No newline at end of file diff --git a/app/backend/viewers/postgresql_dao.py b/app/backend/viewers/postgresql_dao.py new file mode 100644 index 00000000..5d07f420 --- /dev/null +++ b/app/backend/viewers/postgresql_dao.py @@ -0,0 +1,1274 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +# ****************************************************************************** +# * PostgreSQL DAO for persisted Scipion sets. +# * +# * This DAO implements the metadata-viewer DAO contract so PostgreSQL can be +# * consumed through ObjectManager just like SQLite/STAR metadata sources. +# * +# ****************************************************************************** + +import ast +import json +import logging +import re +from typing import Any, Dict, Iterable, List, Optional + + +import numpy + +from metadataviewer.dao.model import IDAO +from metadataviewer.model import ( + Table, + Column, + BoolRenderer, + FloatRenderer, + ImageRenderer, + StrRenderer, + MatrixRender, +) + +from pwem.convert.transformations import euler_from_matrix + +from app.backend.mapper import ScipionSetPostgresqlMapper + + +logger = logging.getLogger(__name__) + +ALLOWED_COLUMNS_TYPES = [ + "String", + "Float", + "Integer", + "Boolean", + "Matrix", + "CsvList", +] + +EXCLUDED_COLUMNS = ["label", "comment", "creation", "_streamState"] +PERMANENT_COLUMNS = ["id", "enabled"] + +OBJECT_TABLE = "objects" +PROPERTIES_TABLE = "Properties" +ENABLED_COLUMN = "enabled" +EXTENDED_COLUMN_NAME = "stack" +PROPERTY_KEY_COLUMN = "key" +PROPERTY_VALUE_COLUMN = "value" +ADITIONAL_INFO_DISPLAY_COLUMN_LIST = ["_size", "id"] + + +def _guessType(value): + if value is None: + return str + + if isinstance(value, bool): + return bool + + if isinstance(value, int): + return int + + if isinstance(value, float): + return float + + try: + int(value) + return int + except Exception: + pass + + try: + float(value) + return float + except Exception: + pass + + return str + + +class ScipionColumn(Column): + def __init__(self, name, renderer=None, callback=None): + super().__init__(name, renderer=renderer) + self.callback = callback + + def setCallback(self, callback): + self.callback = callback + + def calculate(self, row, values): + if self.callback is not None: + self.callback(row, values) + + +class PostgresqlDAO(IDAO): + """ + DAO compatible with metadata-viewer ObjectManager. + + It reads persisted Scipion SetOf... outputs from PostgreSQL using + ScipionSetPostgresqlMapper, but exposes them as metadata-viewer Tables, + Columns and Pages. + """ + + def __init__( + self, + db, + projectId: int, + protocolId: int, + outputName: str, + ): + self.db = db + self.projectId = int(projectId) + self.protocolId = int(protocolId) + self.outputName = str(outputName) + + self.setMapper = ScipionSetPostgresqlMapper(db) + + self._tables: Dict[str, Table] = {} + self._tableCount: Dict[str, int] = {} + self._labels: Dict[str, List[str]] = {} + self._labelsTypes: Dict[str, List[Any]] = {} + self._columns: List[Dict[str, Any]] = [] + self._storedSet: Optional[Dict[str, Any]] = None + + self._logicalTables: Dict[str, Dict[str, Any]] = {} + self._tableColumns: Dict[str, List[Dict[str, Any]]] = {} + self._useLogicalTables = False + self._tableWithAdditionalInfo = None + self._aliases: Dict[str, str] = {} + self._objectsType: Dict[str, str] = {} + + # ------------------------------------------------------------------------- + # metadata-viewer DAO API + # ------------------------------------------------------------------------- + + @classmethod + def getCompatibleFileTypes(cls): + # Kept for compatibility with ObjectManager.selectDAO(), even though + # ScipionWeb injects this DAO directly. + return ["pgset"] + + def hasOutput(self) -> bool: + try: + return self._getStoredSet(limit=0, offset=0) is not None + except Exception: + return False + + def getTables(self): + if self._tables: + return self._tables + + storedSet = self._requireStoredSet(limit=0, offset=0) + setId = int(storedSet["id"]) + + logicalTables = [] + try: + logicalTables = self.setMapper.listStoredSetTables(setId) or [] + except Exception: + logicalTables = [] + + if logicalTables: + self._useLogicalTables = True + + for logicalTable in logicalTables: + tableName = logicalTable.get("name") + if not tableName: + continue + + tableAlias = logicalTable.get("alias") or tableName + + table = Table(tableName) + table.setAlias(tableAlias) + + self._tables[tableName] = table + self._aliases[tableName] = tableAlias + self._logicalTables[tableName] = logicalTable + self._tableColumns[tableName] = self.setMapper.getStoredSetTableColumns( + int(logicalTable["id"]) + ) + + self._addPropertiesTable() + self.composeObjectType() + return self._tables + + tableAlias = storedSet.get("setClassName") or self.outputName + + table = Table(OBJECT_TABLE) + table.setAlias(tableAlias) + + self._tables[OBJECT_TABLE] = table + self._aliases[OBJECT_TABLE] = tableAlias + self._columns = self._normalizeColumns(storedSet.get("columns") or []) + + self._addPropertiesTable() + self.composeObjectType() + return self._tables + + def _addPropertiesTable(self) -> None: + if PROPERTIES_TABLE in self._tables: + return + + table = Table(PROPERTIES_TABLE) + table.setAlias(PROPERTIES_TABLE) + self._tables[PROPERTIES_TABLE] = table + self._aliases[PROPERTIES_TABLE] = PROPERTIES_TABLE + + def fillTable(self, table, objectManager): + tableName = table.getName() + if tableName != OBJECT_TABLE and not self._useLogicalTables: + return + + firstRow = self.getTableRow(tableName, 0) + columns = self._getColumnsForTable(tableName) + + if "id" not in firstRow: + table.setHasColumnId(False) + + labels = [ + key + for key in firstRow.keys() + if key not in EXCLUDED_COLUMNS + ] + + self._labels[tableName] = labels + self._labelsTypes[tableName] = [ + _guessType(firstRow.get(key)) + for key in labels + ] + + imgRenderer = None + computedColsCount = 0 + + for index, colName in enumerate(labels): + value = firstRow.get(colName) + isFileNameCol = imgRenderer is None and colName.endswith("_filename") + columnDef = next( + ( + column + for column in columns + if str(column.get("labelProperty") or "").strip() == str(colName) + ), + None, + ) + columnClassName = self._getColumnClassName(columnDef) + columnClassNameLower = columnClassName.lower() + + if colName == ENABLED_COLUMN or columnClassNameLower == "boolean": + renderer = BoolRenderer() + elif columnClassNameLower == "matrix" or colName.endswith("_matrix") or self._looksLikeMatrix(value): + renderer = MatrixRender() + elif columnClassNameLower == "float": + renderer = FloatRenderer() + elif isFileNameCol: + renderer = StrRenderer() + else: + renderer = table.guessRenderer("" if value is None else str(value)) + + newCol = ScipionColumn(colName, renderer) + newCol.setIsSorteable(True) + + if tableName == OBJECT_TABLE: + newCol.setIsVisible(objectManager.isLabelVisible(colName)) + else: + newCol.setIsVisible(True) + + table.addColumn(newCol) + + if isFileNameCol: + previousCol = labels[index - 1] if index > 0 else "" + if previousCol.endswith("_index"): + extraRenderer = renderer + imageValue = "" if value is None else str(value) + + try: + if imageValue and ImageRenderer().getImageReader(imageValue) is not None: + extraRenderer = ImageRenderer() + imgRenderer = extraRenderer + except Exception: + pass + + extraCol = ScipionColumn(EXTENDED_COLUMN_NAME, extraRenderer) + extraCol.setCallback(self.composeImageFilename) + extraCol.setIsVisible(newCol.isVisible()) + extraCol.setIsSorteable(False) + + table.addColumn(extraCol) + newCol.setIsVisible(False) + computedColsCount += 1 + + elif colName.endswith("_matrix"): + + def addAlignmentColumn(name, offset, position): + extraCol = ScipionColumn(name, renderer=FloatRenderer()) + extraCol.setIsSorteable(False) + extraCol.setIsVisible(newCol.isVisible()) + extraCol.setCallback( + lambda row, values, off=offset, pos=position: + self.extractAngularValue(values, off, pos) + ) + table.addColumn(extraCol) + + def addShiftColumn(name, offset, position): + extraCol = ScipionColumn(name, renderer=FloatRenderer()) + extraCol.setIsVisible(newCol.isVisible()) + extraCol.setIsSorteable(False) + extraCol.setCallback( + lambda row, values, off=offset, pos=position: + self.extractShift(values, off, pos) + ) + table.addColumn(extraCol) + + colNamePrefix = colName.split("_matrix")[0] + + addAlignmentColumn(colNamePrefix + "_rot", -1, 0) + computedColsCount += 1 + + if imgRenderer: + imgRenderer.setRotationColumnIndex(index + computedColsCount) + + addAlignmentColumn(colNamePrefix + "_tilt", -2, 1) + computedColsCount += 1 + + addAlignmentColumn(colNamePrefix + "_psi", -3, 2) + computedColsCount += 1 + + addShiftColumn(colNamePrefix + "_shiftX", -4, 0) + computedColsCount += 1 + + addShiftColumn(colNamePrefix + "_shiftY", -5, 1) + computedColsCount += 1 + + self.generateTableActions(table, objectManager) + + if table.getColumns(): + table.setSortingColumn(table.getColumns()[0].getName()) + + def composeObjectType(self) -> Dict[str, str]: + self._objectsType = {} + + rootAlias = self._getActionAliasForTableName(OBJECT_TABLE) + rootObjectType = self._getRootObjectType() + + if rootAlias and rootObjectType: + self._objectsType[rootAlias] = rootObjectType + + for tableName in list(self._tables.keys()): + if tableName == PROPERTIES_TABLE: + continue + + aliasText = self._getActionAliasForTableName(tableName) + if not aliasText: + continue + + aliasParts = aliasText.rsplit("_", 1) + if len(aliasParts) != 2: + continue + + objectName = aliasParts[1].strip() + if not objectName: + continue + + objectType = self._composeSetObjectTypeFromAliasPart(objectName) + if objectName not in self._objectsType: + self._objectsType[objectName] = objectType + + return self._objectsType + + def _getRootObjectType(self) -> str: + try: + storedSet = self._getStoredSetHeader() + except Exception: + return self.outputName + + setClassName = storedSet.get("setClassName") + if setClassName: + return str(setClassName) + + return self.outputName + + def _getActionAliasForTableName(self, tableName: str) -> str: + if tableName == PROPERTIES_TABLE: + return PROPERTIES_TABLE + + logicalTable = self._logicalTables.get(tableName) + if logicalTable: + itemClassName = str(logicalTable.get("itemClassName") or "").strip() + if itemClassName: + if tableName == OBJECT_TABLE: + return itemClassName + + if tableName.endswith("_Objects"): + prefix = tableName[: -len("_Objects")] + return "%s_%s" % (prefix, itemClassName) + + if tableName == OBJECT_TABLE: + try: + storedSet = self._getStoredSetHeader() + itemClassName = str(storedSet.get("itemClassName") or "").strip() + if itemClassName: + return itemClassName + except Exception: + pass + + return str(self._aliases.get(tableName) or tableName) + + def _composeSetObjectTypeFromAliasPart(self, objectName: str) -> str: + objectName = str(objectName or "").strip() + if not objectName: + return "" + + normalizedName = "ParticlesFlex" if objectName == "ParticleFlex" else objectName + lastChar = normalizedName[-1] + + suffix = ( + "s" + if lastChar in "aeiouAEIOU" or (lastChar != "s" and lastChar != "x") + else "" + ) + + return "SetOf%s%s" % (normalizedName, suffix) + + def generateTableActions(self, table, objectManager) -> None: + if table.getName() == PROPERTIES_TABLE: + return + + self.composeObjectType() + + alias = self._getActionAliasForTableName(table.getName()) + if not alias: + return + + aliasParts = alias.rsplit("_", 1) + + if alias.startswith("Class") and len(aliasParts) == 1: + rootObjectType = self._objectsType.get(alias) + if rootObjectType: + self._addTableAction(table, alias, rootObjectType, objectManager) + + for actionName, objectType in self._objectsType.items(): + if actionName == alias: + continue + + self._addTableAction(table, actionName, objectType, objectManager) + break + + if alias == "Class2D": + self._addTableAction(table, "Averages", "SetOfAverages", objectManager) + elif alias == "Class3D": + self._addTableAction(table, "Volumes", "SetOfVolumes", objectManager) + + elif alias.startswith("Class") and len(aliasParts) > 1: + actionName = aliasParts[1] + objectType = self._objectsType.get(actionName) + if objectType: + self._addTableAction(table, actionName, objectType, objectManager) + elif len(aliasParts) > 1: + actionName = aliasParts[1] + objectType = self._objectsType.get(actionName) + if objectType: + self._addTableAction(table, actionName, objectType, objectManager) + + elif alias in self._objectsType and str(self._objectsType[alias]).startswith("SetOf"): + self._addTableAction(table, alias, self._objectsType[alias], objectManager) + + def _addTableAction(self, table, actionName: str, objectType: str, objectManager) -> None: + actionName = str(actionName or "").strip() + objectType = str(objectType or "").strip() + + if not actionName or not objectType: + return + + try: + for action in table.getActions() or []: + if action.getName() == actionName: + return + except Exception: + pass + + try: + table.addAction( + actionName, + lambda table=table, objectType=objectType, objectManager=objectManager: + self.createSubsetCallback(table, objectType, objectManager), + ) + except Exception: + logger.debug( + "Could not add PostgreSQL metadata action '%s' for object type '%s'", + actionName, + objectType, + exc_info=True, + ) + + def createSubsetCallback(self, table: Table, objectType: str, objectManager) -> bool: + logger.info( + "PostgreSQL metadata subset action requested. table=%s objectType=%s", + table.getName(), + objectType, + ) + return False + + def _getColumnsForTable(self, tableName: str) -> List[Dict[str, Any]]: + if tableName == PROPERTIES_TABLE: + return self._getPropertiesColumns() + + if self._useLogicalTables: + return self._normalizeColumns(self._tableColumns.get(tableName) or []) + + return self._columns + + def _getPropertiesColumns(self) -> List[Dict[str, Any]]: + return [ + { + "labelProperty": PROPERTY_KEY_COLUMN, + "position": 0, + "className": "String", + }, + { + "labelProperty": PROPERTY_VALUE_COLUMN, + "position": 1, + "className": "String", + }, + ] + + def _getLogicalTable(self, tableName: str) -> Optional[Dict[str, Any]]: + if not self._useLogicalTables: + return None + return self._logicalTables.get(tableName) + + def fillPage(self, page, actualColumn: str, orderAsc=True): + table = page.getTable() + tableName = table.getName() + + pageNumber = page.getPageNumber() + pageSize = page.getPageSize() + firstRow = pageNumber * pageSize - pageSize + + columnLabel = actualColumn or "id" + mode = "ASC" if orderAsc else "DESC" + + for rowCount, row in enumerate( + self.iterTable( + tableName, + start=firstRow, + limit=pageSize, + orderBy=columnLabel, + mode=mode, + ) + ): + values = [] + + for column in page.getTable().getColumns(): + if column.isSorteable(): + values.append(row.get(column.getName())) + else: + column.calculate(row, values) + + idValue = row.get("id", firstRow + rowCount + 1) + page.addRow((int(idValue), values)) + + def getTableRow(self, tableName, rowIndex): + rows = self._getRows( + tableName=tableName, + start=max(0, int(rowIndex or 0)), + limit=1, + orderBy="id", + orderAsc=True, + ) + + if rows: + return rows[0] + + return self._emptyRow(tableName) + + def getSelectedRangeRowsIds( + self, + tableName, + startRow, + numberOfRows, + column, + reverse=True, + ): + rows = list( + self.iterTable( + tableName, + start=max(0, int(startRow) - 1), + limit=int(numberOfRows), + orderBy=column or "id", + mode="ASC" if reverse else "DESC", + ) + ) + return [int(row.get("id")) for row in rows if row.get("id") is not None] + + def getColumnsValues( + self, + tableName, + columns, + xAxis, + selection, + limit, + useSelection, + reverse=True, + ): + selectedColumns = list(columns or []) + if xAxis and xAxis not in selectedColumns: + selectedColumns.append(xAxis) + if "id" not in selectedColumns: + selectedColumns.append("id") + + rows = list( + self.iterTable( + tableName, + start=0, + limit=limit, + orderBy=xAxis or "id", + mode="ASC" if reverse else "DESC", + ) + ) + + if useSelection and selection is not None: + try: + selectedIds = set(selection.getSelection().keys()) + rows = [row for row in rows if row.get("id") in selectedIds] + except Exception: + pass + + values = {col: [] for col in selectedColumns} + for row in rows: + for col in selectedColumns: + values[col].append(row.get(col)) + + return values + + def getTableWithAdditionalInfo(self): + return self._tableWithAdditionalInfo, ADITIONAL_INFO_DISPLAY_COLUMN_LIST + + def close(self): + # Do not close the shared PostgreSQL connection here. + pass + + # ------------------------------------------------------------------------- + # Row/table helpers + # ------------------------------------------------------------------------- + + def iterTable(self, tableName, **kwargs): + start = max(0, int(kwargs.get("start", 0) or 0)) + limit = kwargs.get("limit", None) + limit = int(limit) if limit is not None else None + + orderBy = kwargs.get("orderBy") or "id" + mode = str(kwargs.get("mode") or "ASC").upper() + orderAsc = mode != "DESC" + + rows = self._getRows( + tableName=tableName, + start=start, + limit=limit, + orderBy=orderBy, + orderAsc=orderAsc, + ) + + for row in rows: + yield row + + def composeImageFilename(self, row, values): + if len(values) < 2: + values.append("") + return + + indexValue = values[-2] + filenameValue = values[-1] + + if filenameValue is None: + values.append("") + return + + indexStr = "" + try: + if int(indexValue) != 0: + indexStr = "%s@" % indexValue + except Exception: + if indexValue not in (None, "", 0, "0"): + indexStr = "%s@" % indexValue + + values.append("%s%s" % (indexStr, filenameValue)) + + def extractAngularValue(self, values, offset, position): + matrix = values[offset] + matrix = self._toNumpyMatrix(matrix) + + matrixI = numpy.linalg.inv(matrix) + eulerData = euler_from_matrix(matrix=matrixI, axes="szyz") + values.append(numpy.rad2deg(eulerData[position])) + + def extractShift(self, values, offset, position): + matrix = values[offset] + matrix = self._toNumpyMatrix(matrix) + + shape = matrix.shape[0] + values.append(matrix[position, shape - 1]) + + # ------------------------------------------------------------------------- + # PostgreSQL-backed loading + # ------------------------------------------------------------------------- + + def _getStoredSet( + self, + limit: Optional[int], + offset: int, + ) -> Optional[Dict[str, Any]]: + return self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=self.protocolId, + outputName=self.outputName, + limit=limit, + offset=offset, + ) + + def _requireStoredSet( + self, + limit: Optional[int], + offset: int, + ) -> Dict[str, Any]: + storedSet = self._getStoredSet(limit=limit, offset=offset) + if storedSet is None: + raise ValueError( + "Persisted Scipion set was not found: projectId=%s protocolId=%s outputName=%s" + % (self.projectId, self.protocolId, self.outputName) + ) + return storedSet + + def _getStoredSetHeader(self) -> Dict[str, Any]: + if self._storedSet is None: + self._storedSet = self._requireStoredSet(limit=0, offset=0) + self._columns = self._normalizeColumns(self._storedSet.get("columns") or []) + return self._storedSet + + def _getRows( + self, + tableName: str, + start: int, + limit: Optional[int], + orderBy: str, + orderAsc: bool, + ) -> List[Dict[str, Any]]: + orderBy = str(orderBy or "id") + + if tableName == PROPERTIES_TABLE: + rows = self._getPropertiesRows() + rows.sort( + key=lambda row: self._sortValue(row.get(orderBy)), + reverse=not orderAsc, + ) + + if limit is None: + return rows[start:] + + return rows[start:start + limit] + + if self._useLogicalTables: + logicalTable = self._getLogicalTable(tableName) + if logicalTable is None: + return [] + + tableId = int(logicalTable["id"]) + columns = self._getColumnsForTable(tableName) + + canUsePagedRead = orderAsc and orderBy in ("id", "_objId", "SCIPION_OBJECT_ID") + + if canUsePagedRead: + items = self.setMapper.getStoredSetTableItems( + tableId=tableId, + limit=limit, + offset=start, + ) + return [self._itemToRow(item, columns) for item in items] + + items = self.setMapper.getStoredSetTableItems( + tableId=tableId, + limit=None, + offset=0, + ) + rows = [self._itemToRow(item, columns) for item in items] + rows.sort( + key=lambda row: self._sortValue(row.get(orderBy)), + reverse=not orderAsc, + ) + + if limit is None: + return rows[start:] + + return rows[start:start + limit] + + canUsePagedRead = orderAsc and orderBy in ("id", "_objId", "SCIPION_OBJECT_ID") + + if canUsePagedRead: + storedSet = self._requireStoredSet(limit=limit, offset=start) + items = storedSet.get("items") or [] + self._columns = self._normalizeColumns(storedSet.get("columns") or self._columns) + return [self._itemToRow(item, self._columns) for item in items] + + storedSet = self._requireStoredSet(limit=None, offset=0) + items = storedSet.get("items") or [] + self._columns = self._normalizeColumns(storedSet.get("columns") or self._columns) + + rows = [self._itemToRow(item, self._columns) for item in items] + rows.sort( + key=lambda row: self._sortValue(row.get(orderBy)), + reverse=not orderAsc, + ) + + if limit is None: + return rows[start:] + + return rows[start:start + limit] + + def _itemToRow( + self, + item: Dict[str, Any], + columns: List[Dict[str, Any]], + ) -> Dict[str, Any]: + values = item.get("values") or {} + if not isinstance(values, dict): + values = {} + + itemId = item.get("scipionItemId") + if itemId is None: + itemId = item.get("id") + + row: Dict[str, Any] = { + "id": itemId, + "enabled": self._toBool(item.get("enabled", True), default=True), + } + + for column in columns: + label = column.get("labelProperty") + if not label: + continue + + value = values.get(label) + if value is None: + value = values.get(str(label).strip()) + + row[str(label)] = self._normalizeValue(str(label), value, column) + + return row + + def _emptyRow(self, tableName: str) -> Dict[str, Any]: + if tableName == PROPERTIES_TABLE: + return { + PROPERTY_KEY_COLUMN: "", + PROPERTY_VALUE_COLUMN: "", + } + + row = { + "id": 0, + "enabled": True, + } + + for column in self._getColumnsForTable(tableName): + label = column.get("labelProperty") + if label: + row[str(label)] = None + + return row + + def getTableRowCount(self, tableName): + if tableName not in self._tableCount: + if tableName == PROPERTIES_TABLE: + self._tableCount[tableName] = len(self._getPropertiesRows()) + elif self._useLogicalTables: + logicalTable = self._getLogicalTable(tableName) + if logicalTable is None: + self._tableCount[tableName] = 0 + else: + self._tableCount[tableName] = self._getStoredSetTableItemsCount( + int(logicalTable["id"]) + ) + else: + self._tableCount[tableName] = self._getStoredSetItemsCount() + + return self._tableCount[tableName] + + def _getStoredSetTableItemsCount(self, tableId: int) -> int: + row = self.db.fetchOne( + """ + SELECT COUNT(*) AS count + FROM scipion_set_table_items + WHERE "tableId" = %s + """, + (tableId,), + ) + if not row: + return 0 + return int(row.get("count") or 0) + + # ------------------------------------------------------------------------- + # Metadata helpers + # ------------------------------------------------------------------------- + + def _getPropertiesRows(self) -> List[Dict[str, Any]]: + storedSet = self._getStoredSetHeader() + + rows: List[Dict[str, Any]] = [] + seen = set() + + self._addPropertyRow( + rows, + seen, + "self", + storedSet.get("setClassName") or self.outputName, + ) + + self._addPropertyRow( + rows, + seen, + "_size", + self._getStoredSetItemsCount(), + ) + + for prop in storedSet.get("setProperties") or []: + if not isinstance(prop, dict): + continue + + self._addPropertyRow( + rows, + seen, + prop.get("key"), + prop.get("value"), + ) + + properties = storedSet.get("properties") or {} + if isinstance(properties, dict): + for key in sorted(properties.keys()): + self._addPropertyRow( + rows, + seen, + key, + properties.get(key), + ) + + return rows + + def _addPropertyRow( + self, + rows: List[Dict[str, Any]], + seen: set, + key: Any, + value: Any, + ) -> None: + keyText = str(key or "").strip() + if not keyText or keyText in seen: + return + + seen.add(keyText) + rows.append( + { + PROPERTY_KEY_COLUMN: keyText, + PROPERTY_VALUE_COLUMN: self._normalizePropertyValue(value), + } + ) + + def _normalizePropertyValue(self, value: Any) -> str: + if value is None: + return "" + + if isinstance(value, numpy.ndarray): + value = value.tolist() + + if isinstance(value, (dict, list, tuple)): + try: + return json.dumps(value, sort_keys=True, default=str) + except Exception: + return str(value) + + return str(value) + + def _normalizeColumns(self, columns: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + def sortKey(column: Dict[str, Any]): + try: + return int(column.get("position") or 0) + except Exception: + return 0 + + return sorted(columns or [], key=sortKey) + + def _getStoredSetItemsCount(self) -> int: + storedSet = self._getStoredSetHeader() + properties = storedSet.get("properties") or {} + + if isinstance(properties, dict): + count = self._toInt(properties.get("itemsCount")) + if count is not None: + return count + + for prop in storedSet.get("setProperties") or []: + if str(prop.get("key")) == "itemsCount": + count = self._toInt(prop.get("value")) + if count is not None: + return count + + return 0 + + @staticmethod + def _getColumnClassName(column: Optional[Dict[str, Any]]) -> str: + if not isinstance(column, dict): + return "" + + for key in ("className", "type", "columnType"): + value = column.get(key) + if value is not None: + return str(value).strip() + + return "" + + @staticmethod + def _isBooleanString(value: Any) -> bool: + if not isinstance(value, str): + return False + + return value.strip().lower() in { + "true", + "false", + "t", + "f", + "yes", + "no", + "y", + "n", + "1", + "0", + "on", + "off", + } + + @staticmethod + def _toBool(value: Any, default: bool = False) -> bool: + if isinstance(value, bool): + return value + + if value is None: + return default + + if isinstance(value, (int, float)): + return bool(value) + + if isinstance(value, str): + text = value.strip().lower() + if text in {"true", "t", "yes", "y", "1", "on"}: + return True + if text in {"false", "f", "no", "n", "0", "off"}: + return False + + return default + + @staticmethod + def _toFloat(value: Any) -> Optional[float]: + if value is None or value == "": + return None + + try: + return float(value) + except Exception: + return None + + @staticmethod + def _looksLikeMatrix(value: Any) -> bool: + if isinstance(value, numpy.ndarray): + return value.ndim == 2 + + if isinstance(value, (list, tuple)): + try: + return numpy.asarray(value).ndim == 2 + except Exception: + return False + + if not isinstance(value, str): + return False + + text = value.strip() + return text.startswith("[[") and text.endswith("]]") + + def _normalizeValue( + self, + label: str, + value: Any, + column: Optional[Dict[str, Any]] = None, + ) -> Any: + if value is None: + return None + + className = self._getColumnClassName(column) + classNameLower = className.lower() + + if classNameLower == "boolean" or label == ENABLED_COLUMN: + return self._toBool(value) + + if classNameLower == "integer": + parsed = self._toInt(value) + return parsed if parsed is not None else value + + if classNameLower == "float": + parsed = self._toFloat(value) + return parsed if parsed is not None else value + + if classNameLower == "matrix" or label.endswith("_matrix") or self._looksLikeMatrix(value): + return self._toNumpyMatrix(value) + + if classNameLower == "csvlist": + return self._toListValue(value) + + if isinstance(value, str): + if self._isBooleanString(value): + return self._toBool(value) + + return value + + if isinstance(value, (int, float, bool)): + return value + + if isinstance(value, list): + return [ + self._normalizeValue(label, item, column) + for item in value + ] + + if isinstance(value, tuple): + return [ + self._normalizeValue(label, item, column) + for item in value + ] + + if isinstance(value, dict): + return { + str(key): self._normalizeValue(label, item, column) + for key, item in value.items() + } + + return str(value) + + def _toListValue(self, value: Any) -> List[Any]: + if value is None: + return [] + + if isinstance(value, numpy.ndarray): + return value.tolist() + + if isinstance(value, list): + return value + + if isinstance(value, tuple): + return list(value) + + if isinstance(value, str): + text = value.strip() + if not text: + return [] + + try: + parsed = json.loads(text) + if isinstance(parsed, list): + return parsed + except Exception: + pass + + try: + parsed = ast.literal_eval(text) + if isinstance(parsed, (list, tuple)): + return list(parsed) + except Exception: + pass + + return [item.strip() for item in text.split(",") if item.strip()] + + return [value] + + def _toNumpyMatrix(self, value: Any): + if isinstance(value, numpy.ndarray): + return value + + if isinstance(value, str): + text = value.strip() + if not text: + return numpy.array([]) + + parsed = None + + try: + parsed = json.loads(text) + except Exception: + parsed = None + + if parsed is None: + try: + parsed = ast.literal_eval(text) + except Exception: + parsed = None + + if parsed is None: + rowTexts = re.findall(r"\[([^\[\]]+)\]", text) + rows = [] + + for rowText in rowTexts: + numbers = re.findall( + r"[-+]?(?:\d+\.\d*|\.\d+|\d+)(?:[eE][-+]?\d+)?", + rowText, + ) + if numbers: + rows.append([float(number) for number in numbers]) + + parsed = rows + + try: + return numpy.asarray(parsed, dtype=float) + except Exception: + return numpy.array([]) + + try: + return numpy.asarray(value, dtype=float) + except Exception: + return numpy.array([]) + + def _sortValue(self, value: Any): + if value is None: + return "" + + if isinstance(value, numpy.ndarray): + return str(value.tolist()) + + if isinstance(value, (list, tuple, dict)): + return str(value) + + return value + + def _toInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + + try: + return int(value) + except Exception: + return None + + +# Backward-compatible alias while project_service.py is migrated. +PostgresqlMetadataDAO = PostgresqlDAO \ No newline at end of file diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py new file mode 100644 index 00000000..d68e5bdb --- /dev/null +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -0,0 +1,1600 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +import json +import logging +from datetime import date, datetime +from typing import Any, Dict, List, Optional + +from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper +from app.backend.viewers.postgresql_coords3d_reader import PostgresqlCoords3dReader +from app.backend.viewers.postgresql_ctftomo_reader import PostgresqlCtftomoReader +from app.backend.viewers.postgresql_tiltseries_reader import PostgresqlTiltSeriesReader + +logger = logging.getLogger(__name__) + + +class PostgresqlIntegratedContextReader: + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + self.setMapper = ScipionSetPostgresqlMapper(db) + self._rootStoredSet = None + self.lastSkipReason = None + + def hasOutput(self) -> bool: + storedSet = self._getRootStoredSet() + return storedSet is not None and self._getIntegratedKind(storedSet) is not None + + def getContext(self) -> Optional[Dict[str, Any]]: + self.lastSkipReason = None + + storedSet = self._getRootStoredSet() + if storedSet is None: + self.lastSkipReason = "stored_set_not_found" + return None + + rootKind = self._getIntegratedKind(storedSet) + if rootKind is None: + self.lastSkipReason = "unsupported_integrated_context_kind" + return None + + links = { + "tiltSeries": None, + "ctf": None, + "tomogram": None, + "coordinates3d": None, + } + summaries = { + "tiltSeries": None, + "ctf": None, + "tomogram": None, + "coordinates3d": None, + } + relationsByKey: Dict[str, Dict[str, Any]] = {} + + links[rootKind] = self._buildLink( + protocolId=self.protocolId, + outputName=self.outputName, + storedSet=storedSet, + statusValue="available", + ) + summaries[rootKind] = self._buildSummary(storedSet) + + self._mergeRootRelations( + rootKind=rootKind, + storedSet=storedSet, + relationsByKey=relationsByKey, + links=links, + summaries=summaries, + ) + + if rootKind == "tomogram": + self._mergeRootTomogramRelationsFromInputRefs( + rootStoredSet=storedSet, + relationsByKey=relationsByKey, + ) + + self._mergeExactInputRefs( + rootKind=rootKind, + rootStoredSet=storedSet, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + self._mergeRelatedStoredSets( + rootStoredSet=storedSet, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + return { + "root": { + "projectId": self.projectId, + "protocolId": self.protocolId, + "outputName": self.outputName, + "outputClass": storedSet.get("setClassName") or storedSet.get("itemClassName"), + }, + "links": self._safeValue(links), + "summaries": self._safeValue(summaries), + "relations": self._safeValue( + { + "items": list(relationsByKey.values()), + } + ), + } + + def _getRootStoredSet(self) -> Optional[Dict[str, Any]]: + if self._rootStoredSet is None: + try: + self._rootStoredSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=self.protocolId, + outputName=self.outputName, + limit=None, + offset=0, + ) + except Exception: + logger.debug( + "Could not load PostgreSQL root stored set for integrated context. " + "projectId=%s protocolId=%s outputName=%s", + self.projectId, + self.protocolId, + self.outputName, + exc_info=True, + ) + self._rootStoredSet = None + + return self._rootStoredSet + + def _getIntegratedKind(self, storedSet: Dict[str, Any]) -> Optional[str]: + classText = "%s %s" % ( + storedSet.get("setClassName") or "", + storedSet.get("itemClassName") or "", + ) + + return self._getIntegratedKindFromText(classText) + + def _getIntegratedKindFromText(self, classText: Any) -> Optional[str]: + text = str(classText or "").replace(" ", "").lower() + + if "setofcoordinates3d" in text or "coordinate3d" in text: + return "coordinates3d" + + if "setofctftomoseries" in text or "ctftomoseries" in text: + return "ctf" + + if "setoftiltseries" in text or "tiltseries" in text: + return "tiltSeries" + + if "setoftomograms" in text or "tomogram" in text: + return "tomogram" + + return None + + def _normalizeClassText(self, value: Any) -> str: + return str(value or "").replace(" ", "").replace("_", "").replace(".", "").lower() + + def _isTiltSeriesMClassText(self, value: Any) -> bool: + text = self._normalizeClassText(value) + + return ( + "setoftiltseriesm" in text + or "tiltseriesm" in text + or "tiltimagem" in text + or "movie" in text + or "movies" in text + ) + + def _isRegularTiltSeriesStoredSet(self, storedSet: Optional[Dict[str, Any]]) -> bool: + if storedSet is None: + return False + + text = self._normalizeClassText( + "%s %s" % ( + storedSet.get("setClassName") or "", + storedSet.get("itemClassName") or "", + ) + ) + + if self._isTiltSeriesMClassText(text): + return False + + if "ctftomo" in text: + return False + + return "tiltseries" in text + + def _getStoredSetProperty( + self, + storedSet: Dict[str, Any], + key: str, + default=None, + ): + properties = storedSet.get("properties") or {} + + if isinstance(properties, str): + try: + properties = json.loads(properties) + except Exception: + properties = {} + + if isinstance(properties, dict) and key in properties: + return properties.get(key) + + for item in storedSet.get("setProperties") or []: + if str(item.get("key")) == str(key): + return item.get("value") + + return default + + def _buildLink( + self, + protocolId: Any, + outputName: Any, + storedSet: Dict[str, Any], + label: Optional[str] = None, + statusValue: str = "available", + ) -> Dict[str, Any]: + link = { + "protocolId": protocolId, + "outputName": outputName, + "itemId": storedSet.get("objectId") or storedSet.get("id"), + "label": label or outputName, + "status": statusValue, + } + + publicProtocolId = storedSet.get("publicProtocolId") or storedSet.get("protocolId") + if publicProtocolId is not None and str(publicProtocolId) != str(protocolId): + link["publicProtocolId"] = publicProtocolId + + return link + + def _buildSummary( + self, + storedSet: Dict[str, Any], + extra: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + items = storedSet.get("items") or [] + itemsCount = self._getStoredSetProperty(storedSet, "itemsCount", None) + + if itemsCount is None: + itemsCount = len(items) + + summary = { + "objectClass": storedSet.get("setClassName") or storedSet.get("itemClassName"), + "objectId": storedSet.get("objectId") or storedSet.get("id"), + "size": itemsCount, + "fileName": self._getStoredSetProperty(storedSet, "fileName", None), + "samplingRate": self._getStoredSetProperty(storedSet, "samplingRate", None), + } + + if extra: + summary.update(extra) + + return summary + + def _listProtocolInputRefs(self, protocolDbId: Any) -> List[Dict[str, Any]]: + if protocolDbId is None: + return [] + + try: + rows = self.db.fetchAll( + """ + SELECT + "projectId", + "protocolDbId", + "protocolId", + "inputName", + "itemIndex", + "parentProtocolDbId", + "parentProtocolId", + "parentOutputName", + "objectClassName", + "objectId", + "createdAt", + "updatedAt" + FROM protocol_input_refs + WHERE "projectId" = %s + AND "protocolDbId" = %s + ORDER BY "inputName", "itemIndex" + """, + (self.projectId, int(protocolDbId)), + ) + except Exception: + logger.debug( + "Could not list protocol input refs. projectId=%s protocolDbId=%s", + self.projectId, + protocolDbId, + exc_info=True, + ) + return [] + + return [dict(row) for row in rows or []] + + def _getInputRefKind(self, inputRef: Dict[str, Any]) -> Optional[str]: + text = self._normalizeClassText(inputRef.get("objectClassName")) + + if "ctftomo" in text: + return "ctf" + + if self._isTiltSeriesMClassText(text): + return "tiltSeriesM" + + if self._isRegularTiltSeriesClassText(text): + return "tiltSeries" + + if "setoftomograms" in text or "tomogram" in text: + return "tomogram" + + if "setofcoordinates3d" in text or "coordinate3d" in text: + return "coordinates3d" + + return None + + def _isRegularTiltSeriesClassText(self, value: Any) -> bool: + text = self._normalizeClassText(value) + + if self._isTiltSeriesMClassText(text): + return False + + if "ctftomo" in text: + return False + + return "tiltseries" in text + + def _findInputRefByKind( + self, + inputRefs: List[Dict[str, Any]], + kind: str, + ) -> Optional[Dict[str, Any]]: + for inputRef in inputRefs or []: + if self._getInputRefKind(inputRef) == kind: + return inputRef + + return None + + def _expandInputRefOutputNames(self, outputName: Any) -> List[str]: + outputNameText = str(outputName or "").strip() + if not outputNameText: + return [] + + outputNames = [outputNameText] + + if "." in outputNameText: + outputNames.append(outputNameText.split(".", 1)[0]) + + result = [] + seen = set() + + for item in outputNames: + if item in seen: + continue + + seen.add(item) + result.append(item) + + return result + + def _getStoredSetFromInputRef(self, inputRef: Dict[str, Any]) -> Optional[Dict[str, Any]]: + parentProtocolDbId = inputRef.get("parentProtocolDbId") + if parentProtocolDbId is None: + return None + + for outputName in self._expandInputRefOutputNames(inputRef.get("parentOutputName")): + storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=parentProtocolDbId, + outputName=outputName, + limit=None, + offset=0, + ) + + if storedSet is not None: + return storedSet + + return None + + def _buildInputRefLink( + self, + inputRef: Dict[str, Any], + storedSet: Dict[str, Any], + ) -> Dict[str, Any]: + return { + "protocolId": inputRef.get("parentProtocolDbId") or inputRef.get("parentProtocolId"), + "publicProtocolId": inputRef.get("parentProtocolId"), + "outputName": storedSet.get("outputName") or inputRef.get("parentOutputName"), + "itemId": storedSet.get("objectId") or storedSet.get("id") or inputRef.get("objectId"), + "label": inputRef.get("inputName") or inputRef.get("parentOutputName"), + "status": "inferred", + "source": "inputRef", + } + + def _getRelationKeySet(self, relationsByKey: Dict[str, Dict[str, Any]]) -> Optional[set]: + keys = set() + + for key, relation in (relationsByKey or {}).items(): + candidates = [ + key, + relation.get("key"), + relation.get("label"), + relation.get("tiltSeriesId"), + relation.get("tsId"), + relation.get("ctfSeriesId"), + relation.get("tomogramId"), + relation.get("sourceTomoId"), + relation.get("coordinatesTomogramId"), + ] + + for value in candidates: + text = str(value).strip() if value is not None else "" + if text: + keys.add(text) + + return keys or None + + def _mergeInputRefRelations( + self, + kind: str, + inputRef: Dict[str, Any], + storedSet: Dict[str, Any], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> None: + protocolDbId = storedSet.get("protocolDbId") or inputRef.get("parentProtocolDbId") + outputName = storedSet.get("outputName") or inputRef.get("parentOutputName") + + if protocolDbId is None or not outputName: + return + + allowedRelationKeys = self._getRelationKeySet(relationsByKey) + + if kind == "tiltSeries": + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + items = self._filterIntegratedItemsByAllowedKeys( + reader.listTiltSeries(), + allowedRelationKeys, + ) + self._addTiltSeriesRelations(relationsByKey, items) + return + + if kind == "ctf": + reader = PostgresqlCtftomoReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + items = self._filterIntegratedItemsByAllowedKeys( + reader.listCtftomoSeries(), + allowedRelationKeys, + ) + self._addCtftomoRelations(relationsByKey, items) + return + + if kind == "tomogram": + items = self._filterIntegratedItemsByAllowedKeys( + self._buildTomogramItemsFromStoredSet(storedSet), + allowedRelationKeys, + ) + self._addTomogramRelations(relationsByKey, items) + return + + if kind == "coordinates3d": + reader = PostgresqlCoords3dReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + items = self._filterIntegratedItemsByAllowedKeys( + reader.listTomograms() or [], + allowedRelationKeys, + ) + self._addCoordinates3dRelations(relationsByKey, items) + + def _listRegularTiltSeriesItemsForProtocol( + self, + protocolDbId: Any, + ) -> List[Dict[str, Any]]: + storedSet = self._findRegularTiltSeriesStoredSetForProtocol(protocolDbId) + if storedSet is None: + return [] + + protocolDbId = storedSet.get("protocolDbId") + outputName = storedSet.get("outputName") + + if protocolDbId is None or not outputName: + return [] + + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + + return reader.listTiltSeries() or [] + + def _listTiltSeriesItemsFromInputRefs( + self, + inputRefs: List[Dict[str, Any]], + ) -> List[Dict[str, Any]]: + tiltRef = self._findInputRefByKind(inputRefs, "tiltSeries") + + if tiltRef is None: + ctfRef = self._findInputRefByKind(inputRefs, "ctf") + if ctfRef is not None: + ctfInputRefs = self._listProtocolInputRefs( + ctfRef.get("parentProtocolDbId") + ) + tiltRef = self._findInputRefByKind(ctfInputRefs, "tiltSeries") + + if tiltRef is None: + return [] + + storedSet = self._getStoredSetFromInputRef(tiltRef) + if storedSet is None: + return [] + + protocolDbId = storedSet.get("protocolDbId") or tiltRef.get("parentProtocolDbId") + outputName = storedSet.get("outputName") or tiltRef.get("parentOutputName") + + if protocolDbId is None or not outputName: + return [] + + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + + return reader.listTiltSeries() or [] + + def _getItemMatchKeys( + self, + item: Dict[str, Any], + fields: List[str], + ) -> List[str]: + result = [] + seen = set() + + for field in fields: + text = self._toRelationMatchText((item or {}).get(field)) + if not text or text in seen: + continue + + seen.add(text) + result.append(text) + + return result + + def _getTiltSeriesItemMatchKeys(self, item: Dict[str, Any]) -> List[str]: + return self._getItemMatchKeys( + item, + [ + "tiltSeriesId", + "tsId", + "id", + "label", + "name", + ], + ) + + def _getTomogramItemMatchKeys(self, item: Dict[str, Any]) -> List[str]: + return self._getItemMatchKeys( + item, + [ + "tiltSeriesId", + "tsId", + ], + ) + + def _buildItemMapByMatchKeys( + self, + items: List[Dict[str, Any]], + keyGetter, + ) -> Dict[str, Dict[str, Any]]: + result = {} + + for item in items or []: + for key in keyGetter(item): + result.setdefault(key, item) + + return result + + def _findTiltSeriesItemForTomogram( + self, + tomogramItem: Dict[str, Any], + tiltSeriesByKey: Dict[str, Dict[str, Any]], + tiltSeriesItems: List[Dict[str, Any]], + index: int, + ) -> Optional[Dict[str, Any]]: + for key in self._getTomogramItemMatchKeys(tomogramItem): + match = tiltSeriesByKey.get(key) + if match is not None: + return match + + return None + + def _mergeRootTomogramRelationsFromInputRefs( + self, + rootStoredSet: Dict[str, Any], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> None: + rootProtocolDbId = rootStoredSet.get("protocolDbId") + if rootProtocolDbId is None: + return + + tiltSeriesItems = self._listRegularTiltSeriesItemsForProtocol(rootProtocolDbId) + if not tiltSeriesItems: + return + + tiltSeriesByKey = self._buildItemMapByMatchKeys( + tiltSeriesItems, + self._getTiltSeriesItemMatchKeys, + ) + + tomogramItems = self._buildTomogramItemsFromStoredSet(rootStoredSet) + + for index, tomogramItem in enumerate(tomogramItems): + tiltSeriesItem = self._findTiltSeriesItemForTomogram( + tomogramItem=tomogramItem, + tiltSeriesByKey=tiltSeriesByKey, + tiltSeriesItems=tiltSeriesItems, + index=index, + ) + + if tiltSeriesItem is None: + continue + + tiltSeriesId = ( + tiltSeriesItem.get("tiltSeriesId") + or tiltSeriesItem.get("tsId") + or tiltSeriesItem.get("id") + or tiltSeriesItem.get("label") + ) + + if tiltSeriesId is None: + continue + + tomogramId = ( + tomogramItem.get("tomoId") + or tomogramItem.get("tomogramId") + or tomogramItem.get("id") + or tomogramItem.get("label") + or index + ) + + volumeId = ( + tomogramItem.get("tomogramVolumeId") + if tomogramItem.get("tomogramVolumeId") is not None + else tomogramItem.get("volumeId") + ) + + if volumeId is None: + volumeId = index + + label = ( + tomogramItem.get("label") + or tomogramItem.get("name") + or str(tomogramId) + ) + + self._addRelation( + relationsByKey, + tomogramId, + tomogramId=tomogramId, + sourceTomoId=tomogramItem.get("sourceTomoId"), + tomogramVolumeId=volumeId, + tiltSeriesId=tiltSeriesId, + tsId=tiltSeriesId, + ctfSeriesId=tiltSeriesId, + label=label, + ) + + def _findRegularTiltSeriesStoredSetForProtocol( + self, + protocolDbId: Any, + visited: Optional[set] = None, + ) -> Optional[Dict[str, Any]]: + if protocolDbId is None: + return None + + if visited is None: + visited = set() + + protocolKey = str(protocolDbId) + if protocolKey in visited: + return None + + visited.add(protocolKey) + + sameProtocolStoredSet = self._findRegularTiltSeriesStoredSetInProtocol(protocolDbId) + if sameProtocolStoredSet is not None: + return sameProtocolStoredSet + + inputRefs = self._listProtocolInputRefs(protocolDbId) + + for inputRef in inputRefs: + if self._getInputRefKind(inputRef) != "tiltSeries": + continue + + storedSet = self._getStoredSetFromInputRef(inputRef) + if self._isRegularTiltSeriesStoredSet(storedSet): + return storedSet + + for inputRef in inputRefs: + if self._getInputRefKind(inputRef) != "ctf": + continue + + storedSet = self._findRegularTiltSeriesStoredSetForProtocol( + inputRef.get("parentProtocolDbId"), + visited=visited, + ) + + if storedSet is not None: + return storedSet + + return None + + def _findRegularTiltSeriesStoredSetInProtocol( + self, + protocolDbId: Any, + ) -> Optional[Dict[str, Any]]: + try: + storedSets = self.setMapper.listProtocolStoredSets( + projectId=self.projectId, + protocolDbId=int(protocolDbId), + ) + except Exception: + return None + + for storedSet in storedSets or []: + storedSetDict = dict(storedSet) + if self._isRegularTiltSeriesStoredSet(storedSetDict): + return storedSetDict + + return None + + def _mergeRegularTiltSeriesForCtftomo( + self, + rootProtocolDbId: Any, + links: Dict[str, Optional[Dict[str, Any]]], + summaries: Dict[str, Optional[Dict[str, Any]]], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + if links.get("tiltSeries") is not None: + return None + + storedSet = self._findRegularTiltSeriesStoredSetForProtocol(rootProtocolDbId) + if storedSet is None: + return None + + links["tiltSeries"] = self._buildLink( + protocolId=storedSet.get("protocolDbId"), + outputName=storedSet.get("outputName"), + storedSet=storedSet, + label=storedSet.get("outputName"), + statusValue="inferred", + ) + + summaries["tiltSeries"] = self._buildSummary( + storedSet, + extra={ + "source": "ctfAssociatedTiltSeries", + }, + ) + + protocolDbId = storedSet.get("protocolDbId") + outputName = storedSet.get("outputName") + + if protocolDbId is None or not outputName: + return storedSet + + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + + items = self._filterIntegratedItemsByAllowedKeys( + reader.listTiltSeries(), + self._getRelationKeySet(relationsByKey), + ) + self._addTiltSeriesRelations(relationsByKey, items) + + return storedSet + + def _mergeKindFromInputRefs( + self, + kind: str, + inputRefs: List[Dict[str, Any]], + links: Dict[str, Optional[Dict[str, Any]]], + summaries: Dict[str, Optional[Dict[str, Any]]], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + existingLink = links.get(kind) + if existingLink is not None and not self._shouldReplaceLink(existingLink): + return None + + inputRef = self._findInputRefByKind(inputRefs, kind) + if inputRef is None: + return None + + storedSet = self._getStoredSetFromInputRef(inputRef) + if storedSet is None: + return None + + links[kind] = self._buildInputRefLink(inputRef, storedSet) + summaries[kind] = self._buildSummary( + storedSet, + extra={ + "source": "inputRef", + "inputName": inputRef.get("inputName"), + }, + ) + + self._mergeInputRefRelations( + kind=kind, + inputRef=inputRef, + storedSet=storedSet, + relationsByKey=relationsByKey, + ) + + return inputRef + + def _mergeExactInputRefs( + self, + rootKind: str, + rootStoredSet: Dict[str, Any], + links: Dict[str, Optional[Dict[str, Any]]], + summaries: Dict[str, Optional[Dict[str, Any]]], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> None: + rootProtocolDbId = rootStoredSet.get("protocolDbId") + inputRefs = self._listProtocolInputRefs(rootProtocolDbId) + + if not inputRefs: + return + + if rootKind == "ctf": + self._mergeRegularTiltSeriesForCtftomo( + rootProtocolDbId=rootProtocolDbId, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + return + + if rootKind == "tomogram": + self._mergeRegularTiltSeriesForCtftomo( + rootProtocolDbId=rootProtocolDbId, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + self._mergeKindFromInputRefs( + kind="ctf", + inputRefs=inputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + return + + if rootKind == "coordinates3d": + tomogramRef = self._mergeKindFromInputRefs( + kind="tomogram", + inputRefs=inputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + ctfRef = self._mergeKindFromInputRefs( + kind="ctf", + inputRefs=inputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + tiltRef = self._mergeKindFromInputRefs( + kind="tiltSeries", + inputRefs=inputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + if tomogramRef is not None: + tomogramInputRefs = self._listProtocolInputRefs( + tomogramRef.get("parentProtocolDbId") + ) + + if ctfRef is None: + ctfRef = self._mergeKindFromInputRefs( + kind="ctf", + inputRefs=tomogramInputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + if tiltRef is None: + tiltRef = self._mergeKindFromInputRefs( + kind="tiltSeries", + inputRefs=tomogramInputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + if tiltRef is None and ctfRef is not None: + ctfInputRefs = self._listProtocolInputRefs(ctfRef.get("parentProtocolDbId")) + self._mergeKindFromInputRefs( + kind="tiltSeries", + inputRefs=ctfInputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + def _firstValueByName( + self, + values: Dict[str, Any], + names: List[str], + ): + normalizedNames = { + self._normalizeName(name) + for name in names + } + + for key, value in (values or {}).items(): + if self._normalizeName(key) in normalizedNames: + return value + + return None + + def _normalizeName(self, value: Any) -> str: + return str(value).replace("_", "").replace(".", "").replace("-", "").lower() + + def _toRelationMatchText(self, value: Any) -> Optional[str]: + if value is None: + return None + + text = str(value).strip() + return text or None + + def _iterRelationMatchValues( + self, + keyValue: Any, + values: Dict[str, Any], + ) -> List[str]: + candidates = [ + keyValue, + values.get("key"), + values.get("tiltSeriesId"), + values.get("tsId"), + values.get("ctfSeriesId"), + values.get("tomogramId"), + values.get("sourceTomoId"), + values.get("coordinatesTomogramId"), + ] + + result = [] + seen = set() + + for candidate in candidates: + text = self._toRelationMatchText(candidate) + if not text or text in seen: + continue + + seen.add(text) + result.append(text) + + return result + + def _findExistingRelationKeys( + self, + relationsByKey: Dict[str, Dict[str, Any]], + matchValues: List[str], + ) -> List[str]: + matchSet = set(matchValues or []) + if not matchSet: + return [] + + keys = [] + + for key, relation in (relationsByKey or {}).items(): + existingValues = self._iterRelationMatchValues(key, relation) + if any(value in matchSet for value in existingValues): + keys.append(key) + + return keys + + def _addRelation( + self, + relationsByKey: Dict[str, Dict[str, Any]], + keyValue: Any, + **values: Any, + ) -> None: + requestedKey = self._toRelationMatchText(keyValue) + if not requestedKey: + return + + matchValues = self._iterRelationMatchValues(requestedKey, values) + existingKeys = self._findExistingRelationKeys(relationsByKey, matchValues) + key = existingKeys[0] if existingKeys else requestedKey + + relation = relationsByKey.setdefault( + key, + { + "key": key, + "label": key, + }, + ) + + for otherKey in existingKeys[1:]: + otherRelation = relationsByKey.pop(otherKey, None) + if not otherRelation: + continue + + for name, value in otherRelation.items(): + if value is not None and name not in relation: + relation[name] = value + + for name, value in values.items(): + if value is not None: + relation[name] = value + + def _addTiltSeriesRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tiltSeriesId = item.get("tiltSeriesId") or item.get("id") or index + label = item.get("label") or str(tiltSeriesId) + + self._addRelation( + relationsByKey, + tiltSeriesId, + tiltSeriesId=tiltSeriesId, + tsId=tiltSeriesId, + label=label, + ) + + def _addCtftomoRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tiltSeriesId = item.get("tiltSeriesId") or item.get("id") or index + label = item.get("label") or str(tiltSeriesId) + + self._addRelation( + relationsByKey, + tiltSeriesId, + ctfSeriesId=tiltSeriesId, + tiltSeriesId=tiltSeriesId, + tsId=tiltSeriesId, + label=label, + ) + + def _addTomogramRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tiltSeriesId = ( + item.get("tiltSeriesId") + or item.get("tsId") + ) + + sourceTomoId = ( + item.get("sourceTomoId") + or item.get("tomogramId") + or item.get("tomoId") + ) + + tomogramId = ( + sourceTomoId + or item.get("id") + or item.get("label") + or index + ) + + relationKey = tiltSeriesId or tomogramId + ctfSeriesId = item.get("ctfSeriesId") or tiltSeriesId + label = item.get("label") or item.get("name") or str(tomogramId) + volumeId = ( + item.get("tomogramVolumeId") + if item.get("tomogramVolumeId") is not None + else item.get("volumeId") + ) + + if volumeId is None: + volumeId = index + + self._addRelation( + relationsByKey, + relationKey, + tomogramId=tomogramId, + sourceTomoId=sourceTomoId, + tomogramVolumeId=volumeId, + tiltSeriesId=tiltSeriesId, + tsId=tiltSeriesId, + ctfSeriesId=ctfSeriesId, + label=label, + ) + + def _addCoordinates3dRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tomogramId = ( + item.get("tomoId") + or item.get("tomogramId") + or item.get("id") + or item.get("label") + or index + ) + + label = item.get("label") or item.get("name") or str(tomogramId) + + self._addRelation( + relationsByKey, + tomogramId, + coordinatesTomogramId=tomogramId, + tomogramId=tomogramId, + label=label, + ) + + def _mergeRootRelations( + self, + rootKind: str, + storedSet: Dict[str, Any], + relationsByKey: Dict[str, Dict[str, Any]], + links: Dict[str, Optional[Dict[str, Any]]], + summaries: Dict[str, Optional[Dict[str, Any]]], + ) -> None: + if rootKind == "tiltSeries": + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=self.protocolId, + outputName=self.outputName, + ) + self._addTiltSeriesRelations(relationsByKey, reader.listTiltSeries()) + return + + if rootKind == "ctf": + reader = PostgresqlCtftomoReader( + db=self.db, + projectId=self.projectId, + protocolId=self.protocolId, + outputName=self.outputName, + ) + self._addCtftomoRelations(relationsByKey, reader.listCtftomoSeries()) + return + + if rootKind == "coordinates3d": + reader = PostgresqlCoords3dReader( + db=self.db, + projectId=self.projectId, + protocolId=self.protocolId, + outputName=self.outputName, + ) + tomograms = reader.listTomograms() or [] + self._addCoordinates3dRelations(relationsByKey, tomograms) + + if tomograms: + links["tomogram"] = { + "protocolId": self.protocolId, + "outputName": self.outputName, + "itemId": None, + "label": "Tomograms", + "status": "derived", + "source": "coordinates3d", + } + summaries["tomogram"] = { + "objectClass": "SetOfTomograms", + "size": len(tomograms), + "source": "coordinates3d", + } + return + + if rootKind == "tomogram": + items = self._buildTomogramItemsFromStoredSet(storedSet) + self._addTomogramRelations(relationsByKey, items) + + def _buildTomogramItemsFromStoredSet( + self, + storedSet: Dict[str, Any], + ) -> List[Dict[str, Any]]: + items = [] + + for index, item in enumerate(storedSet.get("items") or []): + values = item.get("values") or {} + + tsId = self._firstValueByName( + values, + ["_tsId", "tsId", "tiltSeriesId", "tilt_series_id"], + ) + + tomoId = self._firstValueByName( + values, + ["_tomoId", "tomoId", "tomogramId", "tomo_id", "tomogram_id"], + ) + + label = self._firstValueByName( + values, + ["_objLabel", "label", "name", "_tsId", "tsId", "tomoId"], + ) + + publicId = tsId or tomoId or item.get("scipionItemId") or index + + row = { + "id": publicId, + "tomoId": tomoId or publicId, + "tomogramId": tomoId or publicId, + "label": label or publicId, + "volumeId": index, + "tomogramVolumeId": index, + } + + if tsId is not None: + row["tsId"] = tsId + row["tiltSeriesId"] = tsId + row["ctfSeriesId"] = tsId + + if tomoId is not None: + row["sourceTomoId"] = tomoId + + items.append(row) + + return items + + def _listRelatedStoredSets(self) -> List[Dict[str, Any]]: + try: + rows = self.db.fetchAll( + """ + WITH root_protocol AS ( + SELECT id, "projectId", "protocolId", "protocolClassName" + FROM protocols + WHERE "projectId" = %s + AND (id = %s OR "protocolId" = %s) + LIMIT 1 + ), + related_protocols AS ( + SELECT + id, + "projectId", + "protocolId", + "protocolClassName", + 'root' AS "relationRole", + 0 AS distance + FROM root_protocol + + UNION ALL + + SELECT + parent.id, + parent."projectId", + parent."protocolId", + parent."protocolClassName", + 'parent' AS "relationRole", + 1 AS distance + FROM protocol_dependencies d + JOIN root_protocol root + ON root.id = d."childProtocolDbId" + AND root."projectId" = d."projectId" + JOIN protocols parent + ON parent.id = d."parentProtocolDbId" + AND parent."projectId" = d."projectId" + + UNION ALL + + SELECT + child.id, + child."projectId", + child."protocolId", + child."protocolClassName", + 'child' AS "relationRole", + 1 AS distance + FROM protocol_dependencies d + JOIN root_protocol root + ON root.id = d."parentProtocolDbId" + AND root."projectId" = d."projectId" + JOIN protocols child + ON child.id = d."childProtocolDbId" + AND child."projectId" = d."projectId" + ) + SELECT DISTINCT ON (s.id) + s.id, + s."projectId", + s."protocolDbId", + s."objectId", + s."outputName", + s."setClassName", + s."itemClassName", + s.properties, + s."createdAt", + s."updatedAt", + rp."protocolId" AS "publicProtocolId", + rp."protocolClassName", + rp."relationRole", + rp.distance + FROM related_protocols rp + JOIN scipion_sets s + ON s."projectId" = rp."projectId" + AND s."protocolDbId" = rp.id + ORDER BY s.id, rp.distance ASC, rp."relationRole" ASC + """, + ( + self.projectId, + int(self.protocolId), + str(self.protocolId), + ), + ) + except Exception: + logger.debug( + "Could not list related PostgreSQL stored sets for integrated context. " + "projectId=%s protocolId=%s outputName=%s", + self.projectId, + self.protocolId, + self.outputName, + exc_info=True, + ) + return [] + + result = [dict(row) for row in rows or []] + result.sort( + key=lambda item: ( + int(item.get("distance") or 0), + str(item.get("relationRole") or ""), + int(item.get("protocolDbId") or 0), + str(item.get("outputName") or ""), + ) + ) + return result + + def _shouldSkipDependencyCandidate( + self, + rootKind: Optional[str], + candidateKind: str, + ) -> bool: + if rootKind == "coordinates3d" and candidateKind == "tomogram": + return True + + if rootKind == "coordinates3d" and candidateKind in {"tiltSeries", "ctf"}: + return True + + if rootKind == "tomogram" and candidateKind == "tiltSeries": + return True + + return False + + def _mergeRelatedStoredSets( + self, + rootStoredSet: Dict[str, Any], + links: Dict[str, Optional[Dict[str, Any]]], + summaries: Dict[str, Optional[Dict[str, Any]]], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> None: + rootKind = self._getIntegratedKind(rootStoredSet) + + for candidate in self._listRelatedStoredSets(): + if self._isSameStoredSet(candidate, rootStoredSet): + continue + + candidateKind = self._getIntegratedKind(candidate) + if candidateKind is None or candidateKind not in links: + continue + + if self._shouldSkipDependencyCandidate(rootKind, candidateKind): + continue + + if self._shouldReplaceLink(links.get(candidateKind)): + links[candidateKind] = self._buildLink( + protocolId=self._getCandidateProtocolId(candidate), + outputName=candidate.get("outputName"), + storedSet=candidate, + statusValue="related", + ) + summaries[candidateKind] = self._buildSummary(candidate) + + allowedRelationKeys = None + if rootKind in {"coordinates3d", "tomogram"}: + allowedRelationKeys = self._getRelationKeySet(relationsByKey) + + self._mergeRelationsForCandidate( + candidate=candidate, + candidateKind=candidateKind, + relationsByKey=relationsByKey, + allowedRelationKeys=allowedRelationKeys, + ) + + def _filterIntegratedItemsByAllowedKeys( + self, + items: List[Dict[str, Any]], + allowedRelationKeys: Optional[set], + ) -> List[Dict[str, Any]]: + if not allowedRelationKeys: + return items or [] + + filteredItems = [] + + for item in items or []: + candidates = [ + item.get("key"), + item.get("label"), + item.get("name"), + item.get("id"), + item.get("tomoId"), + item.get("tomogramId"), + item.get("tiltSeriesId"), + item.get("ctfSeriesId"), + item.get("coordinatesTomogramId"), + item.get("tsId"), + item.get("sourceTomoId"), + ] + + if any( + str(value).strip() in allowedRelationKeys + for value in candidates + if value is not None and str(value).strip() + ): + filteredItems.append(item) + + return filteredItems + + def _mergeRelationsForCandidate( + self, + candidate: Dict[str, Any], + candidateKind: str, + relationsByKey: Dict[str, Dict[str, Any]], + allowedRelationKeys: Optional[set] = None, + ) -> None: + protocolDbId = candidate.get("protocolDbId") + outputName = candidate.get("outputName") + + if protocolDbId is None or not outputName: + return + + if candidateKind == "tiltSeries": + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + items = self._filterIntegratedItemsByAllowedKeys( + reader.listTiltSeries(), + allowedRelationKeys, + ) + self._addTiltSeriesRelations(relationsByKey, items) + return + + if candidateKind == "ctf": + reader = PostgresqlCtftomoReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + items = self._filterIntegratedItemsByAllowedKeys( + reader.listCtftomoSeries(), + allowedRelationKeys, + ) + self._addCtftomoRelations(relationsByKey, items) + return + + if candidateKind == "coordinates3d": + reader = PostgresqlCoords3dReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + items = self._filterIntegratedItemsByAllowedKeys( + reader.listTomograms() or [], + allowedRelationKeys, + ) + self._addCoordinates3dRelations(relationsByKey, items) + return + + if candidateKind == "tomogram": + storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=protocolDbId, + outputName=outputName, + limit=None, + offset=0, + ) + + if storedSet is None: + return + + items = self._filterIntegratedItemsByAllowedKeys( + self._buildTomogramItemsFromStoredSet(storedSet), + allowedRelationKeys, + ) + self._addTomogramRelations(relationsByKey, items) + + def _isSameStoredSet( + self, + left: Dict[str, Any], + right: Dict[str, Any], + ) -> bool: + return ( + str(left.get("protocolDbId")) == str(right.get("protocolDbId")) + and str(left.get("outputName")) == str(right.get("outputName")) + ) + + def _shouldReplaceLink( + self, + existingLink: Optional[Dict[str, Any]], + ) -> bool: + if existingLink is None: + return True + + protocolId = existingLink.get("protocolId") + outputName = existingLink.get("outputName") + statusValue = str(existingLink.get("status") or "") + sourceValue = str(existingLink.get("source") or "") + + if protocolId is None and outputName is None: + return True + + if statusValue == "derived" and sourceValue == "coordinates3d": + return True + + return statusValue == "inferred" and protocolId is None + + def _getCandidateProtocolId(self, candidate: Dict[str, Any]) -> Any: + return candidate.get("protocolDbId") or candidate.get("publicProtocolId") + + def _safeValue(self, value: Any) -> Any: + if isinstance(value, dict): + return { + str(key): self._safeValue(item) + for key, item in value.items() + } + + if isinstance(value, (list, tuple, set)): + return [ + self._safeValue(item) + for item in value + ] + + if isinstance(value, (datetime, date)): + return value.isoformat() + + if hasattr(value, "item"): + try: + return value.item() + except Exception: + pass + + return value \ No newline at end of file diff --git a/app/backend/viewers/postgresql_tiltseries_reader.py b/app/backend/viewers/postgresql_tiltseries_reader.py new file mode 100644 index 00000000..5d121b08 --- /dev/null +++ b/app/backend/viewers/postgresql_tiltseries_reader.py @@ -0,0 +1,406 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +import ast +import json +import math + +from typing import Any, Dict, List, Optional + +from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + +class PostgresqlTiltSeriesReader: + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + self.setMapper = ScipionSetPostgresqlMapper(db) + self._storedSet = None + self._logicalTables = None + + def hasOutput(self) -> bool: + storedSet = self._getStoredSet() + return storedSet is not None and self._isTiltSeriesStoredSet(storedSet) + + def listTiltSeries(self) -> List[Dict[str, Any]]: + storedSet = self._getStoredSet() + if storedSet is None or not self._isTiltSeriesStoredSet(storedSet): + return [] + + result = [] + for index, item in enumerate(storedSet.get("items") or []): + result.append(self._buildTiltSeriesSummary(item, index)) + + return result + + def getTiltSeriesFrames(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: + storedSet = self._getStoredSet() + if storedSet is None or not self._isTiltSeriesStoredSet(storedSet): + return None + + seriesItem = self._findTiltSeriesItem(tiltSeriesId) + + seriesSummary = self._buildTiltSeriesSummary(seriesItem, 0) + childTable = self._findChildTableForParentItem(seriesItem.get("scipionItemId")) + + frames: List[Dict[str, Any]] = [] + if childTable is not None: + childItems = self.setMapper.getStoredSetTableItems(int(childTable["id"])) + for index, item in enumerate(childItems): + frames.append(self._buildTiltImageFrame(item, index)) + + payload: Dict[str, Any] = { + "tiltSeriesId": seriesSummary.get("tiltSeriesId") or str(tiltSeriesId), + "label": seriesSummary.get("label") or str(tiltSeriesId), + "frames": frames, + } + + if "tiltAxisAngle" in seriesSummary: + payload["tiltAxisAngle"] = seriesSummary["tiltAxisAngle"] + + return payload + + def getTiltImageFrame(self, tiltSeriesId: Any, index: Any) -> Optional[Dict[str, Any]]: + payload = self.getTiltSeriesFrames(tiltSeriesId) + if not payload: + return None + + frames = payload.get("frames") or [] + targetIndex = self._toOptionalInt(index) + + if targetIndex is not None: + for frame in frames: + frameIndex = self._toOptionalInt(frame.get("index")) + if frameIndex == targetIndex: + return frame + + if 0 <= targetIndex < len(frames): + return frames[targetIndex] + + return None + + def _getStoredSet(self) -> Optional[Dict[str, Any]]: + if self._storedSet is None: + self._storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=self.protocolId, + outputName=self.outputName, + ) + return self._storedSet + + def _isTiltSeriesStoredSet(self, storedSet: Dict[str, Any]) -> bool: + classText = self._getStoredSetClassText(storedSet) + return "tiltseries" in classText and "ctftomo" not in classText + + def _getStoredSetClassText(self, storedSet: Dict[str, Any]) -> str: + return ("%s %s" % ( + storedSet.get("setClassName") or "", + storedSet.get("itemClassName") or "", + )).replace(" ", "").lower() + + def _getLogicalTables(self) -> List[Dict[str, Any]]: + if self._logicalTables is None: + storedSet = self._getStoredSet() + if storedSet is None: + self._logicalTables = [] + else: + self._logicalTables = self.setMapper.listStoredSetTables( + int(storedSet["id"]) + ) + return self._logicalTables + + def _buildTiltSeriesSummary(self, item: Dict[str, Any], index: int) -> Dict[str, Any]: + values = item.get("values") or {} + itemId = item.get("scipionItemId") + + tiltSeriesId = self._firstValue( + values, + ["_tsId", "tsId", "tiltSeriesId", "id"], + ) + if tiltSeriesId is None: + tiltSeriesId = item.get("label") or itemId or index + + summary: Dict[str, Any] = { + "tiltSeriesId": str(tiltSeriesId), + "label": "TiltSeries %s" % str(tiltSeriesId), + } + + childTable = self._findChildTableForParentItem(itemId) + if childTable is not None: + childItems = self.setMapper.getStoredSetTableItems(int(childTable["id"])) + summary["nViews"] = len(childItems) + + dims = self._firstValueBySuffix( + values, + ["dim", "dims", "dimensions"], + ) + if dims is not None: + summary["dims"] = dims + + pixelSize = self._firstValueBySuffix( + values, + ["samplingrate", "pixelSize", "pixel_size"], + ) + if pixelSize is not None: + summary["pixelSize"] = self._toOptionalFloat(pixelSize) + + tiltAxisAngle = self._firstValueBySuffix( + values, + ["tiltaxisangle"], + ) + if tiltAxisAngle is not None: + summary["tiltAxisAngle"] = self._toOptionalFloat(tiltAxisAngle) + + return summary + + def _findTiltSeriesItem(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: + storedSet = self._getStoredSet() + if storedSet is None: + return None + + targetKey = str(tiltSeriesId) + + for index, item in enumerate(storedSet.get("items") or []): + itemTiltSeriesId = self._getTiltSeriesIdFromItem(item, index) + if str(itemTiltSeriesId) == targetKey: + return item + + return None + + def _getTiltSeriesIdFromItem(self, item: Dict[str, Any], index: int) -> Any: + values = item.get("values") or {} + tiltSeriesId = self._firstValue( + values, + ["_tsId", "tsId", "tiltSeriesId", "id"], + ) + + if tiltSeriesId is not None: + return tiltSeriesId + + return item.get("label") or item.get("scipionItemId") or index + + def _buildTiltImageFrame(self, item: Dict[str, Any], position: int) -> Dict[str, Any]: + values = item.get("values") or {} + + frame: Dict[str, Any] = { + "viewId": item.get("scipionItemId"), + "index": position, + } + + imageIndex = self._firstValueBySuffix(values, ["index"]) + imageIndexInt = self._toOptionalInt(imageIndex) + if imageIndexInt is not None: + frame["index"] = imageIndexInt + + order = self._firstValueBySuffix( + values, + ["acquisitionorder", "acqorder", "order"], + ) + orderInt = self._toOptionalInt(order) + if orderInt is not None: + frame["order"] = orderInt + + tiltAngle = self._firstValueBySuffix(values, ["tiltangle"]) + tiltAngleFloat = self._toOptionalFloat(tiltAngle) + if tiltAngleFloat is not None: + frame["tiltAngle"] = tiltAngleFloat + + excluded = self._getFrameExcluded(item, values) + frame["excluded"] = excluded + + dose = self._firstValueBySuffix(values, ["accumdose", "dose"]) + doseFloat = self._toOptionalFloat(dose) + if doseFloat is not None: + frame["dose"] = doseFloat + + imagePath = self._firstValueBySuffix( + values, + ["filename", "filepath", "path"], + ) + if imagePath: + frame["path"] = "%s@%s" % (str(frame["index"]), str(imagePath)) + + transformValues = self._getFrameTransform(values) + frame.update(transformValues) + + return frame + + def _findChildTableForParentItem(self, parentItemId: Any) -> Optional[Dict[str, Any]]: + if parentItemId is None: + return None + + for table in self._getLogicalTables(): + if table.get("tableKind") != "child": + continue + if str(table.get("parentItemId")) == str(parentItemId): + return table + + return None + + def _firstValue(self, values: Dict[str, Any], keys: List[str]) -> Any: + for key in keys: + if key in values: + return values.get(key) + return None + + def _firstValueBySuffix(self, values: Dict[str, Any], suffixes: List[str]) -> Any: + normalizedSuffixes = [ + str(suffix).replace("_", "").replace(".", "").lower() + for suffix in suffixes + ] + + for key, value in values.items(): + normalizedKey = str(key).replace("_", "").replace(".", "").lower() + for suffix in normalizedSuffixes: + if normalizedKey.endswith(suffix): + return value + + return None + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + try: + return float(value) + except Exception: + return None + + def _getFrameExcluded(self, item: Dict[str, Any], values: Dict[str, Any]) -> bool: + excludedValue = self._firstValueBySuffix( + values, + ["excluded", "skip"], + ) + if excludedValue is not None: + return bool(self._toOptionalBool(excludedValue)) + + enabledValue = item.get("enabled") + if enabledValue is None: + enabledValue = self._firstValueBySuffix(values, ["enabled", "isenabled"]) + + enabled = self._toOptionalBool(enabledValue) + if enabled is None: + return False + + return not enabled + + def _getFrameTransform(self, values: Dict[str, Any]) -> Dict[str, Any]: + matrixValue = self._firstValueBySuffix(values, ["matrix"]) + matrix = self._parseMatrix(matrixValue) + if not matrix: + return {} + + flat = self._flattenMatrix(matrix) + result: Dict[str, Any] = {} + + if len(flat) >= 6: + result["shiftX"] = flat[2] + result["shiftY"] = flat[5] + + try: + if isinstance(matrix, list) and len(matrix) >= 2: + m00 = float(matrix[0][0]) + m10 = float(matrix[1][0]) + result["rot"] = math.degrees(-math.atan2(m10, m00)) + except Exception: + pass + + return result + + def _parseMatrix(self, value: Any) -> Optional[Any]: + if value is None or value == "": + return None + + if isinstance(value, list): + return value + + if isinstance(value, tuple): + return list(value) + + if isinstance(value, str): + text = value.strip() + if not text: + return None + + try: + return json.loads(text) + except Exception: + pass + + try: + return ast.literal_eval(text) + except Exception: + return None + + return None + + def _flattenMatrix(self, matrix: Any) -> List[float]: + values: List[float] = [] + + if not isinstance(matrix, list): + return values + + for row in matrix: + if isinstance(row, list): + for value in row: + parsedValue = self._toOptionalFloat(value) + if parsedValue is not None: + values.append(parsedValue) + else: + parsedValue = self._toOptionalFloat(row) + if parsedValue is not None: + values.append(parsedValue) + + return values + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except Exception: + try: + return int(float(value)) + except Exception: + return None + + def _toOptionalBool(self, value: Any) -> Optional[bool]: + if value is None or value == "": + return None + + if isinstance(value, bool): + return value + + if isinstance(value, (int, float)): + return bool(value) + + text = str(value).strip().lower() + if text in ("1", "true", "yes", "y", "on", "enabled"): + return True + if text in ("0", "false", "no", "n", "off", "disabled"): + return False + + return None \ No newline at end of file diff --git a/app/backend/viewers/postgresql_volume_reader.py b/app/backend/viewers/postgresql_volume_reader.py new file mode 100644 index 00000000..068e5ba1 --- /dev/null +++ b/app/backend/viewers/postgresql_volume_reader.py @@ -0,0 +1,782 @@ +import json +import os +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple, Union + +import numpy as np + +from app.backend.utils.volume_utils import readVolumeArray3d + + +class PostgresqlVolumeReader: + """Read Volume, VolumeMask and SetOfVolumes outputs from PostgreSQL.""" + + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = int(projectId) + self.protocolId = int(protocolId) + self.outputName = str(outputName) + self.lastSkipReason = None + self._protocolDbId = None + self._storedSet = None + self._storedObjectTree = None + self._volumes = None + + def hasOutput(self) -> bool: + if self._getStoredSet() is not None: + return True + + return bool(self._getStoredObjectTree()) + + def listVolumes(self) -> Optional[List[Dict[str, Any]]]: + self.lastSkipReason = None + + if self._volumes is not None: + return self._volumes + + storedSet = self._getStoredSet() + if storedSet is not None: + volumes = self._listSetVolumes(storedSet) + if volumes: + self._volumes = volumes + return self._volumes + + objectTree = self._getStoredObjectTree() + if objectTree: + volume = self._buildSingleVolumeFromObjectTree(objectTree) + if volume is not None: + self._volumes = [volume] + return self._volumes + + self.lastSkipReason = "volume_output_not_resolved" + return None + + def getVolumeInfo(self, volumeId: Union[int, str]) -> Optional[Dict[str, Any]]: + self.lastSkipReason = None + + volumes = self.listVolumes() + if not volumes: + self.lastSkipReason = self.lastSkipReason or "volume_list_empty" + return None + + volume = self._findVolume(volumeId, volumes) + if volume is None: + self.lastSkipReason = "volume_not_found volumeId=%s" % str(volumeId) + return None + + info = dict(volume) + self._ensureVolumeInfoFromFile(info) + return info + + def getVolumeFile(self, volumeId: Union[int, str]) -> Optional[Dict[str, Any]]: + info = self.getVolumeInfo(volumeId) + if info is None: + return None + + fileName = info.get("fileName") or info.get("path") + if not fileName: + self.lastSkipReason = "volume_file_not_found volumeId=%s" % str(volumeId) + return None + + resolvedPath = self._resolveExistingPath(fileName) + if resolvedPath is None: + self.lastSkipReason = "volume_file_missing fileName=%s" % str(fileName) + return None + + info["fileName"] = resolvedPath + info["path"] = resolvedPath + return info + + def getVolumeArray( + self, + volumeId: Union[int, str], + ) -> Optional[Tuple[np.ndarray, Dict[str, Any], Dict[str, Any]]]: + info = self.getVolumeFile(volumeId) + if info is None: + return None + + volumePath = info.get("fileName") or info.get("path") + if not volumePath: + self.lastSkipReason = "volume_file_not_found volumeId=%s" % str(volumeId) + return None + + try: + array, props = readVolumeArray3d(str(volumePath)) + except Exception as exc: + self.lastSkipReason = "volume_read_failed volumeId=%s error=%s" % ( + str(volumeId), + str(exc), + ) + return None + + props = props if isinstance(props, dict) else {} + return array, props, info + + def getHistogram( + self, + volumeId: Union[int, str], + bins: int = 128, + ) -> Optional[Dict[str, Any]]: + result = self.getVolumeArray(volumeId) + if result is None: + return None + + array, _props, _info = result + cleanArray = np.asarray(array, dtype=np.float32) + cleanArray = cleanArray[np.isfinite(cleanArray)] + + if cleanArray.size == 0: + return { + "binEdges": [], + "counts": [], + } + + counts, binEdges = np.histogram(cleanArray, bins=max(4, int(bins or 128))) + + return { + "binEdges": [float(value) for value in binEdges.tolist()], + "counts": [int(value) for value in counts.tolist()], + } + + def _listSetVolumes(self, storedSet: Dict[str, Any]) -> List[Dict[str, Any]]: + volumes: List[Dict[str, Any]] = [] + + for index, item in enumerate(storedSet.get("items") or []): + values = self._normalizeJsonObject(item.get("values")) + volume = self._buildVolumeFromSetItem( + item=item, + values=values, + index=index, + ) + + if volume is not None: + volumes.append(volume) + + return volumes + + def _buildVolumeFromSetItem( + self, + item: Dict[str, Any], + values: Dict[str, Any], + index: int, + ) -> Optional[Dict[str, Any]]: + fileName, locationIndex = self._extractVolumeFile(values) + scipionItemId = item.get("scipionItemId") + + rawLabel = ( + item.get("label") + or self._firstValueBySuffix(values, ["objLabel", "label", "name", "volName"]) + ) + + label = self._normalizeVolumeDisplayName( + label=rawLabel, + fileName=fileName, + index=index, + ) + + volume: Dict[str, Any] = { + "id": int(index), + "index": int(index), + "name": str(label), + "label": str(label), + "relPath": str(label), + } + + if scipionItemId is not None: + volume["objectId"] = scipionItemId + volume["scipionItemId"] = scipionItemId + + className = item.get("className") or self._firstValueBySuffix(values, ["className"]) + if className: + volume["className"] = str(className) + + if fileName: + volume["fileName"] = fileName + volume["path"] = fileName + + if locationIndex is not None: + volume["locationIndex"] = locationIndex + + tsId = self._firstValueBySuffix( + values, + ["tsId", "tiltSeriesId"], + ) + if tsId is not None: + volume["tsId"] = tsId + volume["tiltSeriesId"] = tsId + + tomoId = self._firstValueBySuffix( + values, + ["tomoId", "tomogramId"], + ) + if tomoId is not None: + volume["tomoId"] = tomoId + volume["tomogramId"] = tomoId + + dims = self._extractDims(values) + if dims is not None: + volume["dims"] = dims + + samplingRate = self._extractSamplingRate(values) + if samplingRate is not None: + self._attachSamplingRate(volume, samplingRate) + + return volume + + def _buildSingleVolumeFromObjectTree( + self, + rows: List[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + root = None + valuesByPath: Dict[str, Any] = {} + + for row in rows: + path = str(row.get("path") or "") + if path == self.outputName: + root = row + continue + + if path.startswith(self.outputName + "."): + suffix = path[len(self.outputName) + 1:] + valuesByPath[suffix] = row.get("value") + + if root is None: + return None + + fileName, locationIndex = self._extractVolumeFile(valuesByPath) + rawLabel = root.get("label") or root.get("name") or self.outputName + label = self._normalizeVolumeDisplayName( + label=rawLabel, + fileName=fileName, + index=0, + ) + + volume: Dict[str, Any] = { + "id": 0, + "index": 0, + "name": str(label), + "label": str(label), + "relPath": str(label), + } + + objectId = root.get("scipionObjId") + if objectId is not None: + volume["objectId"] = objectId + volume["scipionItemId"] = objectId + + className = root.get("className") + if className: + volume["className"] = str(className) + + if fileName: + volume["fileName"] = fileName + volume["path"] = fileName + + if locationIndex is not None: + volume["locationIndex"] = locationIndex + + tsId = self._firstValueBySuffix( + valuesByPath, + ["tsId", "tiltSeriesId"], + ) + if tsId is not None: + volume["tsId"] = tsId + volume["tiltSeriesId"] = tsId + + tomoId = self._firstValueBySuffix( + valuesByPath, + ["tomoId", "tomogramId"], + ) + if tomoId is not None: + volume["tomoId"] = tomoId + volume["tomogramId"] = tomoId + + dims = self._extractDims(valuesByPath) + if dims is not None: + volume["dims"] = dims + + samplingRate = self._extractSamplingRate(valuesByPath) + if samplingRate is not None: + self._attachSamplingRate(volume, samplingRate) + + return volume + + def _findVolume( + self, + volumeId: Union[int, str], + volumes: List[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + requested = self._toText(volumeId) + if requested is None: + return None + + for volume in volumes: + if self._toText(volume.get("id")) == requested: + return volume + + for volume in volumes: + if self._toText(volume.get("index")) == requested: + return volume + + requestedInt = self._toOptionalInt(requested) + if requestedInt is not None and 0 <= requestedInt < len(volumes): + return volumes[requestedInt] + + for volume in volumes: + for key in ("objectId", "scipionItemId"): + if self._toText(volume.get(key)) == requested: + return volume + + for volume in volumes: + for key in ("name", "label"): + if self._toText(volume.get(key)) == requested: + return volume + + return None + + def _ensureVolumeInfoFromFile(self, volume: Dict[str, Any]) -> None: + fileName = volume.get("fileName") or volume.get("path") + if not fileName: + return + + resolvedPath = self._resolveExistingPath(fileName) + if resolvedPath is None: + return + + volume["fileName"] = resolvedPath + volume["path"] = resolvedPath + + try: + array, props = readVolumeArray3d(resolvedPath) + except Exception: + return + + if getattr(array, "ndim", None) == 3: + zDim, yDim, xDim = array.shape + volume["dims"] = [int(zDim), int(yDim), int(xDim)] + volume["xyzDims"] = [int(xDim), int(yDim), int(zDim)] + + if "samplingRate" not in volume: + samplingRate = self._extractSamplingRate(props if isinstance(props, dict) else {}) + if samplingRate is not None: + self._attachSamplingRate(volume, samplingRate) + + try: + finite = np.asarray(array, dtype=np.float32) + finite = finite[np.isfinite(finite)] + if finite.size: + volume["min"] = float(np.min(finite)) + volume["max"] = float(np.max(finite)) + volume["mean"] = float(np.mean(finite)) + except Exception: + pass + + def _getStoredSet(self) -> Optional[Dict[str, Any]]: + if self._storedSet is not None: + return self._storedSet + + protocolDbId = self._resolveProtocolDbId() + if protocolDbId is None: + return None + + storedSet = self.db.fetchOne( + """ + SELECT id, "projectId", "protocolDbId", "objectId", "outputName", + "setClassName", "itemClassName", properties, "createdAt", "updatedAt" + FROM scipion_sets + WHERE "projectId" = %s + AND "protocolDbId" = %s + AND "outputName" = %s + """, + (self.projectId, protocolDbId, self.outputName), + ) + + if storedSet is None: + return None + + items = self.db.fetchAll( + """ + SELECT id, "setId", "scipionItemId", enabled, label, comment, + creation, "values", "createdAt", "updatedAt" + FROM scipion_set_items + WHERE "setId" = %s + ORDER BY "scipionItemId" ASC NULLS LAST, id ASC + """, + (storedSet["id"],), + ) + + storedSet["items"] = items or [] + self._storedSet = storedSet + return self._storedSet + + def _getStoredObjectTree(self) -> List[Dict[str, Any]]: + if self._storedObjectTree is not None: + return self._storedObjectTree + + protocolDbId = self._resolveProtocolDbId() + if protocolDbId is None: + self._storedObjectTree = [] + return self._storedObjectTree + + rootPath = str(self.outputName) + rows = self.db.fetchAll( + """ + SELECT id, "projectId", "protocolDbId", "scipionObjId", "parentObjectId", + name, path, "className", value, label, comment, creation, + metadata, "createdAt", "updatedAt" + FROM scipion_objects + WHERE "projectId" = %s + AND "protocolDbId" = %s + AND (path = %s OR path LIKE %s) + ORDER BY path ASC + """, + (self.projectId, protocolDbId, rootPath, rootPath + ".%"), + ) + + self._storedObjectTree = rows or [] + return self._storedObjectTree + + def _resolveProtocolDbId(self) -> Optional[int]: + if self._protocolDbId is not None: + return self._protocolDbId + + row = self.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE id = %s + AND "projectId" = %s + """, + (self.protocolId, self.projectId), + ) + + if row is not None: + self._protocolDbId = int(row["id"]) + return self._protocolDbId + + row = self.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE "projectId" = %s + AND "protocolId" = %s + """, + (self.projectId, str(self.protocolId)), + ) + + if row is not None: + self._protocolDbId = int(row["id"]) + return self._protocolDbId + + self.lastSkipReason = "protocol_not_found" + return None + + def _extractVolumeFile(self, values: Dict[str, Any]) -> Tuple[Optional[str], Optional[int]]: + raw = self._firstValueBySuffix( + values, + [ + "fileName", + "filename", + "location", + "path", + "stack", + ], + ) + + return self._parseLocation(raw) + + def _parseLocation(self, raw: Any) -> Tuple[Optional[str], Optional[int]]: + parsed = self._parseJsonValue(raw) + + if isinstance(parsed, dict): + pathValue = None + for key in ("fileName", "filename", "path", "location", "stack"): + if key in parsed: + pathValue = parsed.get(key) + break + + indexValue = None + for key in ("index", "locationIndex", "slice", "itemIndex"): + if key in parsed: + indexValue = parsed.get(key) + break + + fileName, embeddedIndex = self._parseLocation(pathValue) + locationIndex = self._toOptionalInt(indexValue) + if locationIndex is None: + locationIndex = embeddedIndex + + return fileName, locationIndex + + if isinstance(parsed, (list, tuple)): + if len(parsed) >= 2: + locationIndex = self._toOptionalInt(parsed[0]) + fileName = self._toText(parsed[1]) + return fileName, locationIndex + + if len(parsed) == 1: + return self._parseLocation(parsed[0]) + + return None, None + + text = self._toText(parsed) + if not text: + return None, None + + locationIndex = None + fileName = text + + if "@" in text: + indexText, pathText = text.split("@", 1) + parsedIndex = self._toOptionalInt(indexText) + if parsedIndex is not None: + locationIndex = parsedIndex + fileName = pathText + + return fileName, locationIndex + + def _extractDims(self, values: Dict[str, Any]) -> Optional[List[int]]: + raw = self._firstValueBySuffix( + values, + [ + "dim", + "dims", + "dimensions", + "volumeDim", + "volumeDims", + ], + ) + + parsed = self._parseJsonValue(raw) + if isinstance(parsed, dict): + parsed = [ + parsed.get("x") or parsed.get("X") or parsed.get("nx"), + parsed.get("y") or parsed.get("Y") or parsed.get("ny"), + parsed.get("z") or parsed.get("Z") or parsed.get("nz"), + ] + + if isinstance(parsed, (list, tuple)) and len(parsed) >= 3: + dims = [] + for value in parsed[:3]: + intValue = self._toOptionalInt(value) + if intValue is None: + return None + dims.append(intValue) + return dims + + return None + + def _extractSamplingRate(self, values: Any) -> Optional[float]: + if isinstance(values, dict): + raw = self._firstValueBySuffix( + values, + [ + "samplingRate", + "pixelSize", + "voxelSize", + "sampling", + "apix", + ], + ) + else: + raw = values + + parsed = self._parseJsonValue(raw) + + if isinstance(parsed, dict): + for key in ("x", "X", "samplingRate", "pixelSize", "voxelSize", "apix"): + value = parsed.get(key) + number = self._toOptionalFloat(value) + if number is not None: + return number + return None + + if isinstance(parsed, (list, tuple)) and parsed: + return self._toOptionalFloat(parsed[0]) + + return self._toOptionalFloat(parsed) + + def _attachSamplingRate(self, volume: Dict[str, Any], samplingRate: float) -> None: + samplingRate = float(samplingRate) + volume["samplingRate"] = samplingRate + volume["pixelSize"] = samplingRate + volume["voxelSize"] = [samplingRate, samplingRate, samplingRate] + + def _resolveExistingPath(self, fileName: Any) -> Optional[str]: + text = self._toText(fileName) + if not text: + return None + + path = Path(text).expanduser() + candidates = [] + + if path.is_absolute(): + candidates.append(path) + else: + candidates.append(path) + candidates.append(Path.cwd() / path) + + for candidate in candidates: + try: + resolved = candidate.resolve() + if resolved.exists(): + return str(resolved) + except Exception: + continue + + return None + + def _normalizeVolumeDisplayName( + self, + label: Any, + fileName: Optional[str], + index: int, + ) -> str: + labelText = self._toText(label) + + if labelText: + if self._looksLikePath(labelText): + return Path(labelText).name + return labelText + + if fileName: + return Path(str(fileName)).name + + return "Volume %s" % (int(index) + 1) + + def _looksLikePath(self, value: Any) -> bool: + text = str(value or "").strip() + if not text: + return False + + if "/" in text or "\\" in text: + return True + + suffix = Path(text).suffix.lower() + return suffix in { + ".mrc", + ".map", + ".mrcs", + ".rec", + ".ali", + ".vol", + ".spi", + ".stk", + } + + def _makeVolumeLabel(self, fileName: Optional[str], index: int) -> str: + if fileName: + try: + return Path(str(fileName)).name + except Exception: + pass + + return "Volume %s" % (int(index) + 1) + + def _getVolumeApiKeys(self, volume: Dict[str, Any]) -> Set[str]: + return { + text + for text in ( + self._toText(volume.get("id")), + self._toText(volume.get("index")), + ) + if text + } + + def _firstValueBySuffix(self, values: Dict[str, Any], suffixes: List[str]) -> Any: + if not isinstance(values, dict): + return None + + normalizedSuffixes = [ + self._normalizeKey(suffix) + for suffix in suffixes + if suffix is not None + ] + + for suffix in normalizedSuffixes: + for key, value in values.items(): + normalizedKey = self._normalizeKey(key) + if normalizedKey == suffix or normalizedKey.endswith(suffix): + return value + + return None + + def _normalizeJsonObject(self, value: Any) -> Dict[str, Any]: + parsed = self._parseJsonValue(value) + if isinstance(parsed, dict): + return parsed + return {} + + def _parseJsonValue(self, value: Any) -> Any: + if isinstance(value, (dict, list, tuple)): + return value + + if not isinstance(value, str): + return value + + text = value.strip() + if not text: + return value + + if not ( + text.startswith("{") + or text.startswith("[") + or text.startswith('"') + ): + return value + + try: + return json.loads(text) + except Exception: + return value + + def _normalizeKey(self, value: Any) -> str: + return ( + str(value or "") + .replace("_", "") + .replace(".", "") + .replace("-", "") + .lower() + ) + + def _toText(self, value: Any) -> Optional[str]: + if value is None: + return None + + getter = getattr(value, "get", None) + if callable(getter): + try: + value = getter() + except Exception: + return None + + text = str(value).strip() + return text or None + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + + try: + return int(value) + except Exception: + pass + + try: + return int(float(value)) + except Exception: + return None + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + + try: + number = float(value) + except Exception: + return None + + if not np.isfinite(number): + return None + + return number \ No newline at end of file diff --git a/dump.rdb b/dump.rdb new file mode 100644 index 00000000..052f3288 Binary files /dev/null and b/dump.rdb differ diff --git a/scipion_home/.plugins_revision b/scipion_home/.plugins_revision new file mode 100644 index 00000000..a6b4ce84 --- /dev/null +++ b/scipion_home/.plugins_revision @@ -0,0 +1 @@ +176 \ No newline at end of file diff --git a/scipion_home/.plugins_revision.lock b/scipion_home/.plugins_revision.lock new file mode 100644 index 00000000..e69de29b diff --git a/scipion_home/config/backup/scipion.conf.20260307213812 b/scipion_home/config/backup/scipion.conf.20260307213812 new file mode 100644 index 00000000..9af63a92 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260307213812 @@ -0,0 +1,25 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD = eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" + +CUDA_BIN = /usr/local/cuda-11.8/bin +CUDA_LIB = /usr/local/cuda-11.8/lib64 +MPI_BINDIR = /usr/lib64/mpi/gcc/openmpi/bin +MPI_LIBDIR = /usr/lib64/mpi/gcc/openmpi/lib +MPI_INCLUDE = /usr/lib64/mpi/gcc/openmpi/include +OPENCV = False +CUDA=True + + +[PLUGINS] +CRYOSPARC_HOME = /home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_USER = cfonseca@cnb.csic.es +CRYOSPARC_PASSWORD = 12345 +RELION_HOME = /home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME = /home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELION_ENV_ACTIVATION = conda activate relion-5.0 +#CHIMERA_HOME = /home/yunior/Yunior/Projects/Scipion/software/em/chimerax-1.4 +SCIPION_DEFAULT_EXECUTION_ACTION = 1 +#SCIPION_SPRITES_FILE = /home/yunior/Imágenes/sprites_16.png +#SCIPION_GUI_CANCEL_AUTO_REFRESH = True +WARP_FORCE_MRC_FLOAT32=1 + diff --git a/scipion_home/config/backup/scipion.conf.20260307214508 b/scipion_home/config/backup/scipion.conf.20260307214508 new file mode 100644 index 00000000..9af63a92 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260307214508 @@ -0,0 +1,25 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD = eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" + +CUDA_BIN = /usr/local/cuda-11.8/bin +CUDA_LIB = /usr/local/cuda-11.8/lib64 +MPI_BINDIR = /usr/lib64/mpi/gcc/openmpi/bin +MPI_LIBDIR = /usr/lib64/mpi/gcc/openmpi/lib +MPI_INCLUDE = /usr/lib64/mpi/gcc/openmpi/include +OPENCV = False +CUDA=True + + +[PLUGINS] +CRYOSPARC_HOME = /home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_USER = cfonseca@cnb.csic.es +CRYOSPARC_PASSWORD = 12345 +RELION_HOME = /home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME = /home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELION_ENV_ACTIVATION = conda activate relion-5.0 +#CHIMERA_HOME = /home/yunior/Yunior/Projects/Scipion/software/em/chimerax-1.4 +SCIPION_DEFAULT_EXECUTION_ACTION = 1 +#SCIPION_SPRITES_FILE = /home/yunior/Imágenes/sprites_16.png +#SCIPION_GUI_CANCEL_AUTO_REFRESH = True +WARP_FORCE_MRC_FLOAT32=1 + diff --git a/scipion_home/config/backup/scipion.conf.20260307215448 b/scipion_home/config/backup/scipion.conf.20260307215448 new file mode 100644 index 00000000..b2cbfa3a --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260307215448 @@ -0,0 +1,11 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260307215946 b/scipion_home/config/backup/scipion.conf.20260307215946 new file mode 100644 index 00000000..899c67d6 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260307215946 @@ -0,0 +1,12 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +IMOD_VIEWER_BINNING=2 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260307225850 b/scipion_home/config/backup/scipion.conf.20260307225850 new file mode 100644 index 00000000..899c67d6 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260307225850 @@ -0,0 +1,12 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +IMOD_VIEWER_BINNING=2 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260320071727 b/scipion_home/config/backup/scipion.conf.20260320071727 new file mode 100644 index 00000000..d7f4cfe1 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260320071727 @@ -0,0 +1,13 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +JAVA_MAX_MEMORY=8 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +IMOD_VIEWER_BINNING=2 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260320071826 b/scipion_home/config/backup/scipion.conf.20260320071826 new file mode 100644 index 00000000..cfda30d4 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260320071826 @@ -0,0 +1,13 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +IMOD_VIEWER_BINNING=2 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOLO_CUDA_LIB=/usr/local/cuda12.4/lib64 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260415111435 b/scipion_home/config/backup/scipion.conf.20260415111435 new file mode 100644 index 00000000..ba389111 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260415111435 @@ -0,0 +1,13 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +IMOD_VIEWER_BINNING=2 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOLO_CUDA_LIB=/usr/local/cuda-12.4/lib64 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260415111706 b/scipion_home/config/backup/scipion.conf.20260415111706 new file mode 100644 index 00000000..77bde170 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260415111706 @@ -0,0 +1,14 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_PRIORITY_PACKAGE_LIST="pwem tomo pwchem" + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_VIEWER_BINNING=2 +CRYOLO_CUDA_LIB=/usr/local/cuda-12.4/lib64 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260416211619 b/scipion_home/config/backup/scipion.conf.20260416211619 new file mode 100644 index 00000000..fe4f6c6c --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260416211619 @@ -0,0 +1,13 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_VIEWER_BINNING=2 +CRYOLO_CUDA_LIB=/usr/local/cuda-12.4/lib64 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260421114030 b/scipion_home/config/backup/scipion.conf.20260421114030 new file mode 100644 index 00000000..70fb22c7 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421114030 @@ -0,0 +1,13 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOLO_CUDA_LIB=/usr/local/cuda-12.4/lib64 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_VIEWER_BINNING=2 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260421114143 b/scipion_home/config/backup/scipion.conf.20260421114143 new file mode 100644 index 00000000..d0222551 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421114143 @@ -0,0 +1,16 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_VIEWER_BINNING=2 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +CRYOLO_CUDA_LIB=/usr/local/cuda-12.4/lib64 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260421114423 b/scipion_home/config/backup/scipion.conf.20260421114423 new file mode 100644 index 00000000..556be06f --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421114423 @@ -0,0 +1,9 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] diff --git a/scipion_home/config/backup/scipion.conf.20260421114636 b/scipion_home/config/backup/scipion.conf.20260421114636 new file mode 100644 index 00000000..829a3fe5 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421114636 @@ -0,0 +1,10 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +CHIMERA_OLD_BINARY_PATH=/home/yun diff --git a/scipion_home/config/backup/scipion.conf.20260421115104 b/scipion_home/config/backup/scipion.conf.20260421115104 new file mode 100644 index 00000000..556be06f --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421115104 @@ -0,0 +1,9 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] diff --git a/scipion_home/config/backup/scipion.conf.20260421115119 b/scipion_home/config/backup/scipion.conf.20260421115119 new file mode 100644 index 00000000..7721e45f --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421115119 @@ -0,0 +1,10 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +ATSAS_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/atsas-3.2.2 diff --git a/scipion_home/config/backup/scipion.conf.20260421122630 b/scipion_home/config/backup/scipion.conf.20260421122630 new file mode 100644 index 00000000..16d442ff --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421122630 @@ -0,0 +1,10 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +ATSAS_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/atsas-3.2.1 diff --git a/scipion_home/config/backup/scipion.conf.20260421122638 b/scipion_home/config/backup/scipion.conf.20260421122638 new file mode 100644 index 00000000..e55aa3df --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421122638 @@ -0,0 +1,12 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +ATSAS_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/atsas-3.2.1 +ARETOMO_BIN=AreTomo2_1.1.3_Cuda122 diff --git a/scipion_home/config/backup/scipion.conf.20260421124315 b/scipion_home/config/backup/scipion.conf.20260421124315 new file mode 100644 index 00000000..9af63a92 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421124315 @@ -0,0 +1,25 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD = eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" + +CUDA_BIN = /usr/local/cuda-11.8/bin +CUDA_LIB = /usr/local/cuda-11.8/lib64 +MPI_BINDIR = /usr/lib64/mpi/gcc/openmpi/bin +MPI_LIBDIR = /usr/lib64/mpi/gcc/openmpi/lib +MPI_INCLUDE = /usr/lib64/mpi/gcc/openmpi/include +OPENCV = False +CUDA=True + + +[PLUGINS] +CRYOSPARC_HOME = /home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_USER = cfonseca@cnb.csic.es +CRYOSPARC_PASSWORD = 12345 +RELION_HOME = /home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME = /home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELION_ENV_ACTIVATION = conda activate relion-5.0 +#CHIMERA_HOME = /home/yunior/Yunior/Projects/Scipion/software/em/chimerax-1.4 +SCIPION_DEFAULT_EXECUTION_ACTION = 1 +#SCIPION_SPRITES_FILE = /home/yunior/Imágenes/sprites_16.png +#SCIPION_GUI_CANCEL_AUTO_REFRESH = True +WARP_FORCE_MRC_FLOAT32=1 + diff --git a/scipion_home/config/backup/scipion.conf.20260421124338 b/scipion_home/config/backup/scipion.conf.20260421124338 new file mode 100644 index 00000000..c1930fd9 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421124338 @@ -0,0 +1,17 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=123456 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260422190617 b/scipion_home/config/backup/scipion.conf.20260422190617 new file mode 100644 index 00000000..eca1f6a8 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260422190617 @@ -0,0 +1,17 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260513164709 b/scipion_home/config/backup/scipion.conf.20260513164709 new file mode 100644 index 00000000..7e21c530 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260513164709 @@ -0,0 +1,18 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/imod-4.11.25/IMOD +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260513165026 b/scipion_home/config/backup/scipion.conf.20260513165026 new file mode 100644 index 00000000..0804695a --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260513165026 @@ -0,0 +1,18 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/Scipion/software/em/imod-4.11.25/IMOD diff --git a/scipion_home/config/backup/scipion.conf.20260513165138 b/scipion_home/config/backup/scipion.conf.20260513165138 new file mode 100644 index 00000000..d8edc6a1 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260513165138 @@ -0,0 +1,18 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD/bin diff --git a/scipion_home/config/backup/scipion.conf.20260514101059 b/scipion_home/config/backup/scipion.conf.20260514101059 new file mode 100644 index 00000000..0a836c93 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260514101059 @@ -0,0 +1,18 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD diff --git a/scipion_home/config/backup/scipion.conf.20260519133809 b/scipion_home/config/backup/scipion.conf.20260519133809 new file mode 100644 index 00000000..ad7ba238 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260519133809 @@ -0,0 +1,19 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CHIMERA_HOME=/home/yunior/Yunior/Projects/Scipion/software/em/chimerax-1.6.1 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260520151542 b/scipion_home/config/backup/scipion.conf.20260520151542 new file mode 100644 index 00000000..564e33e7 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260520151542 @@ -0,0 +1,18 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260520154051 b/scipion_home/config/backup/scipion.conf.20260520154051 new file mode 100644 index 00000000..b12f53e7 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260520154051 @@ -0,0 +1,19 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +CHIMERA_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/chimerax-1.6.1 diff --git a/scipion_home/config/backup/scipion.conf.20260520154226 b/scipion_home/config/backup/scipion.conf.20260520154226 new file mode 100644 index 00000000..8aed58e7 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260520154226 @@ -0,0 +1,20 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +CHIMERA_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/chimerax-1.6.1 diff --git a/scipion_home/config/backup/scipion.conf.20260520154544 b/scipion_home/config/backup/scipion.conf.20260520154544 new file mode 100644 index 00000000..3417393a --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260520154544 @@ -0,0 +1,20 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +CHIMERA_OLD_BINARY_PATH=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/chimerax-1.6.1/bin +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260520161814 b/scipion_home/config/backup/scipion.conf.20260520161814 new file mode 100644 index 00000000..99510a28 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260520161814 @@ -0,0 +1,20 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CHIMERA_HOME=/home/yunior/Yunior/Projects/Scipion/software/em/chimerax-1.6.1 \ No newline at end of file diff --git a/scipion_home/config/backup/scipion.conf.20260603210905 b/scipion_home/config/backup/scipion.conf.20260603210905 new file mode 100644 index 00000000..ae7e778a --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260603210905 @@ -0,0 +1,19 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260604132329 b/scipion_home/config/backup/scipion.conf.20260604132329 new file mode 100644 index 00000000..73d1bbf9 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260604132329 @@ -0,0 +1,20 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +CHIMERA_HOME=/home/yunior/Yunior/Projects/Scipion/software/em/chimerax-1.6.1/ diff --git a/scipion_home/config/backup/scipion.conf.20260604133119 b/scipion_home/config/backup/scipion.conf.20260604133119 new file mode 100644 index 00000000..3c66f08c --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260604133119 @@ -0,0 +1,21 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +NEXTPYP_HOME=/home/yunior/Yunior/Projects/nextPYP/nextPYP_0.7.3/config.toml +NEXTPYP_CONFIG=/home/yunior/Yunior/Projects/nextPYP/nextPYP_0.7.3 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/hosts (Copy).conf b/scipion_home/config/hosts (Copy).conf new file mode 100644 index 00000000..24c37002 --- /dev/null +++ b/scipion_home/config/hosts (Copy).conf @@ -0,0 +1,46 @@ +; This is a comment line +[localhost] +PARALLEL_COMMAND = mpirun -np %_(JOB_NODES)d %_(COMMAND)s +NAME = PBS/TORQUE +MANDATORY = False +SUBMIT_COMMAND = qsub %_(JOB_SCRIPT)s +SUBMIT_TEMPLATE = #!/bin/bash + ### Inherit all current environment variables + #PBS -V + ### Job name + #PBS -N %_(JOB_NAME)s + ### Queue name + ###PBS -q %_(JOB_QUEUE)s + ### Standard output and standard error messages + #PBS -k eo + ### Specify the number of nodes and thread (ppn) for your job. + #PBS -l nodes=%_(JOB_NODES)d:ppn=%_(JOB_THREADS)d + ### Tell PBS the anticipated run-time for your job, where walltime=HH:MM:SS + #PBS -l walltime=%_(JOB_HOURS)d:00:00 + # Use as working dir the path where qsub was launched + WORKDIR=$PBS_O_WORKDIR + ################################# + ### Set environment variable to know running mode is non interactive + export XMIPP_IN_QUEUE=1 + ### Switch to the working directory; + cd $WORKDIR + # Make a copy of PBS_NODEFILE + cp $PBS_NODEFILE %_(JOB_NODEFILE)s + # Calculate the number of processors allocated to this run. + NPROCS=`wc -l < $PBS_NODEFILE` + # Calculate the number of nodes allocated. + NNODES=`uniq $PBS_NODEFILE | wc -l` + ### Display the job context + echo Running on host `hostname` + echo Time is `date` + echo Working directory is `pwd` + echo Using ${NPROCS} processors across ${NNODES} nodes + echo PBS_NODEFILE: + cat $PBS_NODEFILE + ################################# + %_(JOB_COMMAND)s +CANCEL_COMMAND = canceljob %_(JOB_ID)s +CHECK_COMMAND = qstat %_(JOB_ID)s +; Next variable is used to provide a regex to check if a job is finished on a queue system +JOB_DONE_REGEX = "" +QUEUES = { "default": {} } diff --git a/scipion_home/config/hosts.conf b/scipion_home/config/hosts.conf new file mode 100644 index 00000000..27a580db --- /dev/null +++ b/scipion_home/config/hosts.conf @@ -0,0 +1,30 @@ +[localhost] +PARALLEL_COMMAND = mpirun -np %_(JOB_NODES)d %_(COMMAND)s +NAME = SLURM +MANDATORY = False +SUBMIT_COMMAND = sbatch --parsable %_(JOB_SCRIPT)s +CANCEL_COMMAND = scancel %_(JOB_ID)s +CHECK_COMMAND = squeue -h -j %_(JOB_ID)s +SUBMIT_PREFIX = scipion +SUBMIT_TEMPLATE = #!/bin/bash + ### Inherit all current environment variables + #SBATCH --export=ALL + ### Job name + #SBATCH -J %_(JOB_NAME)s + ### Outputs + #SBATCH -o %_(JOB_SCRIPT)s.out + #SBATCH -e %_(JOB_SCRIPT)s.err + #SBATCH --open-mode=append + ### Partition (queue) name + #SBATCH -p %_(JOB_QUEUE)s + ### Specify time, number of nodes (tasks), cores and memory(MB) for your job + #SBATCH --time=%_(JOB_TIME)s:00:00 --ntasks=%_(JOB_NODES)d --cpus-per-task=%_(JOB_THREADS)d --mem=%_(JOB_MEMORY)s --gres=gpu:%_(GPU_COUNT)s + + %_(JOB_COMMAND)s + +QUEUES = { + "debug": [["JOB_MEMORY", "8192", "Memory (MB)", "Select amount of memory in megabytes for this job"], + ["JOB_TIME", "2", "Time (hours)", "Select the expected time in hours for this job"], + ["GPU_COUNT", "1", "Number of GPUs", "Select number of GPUs for this job"], + ["QUEUE_FOR_JOBS", "N", "Use queue for jobs", "Send individual jobs to queue"]] + } diff --git a/scipion_home/config/scipion.conf b/scipion_home/config/scipion.conf new file mode 100644 index 00000000..df60dfb8 --- /dev/null +++ b/scipion_home/config/scipion.conf @@ -0,0 +1,21 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +NEXTPYP_HOME=/home/yunior/Yunior/Projects/nextPYP/nextPYP_0.7.3 +NEXTPYP_CONFIG=/home/yunior/Yunior/Projects/nextPYP/nextPYP_0.7.3/config.toml +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/scipionweb_environment.json b/scipion_home/config/scipionweb_environment.json new file mode 100644 index 00000000..45f9686a --- /dev/null +++ b/scipion_home/config/scipionweb_environment.json @@ -0,0 +1,4 @@ +{ + "PEPE_HOME": "/home/pepe_app", + "YUNIOR": "yunior/yun" +} diff --git a/scipion_home/software b/scipion_home/software new file mode 120000 index 00000000..22f6a51d --- /dev/null +++ b/scipion_home/software @@ -0,0 +1 @@ +/home/yunior/Yunior/Projects/Scipion/software/ \ No newline at end of file diff --git a/scipion_home/web/devel_plugins.json b/scipion_home/web/devel_plugins.json new file mode 100644 index 00000000..9a03260e --- /dev/null +++ b/scipion_home/web/devel_plugins.json @@ -0,0 +1,106 @@ +[ + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-aretomo", + "pipName": "scipion-em-aretomo", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-cistem", + "pipName": "scipion-em-cistem", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-cryosparc2", + "pipName": "scipion-em-cryosparc2", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-miffi", + "pipName": "scipion-em-miffi", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-motioncorr", + "pipName": "scipion-em-motioncorr", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-nextpyp", + "pipName": "scipion-em-nextpyp", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-relion", + "pipName": "scipion-em-relion", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-reliontomo", + "pipName": "scipion-em-reliontomo", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-sphire", + "pipName": "scipion-em-sphire", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-tomo", + "pipName": "scipion-em-tomo", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-warp", + "pipName": "scipion-em-warp", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-warp", + "pipName": "scipion-em-warp", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-xmipp", + "pipName": "scipion-em-xmipp", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + } +] \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 55cc7028..1e5690de 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ import pytest from fastapi import APIRouter, FastAPI from fastapi.testclient import TestClient -from starlette.responses import PlainTextResponse +from starlette.responses import PlainTextResponse, Response ROOT_DIR = Path(__file__).resolve().parents[1] if str(ROOT_DIR) not in sys.path: @@ -97,6 +97,7 @@ def _importMainModule(): if str(ROOT_DIR) not in sys.path: sys.path.insert(0, str(ROOT_DIR)) + def makeProjectOut(projectId: int = 1, name: str = "Demo Project", **overrides): # makeProjectOut payload = { @@ -126,6 +127,10 @@ def __init__(self): self.projectByIdResult = makeProjectOut() self.lastGetProjectByIdCall = None + self.projectDbRowResult = makeProjectOut() + self.lastGetProjectDbRowCall = None + self.lastGetProtocolsCall = None + self.protocolsResult = [{"id": 11, "name": "Prot A"}] self.logChannelsResult = [{"id": "stdout", "label": "Output"}] @@ -135,8 +140,9 @@ def __init__(self): } } self.lastPollLogsCall = None + self.lastListProtocolLogChannelsCall = None - self.resolveViewerResult = {"handled": True, "viewer": "volume"} + self.resolveViewerResult = {"handled": False} self.resolveViewerError = None self.lastResolveViewerCall = None @@ -149,6 +155,72 @@ def __init__(self): ) self.lastExportMetadataTableCall = None + self.consistencyResult = { + "ok": True, + "projectId": 1, + "summary": { + "runtimeProtocols": 2, + "postgresqlProtocols": 2, + "runtimeDependencies": 1, + "postgresqlDependencies": 1, + "issues": 0, + }, + "issues": { + "missingProtocols": [], + "extraProtocols": [], + "statusMismatches": [], + "missingDependencies": [], + "extraDependencies": [], + }, + } + self.lastValidateProjectPostgresqlConsistencyCall = None + + self.metadataTablesResult = [ + { + "name": "objects", + "alias": "Particles", + "rowCount": 1, + "hasColumnId": True, + } + ] + + self.lastListOutputMetadataTablesCall = None + + self.metadataTableSchemaResult = { + "name": "objects", + "alias": "Particles", + "hasColumnId": True, + "actions": ["Particle"], + "columns": [], + } + self.lastGetMetadataTableSchemaCall = None + + self.metadataTablePageResult = { + "pageNumber": 1, + "pageSize": 20, + "totalRows": 1, + "rows": [ + { + "id": 1, + "values": ["row-1"], + } + ], + } + self.metadataTableWindowResult = { + "offset": 10, + "limit": 25, + "totalRows": 1, + "rows": [{"id": 1, "values": ["row-1"]}], + } + self.lastGetMetadataTableWindowCall = None + self.lastGetMetadataTablePageCall = None + + self.renderMetadataImageCellResponse = PlainTextResponse( + "image-bytes", + media_type="image/png", + ) + self.lastRenderMetadataImageCellCall = None + self.listProjectsResult = [makeProjectOut()] self.lastListProjectsCall = None @@ -201,6 +273,7 @@ def __init__(self): self.lastSaveProtocolCall = None self.nextProtocolSuggestionsResult = [{"id": "next-1", "name": "Next protocol"}] + self.nextProtocolSuggestionsError = None self.lastGetNextProtocolSuggestionsCall = None self.renameProtocolError = None @@ -229,50 +302,443 @@ def __init__(self): self.stopProtocolError = None self.lastStopProtocolCall = None + self.volumeItemsResult = [ + { + "id": "vol-1", + "label": "Volume 1", + "fileName": "/tmp/volume.mrc", + } + ] + self.lastListOutputVolumesCall = None + + self.volumeInfoResult = { + "id": "vol-1", + "label": "Volume 1", + "dimensions": [64, 64, 64], + } + self.lastGetVolumeInfoCall = None + + self.volumeHistogramResult = { + "binEdges": [0.0, 1.0], + "counts": [10], + } + self.lastGetVolumeHistogramCall = None + + self.tiltSeriesResult = [ + { + "tiltSeriesId": "TS_001", + "label": "TS_001", + "nViews": 3, + } + ] + self.lastListOutputTiltSeriesCall = None + + self.tiltSeriesFramesResult = { + "tiltSeriesId": "TS_001", + "label": "TS_001", + "frames": [ + {"index": 0, "tiltAngle": -1.0}, + {"index": 1, "tiltAngle": 0.0}, + ], + } + self.lastGetTiltSeriesFramesCall = None + + self.ctftomoSeriesResult = [ + { + "tiltSeriesId": "TS_001", + "label": "TS_001", + "nViews": 3, + } + ] + self.lastListOutputCtftomoSeriesCall = None + + self.ctftomoSeriesViewsResult = { + "tiltSeriesId": "TS_001", + "label": "TS_001", + "frames": [ + {"index": 0, "defocusU": 10000.0}, + {"index": 1, "defocusU": 11000.0}, + ], + } + self.lastGetCtftomoSeriesViewsCall = None + + self.coords3dTomogramsResult = [ + { + "id": "tomo-1", + "name": "Tomogram 1", + "label": "tomo-1", + "dims": [64, 64, 64], + "voxelSize": [1.0, 1.0, 1.0], + } + ] + self.lastListCoordinates3dTomogramsCall = None + + self.coords3dPointsResult = [ + { + "id": 1, + "x": 10.0, + "y": 20.0, + "z": 30.0, + "tomoId": "tomo-1", + } + ] + self.lastGetCoordinates3dPointsCall = None + + self.integratedAnalyzeContextResult = { + "root": { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + }, + "links": {}, + "summaries": {}, + "relations": {"items": []}, + } + self.lastGetIntegratedAnalyzeContextCall = None + + self.projectTagsResult = [ + { + "id": "tag-1", + "title": "Good", + "description": "Good protocols", + "color": "#00ff00", + } + ] + self.lastListProjectTagsCall = None + + self.createProjectTagResult = { + "id": "tag-2", + "title": "New tag", + "description": None, + "color": "#ff0000", + } + self.lastCreateProjectTagCall = None + + self.updateProjectTagResult = { + "id": "tag-1", + "title": "Updated tag", + "description": "Updated", + "color": "#0000ff", + } + self.lastUpdateProjectTagCall = None + + self.deleteProjectTagResult = True + self.lastDeleteProjectTagCall = None + + self.protocolTagsResult = { + "protocolId": "2", + "protocolDbId": 22, + "tagIds": ["tag-1"], + } + self.lastListProtocolTagsCall = None + + self.setProtocolTagsResult = { + "protocolId": "2", + "protocolDbId": 22, + "tagIds": ["tag-1", "tag-2"], + } + self.lastSetProtocolTagsCall = None + + self.contextMenuVisibilityResult = { + "open": True, + "delete": True, + "manageTags": True, + } + self.lastGetContextMenuVisibilityPolicyCall = None + + self.fscRowsResult = { + "threshold": 0.143, + "curves": [ + { + "label": "FSC 1", + "resolution": 3.2, + "x": [0.01, 0.02], + "y": [0.95, 0.87], + } + ], + } + self.lastGetFscRowsCall = None + + self.projectEffectiveSettingsResult = { + "projectId": 1, + "settings": { + "user": {"protocolView": "tree"}, + "instance": {"executionMode": "local"}, + "host": {"queueSystem": "slurm"}, + }, + } + self.lastGetProjectEffectiveSettingsCall = None + + self.coords3dSliceResponse = Response( + content=b"slice-bytes", + media_type="image/png", + ) + self.lastRenderCoords3dTomogramSliceCall = None + + self.resolveAnalyzeViewerDecisionResult = { + "handled": False, + } + self.resolveViewerError = None + + self.lastResolveViewerCall = None + self.lastResolveAnalyzeViewerDecisionCall = None + + self.volumeSliceResponse = Response( + content=b"volume-slice-bytes", + media_type="image/png", + ) + self.lastRenderVolumeSliceCall = None + + self.volumeData3dResult = { + "dims": [2, 2, 2], + "values": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0], + } + self.lastGetVolumeData3dCall = None + + self.volumeSurfaceMeshResult = { + "vertices": [[0.0, 0.0, 0.0]], + "faces": [[0, 0, 0]], + "level": 0.5, + "volumeId": "vol-1", + } + self.lastGetVolumeSurfaceMeshCall = None + + self.tiltSeriesImageResponse = Response( + content=b"tilt-image-bytes", + media_type="image/png", + ) + self.lastRenderTiltSeriesImageCall = None + + self.tiltSeriesBatchResult = { + "tiltSeriesId": "TS_001", + "size": 256, + "fmt": "webp", + "applyTransform": True, + "items": [ + { + "index": 0, + "contentType": "image/png", + "dataUrl": "data:image/png;base64,AAAA", + "cache": "MISS", + } + ], + "errors": [], + } + self.lastRenderTiltSeriesImagesBatchCall = None + + self.ctftomoPsdResponse = Response( + content=b"ctftomo-psd-bytes", + media_type="image/png", + ) + self.lastRenderCtfTomoPsdImageCall = None + self.renderCtfTomoPsdImageError = None def listProjectWorkflows(self): if self.listProjectWorkflowsError is not None: raise self.listProjectWorkflowsError return self.listProjectWorkflowsResult - def getProjectById(self, mapper, projectId, currentUser, refresh=False, checkPid=False): + def getProjectById( + self, + mapper, + projectId, + currentUser, + refresh=False, + checkPid=False, + validateConsistency=False, + failOnConsistencyError=False, + ): self.lastGetProjectByIdCall = { "mapper": mapper, "projectId": projectId, "currentUser": currentUser, "refresh": refresh, "checkPid": checkPid, + "validateConsistency": validateConsistency, + "failOnConsistencyError": failOnConsistencyError, } return self.projectByIdResult - def getProtocols(self, mapper, projectId, currentUser): - return self.protocolsResult + def getProjectDbRow(self, mapper, projectId, currentUser): + self.lastGetProjectDbRowCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + } + return self.projectDbRowResult + + def validateProjectPostgresqlConsistency( + self, + mapper, + projectId, + currentUser, + refresh=True, + checkPid=True, + ): + self.lastValidateProjectPostgresqlConsistencyCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + "refresh": refresh, + "checkPid": checkPid, + } + return self.consistencyResult def listProjectLogChannelsService(self, projectId, protocolId): return self.logChannelsResult - def listProtocolLogChannelsService(self, projectId, protocolId): + def listProtocolLogChannelsService( + self, + projectId, + protocolId, + mapper=None, + currentUser=None, + ): + self.lastListProtocolLogChannelsCall = { + "projectId": projectId, + "protocolId": protocolId, + "mapper": mapper, + "currentUser": currentUser, + } return self.logChannelsResult - def pollProtocolLogsService(self, projectId, protocolId, offsets, maxBytes, maxLines): + def pollProtocolLogsService( + self, + projectId, + protocolId, + offsets, + maxBytes, + maxLines, + mapper=None, + currentUser=None, + ): self.lastPollLogsCall = { "projectId": projectId, "protocolId": protocolId, "offsets": dict(offsets), "maxBytes": maxBytes, "maxLines": maxLines, + "mapper": mapper, + "currentUser": currentUser, } return self.pollLogsResult - def resolveAnalyzeViewerDecision(self, projectId, protocolId, ctx): - self.lastResolveViewerCall = { + def listOutputMetadataTablesService( + self, + projectId, + protocolId, + outputName, + mapper, + ): + self.lastListOutputMetadataTablesCall = { "projectId": projectId, "protocolId": protocolId, - "ctx": ctx, + "outputName": outputName, + "mapper": mapper, } - if self.resolveViewerError is not None: - raise self.resolveViewerError - return self.resolveViewerResult + return self.metadataTablesResult + + def getMetadataTableSchemaService( + self, + projectId, + protocolId, + outputName, + tableName, + mapper, + ): + self.lastGetMetadataTableSchemaCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tableName": tableName, + "mapper": mapper, + } + return self.metadataTableSchemaResult + + def getMetadataTablePageService( + self, + projectId, + protocolId, + outputName, + tableName, + page, + pageSize, + sortBy, + asc, + selectionOnly, + mapper, + ): + self.lastGetMetadataTablePageCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tableName": tableName, + "page": page, + "pageSize": pageSize, + "sortBy": sortBy, + "asc": asc, + "selectionOnly": selectionOnly, + "mapper": mapper, + } + return self.metadataTablePageResult + + def getMetadataTableWindowService( + self, + projectId, + protocolId, + outputName, + tableName, + offset, + limit, + selectionOnly, + sortBy, + asc, + mapper, + ): + self.lastGetMetadataTableWindowCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tableName": tableName, + "offset": offset, + "limit": limit, + "selectionOnly": selectionOnly, + "sortBy": sortBy, + "asc": asc, + "mapper": mapper, + } + return self.metadataTableWindowResult + + def renderMetadataImageCellService( + self, + projectId, + protocolId, + outputName, + tableName, + rowId, + rowIndex, + columnName, + size, + applyTransform, + inline, + fmt, + mapper, + ): + self.lastRenderMetadataImageCellCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tableName": tableName, + "rowId": rowId, + "rowIndex": rowIndex, + "columnName": columnName, + "size": size, + "applyTransform": applyTransform, + "inline": inline, + "fmt": fmt, + "mapper": mapper, + } + return self.renderMetadataImageCellResponse def runMetadataTableActionService( self, @@ -308,6 +774,7 @@ def exportMetadataTableService( fmt, selectionOnly, ids, + mapper, ): self.lastExportMetadataTableCall = { "projectId": projectId, @@ -317,6 +784,7 @@ def exportMetadataTableService( "fmt": fmt, "selectionOnly": selectionOnly, "ids": ids, + "mapper": mapper, } return self.exportMetadataTableResponse @@ -378,6 +846,14 @@ def listProjectShares(self, mapper, projectId, currentUser): } return self.projectSharesResult + def getProtocols(self, mapper, projectId, currentUser): + self.lastGetProtocolsCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + } + return self.protocolsResult + def syncProjectGraphAfterMutation(self, mapper, projectId, actionLabel, refresh=True, checkPid=True): self.lastSyncProjectGraphAfterMutationCall = { "mapper": mapper, @@ -401,10 +877,11 @@ def applyWorkflowToProject(self, mapper, projectId, workflowId, currentUser): raise self.applyWorkflowError return self.applyWorkflowResult - def getProtocolParams(self, projectId, protocolId): + def getProtocolParams(self, projectId, protocolId, mapper=None): self.lastGetProtocolParamsCall = { "projectId": projectId, "protocolId": protocolId, + "mapper": mapper, } return self.protocolParamsResult @@ -439,14 +916,23 @@ def saveProtocol(self, mapper, projectId, protocolId, protocolClassName, params) raise self.saveProtocolError return self.saveProtocolResult - def getNextProtocolSuggestions(self, protocolId): - self.lastGetNextProtocolSuggestionsCall = {"protocolId": protocolId} + def getNextProtocolSuggestions(self, protocolId, mapper=None, projectId=None): + self.lastGetNextProtocolSuggestionsCall = { + "protocolId": protocolId, + "mapper": mapper, + "projectId": projectId, + } + if self.nextProtocolSuggestionsError is not None: + raise self.nextProtocolSuggestionsError return self.nextProtocolSuggestionsResult - def renameProtocol(self, protocolId, newName): + def renameProtocol(self, mapper, projectId, protocolId, newName, newComment=""): self.lastRenameProtocolCall = { + "mapper": mapper, + "projectId": projectId, "protocolId": protocolId, "newName": newName, + "newComment": newComment, } if self.renameProtocolError is not None: raise self.renameProtocolError @@ -470,8 +956,12 @@ def deleteProtocol(self, mapper, projectId, protocolIds): if self.deleteProtocolError is not None: raise self.deleteProtocolError - def restartProtocolAll(self, protocolId): - self.lastRestartProtocolAllCall = {"protocolId": protocolId} + def restartProtocolAll(self, mapper, projectId, protocolId): + self.lastRestartProtocolAllCall = { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + } if self.restartProtocolAllError is not None: raise self.restartProtocolAllError return self.restartProtocolAllResult @@ -486,16 +976,507 @@ def continueProtocolAll(self, mapper, projectId, protocolId, currentUser): if self.continueProtocolAllError is not None: raise self.continueProtocolAllError - def resetProtocolFrom(self, protocolId): - self.lastResetProtocolFromCall = {"protocolId": protocolId} + def resetProtocolFrom(self, mapper, projectId, protocolId): + self.lastResetProtocolFromCall = { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + } if self.resetProtocolFromError is not None: raise self.resetProtocolFromError - def stopProtocol(self, protocolIds): - self.lastStopProtocolCall = {"protocolIds": protocolIds} + def stopProtocol(self, mapper, projectId, protocolIds): + self.lastStopProtocolCall = { + "mapper": mapper, + "projectId": projectId, + "protocolIds": protocolIds, + } if self.stopProtocolError is not None: raise self.stopProtocolError + def listOutputVolumesService( + self, + projectId, + protocolId, + outputName, + mapper=None, + ): + self.lastListOutputVolumesCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + return self.volumeItemsResult + + def getVolumeInfoService( + self, + projectId, + protocolId, + outputName, + volumeId, + mapper=None, + ): + self.lastGetVolumeInfoCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "volumeId": volumeId, + "mapper": mapper, + } + return self.volumeInfoResult + + def getVolumeHistogramService( + self, + projectId, + protocolId, + outputName, + volumeId, + bins=128, + mapper=None, + ): + self.lastGetVolumeHistogramCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "volumeId": volumeId, + "bins": bins, + "mapper": mapper, + } + return self.volumeHistogramResult + + def listOutputTiltSeriesService( + self, + projectId, + protocolId, + outputName, + mapper=None, + ): + self.lastListOutputTiltSeriesCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + return self.tiltSeriesResult + + def getTiltSeriesFramesService( + self, + projectId, + protocolId, + outputName, + tiltSeriesId, + mapper=None, + ): + self.lastGetTiltSeriesFramesCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tiltSeriesId": tiltSeriesId, + "mapper": mapper, + } + return self.tiltSeriesFramesResult + + def listOutputCtftomoSeriesService( + self, + projectId, + protocolId, + outputName, + mapper=None, + ): + self.lastListOutputCtftomoSeriesCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + return self.ctftomoSeriesResult + + def getCtftomoSeriesViewsService( + self, + projectId, + protocolId, + outputName, + tiltSeriesId, + mapper=None, + ): + self.lastGetCtftomoSeriesViewsCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tiltSeriesId": tiltSeriesId, + "mapper": mapper, + } + return self.ctftomoSeriesViewsResult + + def listCoordinates3dTomogramsService( + self, + projectId, + protocolId, + outputName, + mapper=None, + ): + self.lastListCoordinates3dTomogramsCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + return self.coords3dTomogramsResult + + def getCoordinates3dPointsService( + self, + projectId, + protocolId, + outputName, + tomogramId, + mapper=None, + ): + self.lastGetCoordinates3dPointsCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tomogramId": tomogramId, + "mapper": mapper, + } + return self.coords3dPointsResult + + def getIntegratedAnalyzeContextService( + self, + projectId, + protocolId, + outputName, + mapper=None, + ): + self.lastGetIntegratedAnalyzeContextCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + return self.integratedAnalyzeContextResult + + def listProjectTags(self, mapper, projectId, currentUser): + self.lastListProjectTagsCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + } + return self.projectTagsResult + + def createProjectTag(self, mapper, projectId, currentUser, payload): + self.lastCreateProjectTagCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + "payload": payload, + } + return self.createProjectTagResult + + def updateProjectTag(self, mapper, projectId, tagId, currentUser, payload): + self.lastUpdateProjectTagCall = { + "mapper": mapper, + "projectId": projectId, + "tagId": tagId, + "currentUser": currentUser, + "payload": payload, + } + return self.updateProjectTagResult + + def deleteProjectTag(self, mapper, projectId, tagId, currentUser): + self.lastDeleteProjectTagCall = { + "mapper": mapper, + "projectId": projectId, + "tagId": tagId, + "currentUser": currentUser, + } + return self.deleteProjectTagResult + + def listProtocolTags(self, mapper, projectId, protocolId, currentUser): + self.lastListProtocolTagsCall = { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + "currentUser": currentUser, + } + return self.protocolTagsResult + + def setProtocolTags(self, mapper, projectId, protocolId, tagIds, currentUser): + self.lastSetProtocolTagsCall = { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + "tagIds": tagIds, + "currentUser": currentUser, + } + return self.setProtocolTagsResult + + def getContextMenuVisibilityPolicy(self): + self.lastGetContextMenuVisibilityPolicyCall = {} + return self.contextMenuVisibilityResult + + def getFscRowsService( + self, + projectId, + protocolId, + outputName, + mapper=None, + ): + self.lastGetFscRowsCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + return self.fscRowsResult + + def getProjectEffectiveSettings(self, mapper, projectId, currentUser): + self.lastGetProjectEffectiveSettingsCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + } + return self.projectEffectiveSettingsResult + + def renderCoords3dTomogramSliceService( + self, + projectId, + protocolId, + outputName, + tomogramId, + sliceIndex, + axis="z", + colormap=None, + normalize="minmax", + scale=1.0, + inline=True, + fmt="webp", + thumb=None, + fast=True, + quality=75, + mapper=None, + ): + self.lastRenderCoords3dTomogramSliceCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tomogramId": tomogramId, + "sliceIndex": sliceIndex, + "axis": axis, + "colormap": colormap, + "normalize": normalize, + "scale": scale, + "inline": inline, + "fmt": fmt, + "thumb": thumb, + "fast": fast, + "quality": quality, + "mapper": mapper, + } + return self.coords3dSliceResponse + + def resolveAnalyzeViewerDecision( + self, + projectId, + protocolId, + ctx, + mapper=None, + ): + call = { + "projectId": projectId, + "protocolId": protocolId, + "ctx": ctx, + "mapper": mapper, + } + + self.lastResolveViewerCall = call + self.lastResolveAnalyzeViewerDecisionCall = call + + if self.resolveViewerError is not None: + raise self.resolveViewerError + + return self.resolveAnalyzeViewerDecisionResult + + def renderVolumeSliceService( + self, + projectId, + protocolId, + outputName, + volumeId, + sliceIndex, + axis, + colormap, + normalize, + scale, + inline, + fmt="webp", + thumb=None, + fast=True, + quality=75, + mapper=None, + ): + self.lastRenderVolumeSliceCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "volumeId": volumeId, + "sliceIndex": sliceIndex, + "axis": axis, + "colormap": colormap, + "normalize": normalize, + "scale": scale, + "inline": inline, + "fmt": fmt, + "thumb": thumb, + "fast": fast, + "quality": quality, + "mapper": mapper, + } + return self.volumeSliceResponse + + def getVolumeData3dService( + self, + projectId, + protocolId, + outputName, + volumeId, + maxDim=160, + method="binning", + mapper=None, + ): + self.lastGetVolumeData3dCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "volumeId": volumeId, + "maxDim": maxDim, + "method": method, + "mapper": mapper, + } + return self.volumeData3dResult + + def getVolumeSurfaceMesh( + self, + projectId, + protocolId, + outputName, + volumeId, + level=None, + maxDim=192, + method="stride", + maxTriangles=350000, + currentUser=None, + mapper=None, + ): + self.lastGetVolumeSurfaceMeshCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "volumeId": volumeId, + "level": level, + "maxDim": maxDim, + "method": method, + "maxTriangles": maxTriangles, + "currentUser": currentUser, + "mapper": mapper, + } + return self.volumeSurfaceMeshResult + + def renderTiltSeriesImageService( + self, + projectId, + protocolId, + outputName, + tiltSeriesId, + index=0, + size=1024, + fmt="png", + applyTransform=True, + inline=True, + requestHeaders=None, + mapper=None, + ): + self.lastRenderTiltSeriesImageCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tiltSeriesId": tiltSeriesId, + "index": index, + "size": size, + "fmt": fmt, + "applyTransform": applyTransform, + "inline": inline, + "requestHeaders": requestHeaders, + "mapper": mapper, + } + return self.tiltSeriesImageResponse + + def renderTiltSeriesImagesBatchService( + self, + projectId, + protocolId, + outputName, + tiltSeriesId, + indices, + size=512, + fmt="webp", + applyTransform=True, + inline=True, + requestHeaders=None, + mapper=None, + ): + self.lastRenderTiltSeriesImagesBatchCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tiltSeriesId": tiltSeriesId, + "indices": indices, + "size": size, + "fmt": fmt, + "applyTransform": applyTransform, + "inline": inline, + "requestHeaders": requestHeaders, + "mapper": mapper, + } + return self.tiltSeriesBatchResult + + def renderCtfTomoPsdImageService( + self, + projectId, + protocolId, + outputName, + psdPath, + size=1024, + fmt="png", + inline=True, + index=0, + quality=75, + applyTransform=False, + rot=None, + shifts=None, + mapper=None, + ): + self.lastRenderCtfTomoPsdImageCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "psdPath": psdPath, + "size": size, + "fmt": fmt, + "inline": inline, + "index": index, + "quality": quality, + "applyTransform": applyTransform, + "rot": rot, + "shifts": shifts, + "mapper": mapper, + } + + if self.renderCtfTomoPsdImageError is not None: + raise self.renderCtfTomoPsdImageError + + return self.ctftomoPsdResponse + @pytest.fixture def authTestEnv(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): @@ -751,4 +1732,5 @@ async def fakeSendVerificationEmail(email: str, code: str): with TestClient(app) as client: yield client - app.dependency_overrides.clear() \ No newline at end of file + app.dependency_overrides.clear() + diff --git a/tests/integration/api/test_project_router_protocol_steps.py b/tests/integration/api/test_project_router_protocol_steps.py index 8e5597f6..01171e12 100644 --- a/tests/integration/api/test_project_router_protocol_steps.py +++ b/tests/integration/api/test_project_router_protocol_steps.py @@ -69,6 +69,8 @@ def __init__(self): } self.updateProtocolStepStatusError = None self.lastUpdateProtocolStepStatusCall = None + self.projectDbRowResult = {"id": 1, "name": "Demo project"} + self.lastGetProjectDbRowCall = None def getProjectById(self, mapper, projectId, currentUser, refresh=False, checkPid=False): self.lastGetProjectByIdCall = { @@ -88,6 +90,14 @@ def listProtocolStepsService(self, mapper, projectId, protocolId): } return self.protocolStepsResult + def getProjectDbRow(self, mapper, projectId, currentUser): + self.lastGetProjectDbRowCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + } + return self.projectDbRowResult + def updateProtocolStepStatusService( self, mapper, @@ -142,7 +152,7 @@ def test_ListProtocolStepsReturns404WhenProjectMissing( protocolStepsClient, fakeProtocolStepsProjectService, ): - fakeProtocolStepsProjectService.projectByIdResult = None + fakeProtocolStepsProjectService.projectDbRowResult = None response = protocolStepsClient.get("/projects/1/protocols/10/steps") @@ -164,6 +174,16 @@ def test_ListProtocolStepsDelegatesToService( "projectId": 1, "protocolId": 10, } + assert fakeProtocolStepsProjectService.lastGetProjectDbRowCall == { + "mapper": fakeProjectMapper, + "projectId": 1, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + } + assert fakeProtocolStepsProjectService.lastGetProjectByIdCall is None def test_UpdateProtocolStepStatusReturns404WhenProjectMissing( diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 888bc1cb..c74a6653 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -23,6 +23,82 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ****************************************************************************** +import pytest +from fastapi import HTTPException + + +@pytest.mark.parametrize( + ("method", "url"), + [ + ( + "get", + "/projects/1/protocols/2/outputs/out/metadata/tables", + ), + ( + "get", + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/schema", + ), + ( + "get", + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/page", + ), + ( + "get", + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/export", + ), + ( + "get", + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/image?column=stack", + ), + ], +) +def test_MetadataReadEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + method, + url, +): + fakeProjectService.projectDbRowResult = None + + response = getattr(projectClient, method)(url) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_GetMetadataTableWindowDelegatesMapperToService(projectClient, fakeProjectService): + fakeProjectService.metadataTableWindowResult = { + "offset": 10, + "limit": 25, + "totalRows": 1, + "rows": [{"id": 1, "values": ["row-1"]}], + } + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/rows" + "?offset=10&limit=25&sortBy=id&asc=false&selectionOnly=true" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.metadataTableWindowResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetMetadataTableWindowCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tableName": "objects", + "offset": 10, + "limit": 25, + "selectionOnly": True, + "sortBy": "id", + "asc": False, + "mapper": fakeProjectService.lastGetMetadataTableWindowCall["mapper"], + } + + def test_ListProjectWorkflowsReturnsServiceResult(projectClient): response = projectClient.get("/projects/workflows") @@ -63,16 +139,39 @@ def test_GetProjectCallsServiceWithRefreshAndCheckPid(projectClient, fakeProject }, "refresh": True, "checkPid": True, + "validateConsistency": False, + "failOnConsistencyError": False, + } + + +def test_CheckProjectPostgresqlConsistencyCallsService(projectClient, fakeProjectService): + response = projectClient.post( + "/projects/1/consistency/check?refresh=false&checkPid=false" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.consistencyResult + assert fakeProjectService.lastValidateProjectPostgresqlConsistencyCall == { + "mapper": fakeProjectService.lastValidateProjectPostgresqlConsistencyCall["mapper"], + "projectId": 1, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + "refresh": False, + "checkPid": False, } def test_LoadProtocolsReturns404WhenProjectDoesNotExist(projectClient, fakeProjectService): - fakeProjectService.projectByIdResult = None + fakeProjectService.projectDbRowResult = None response = projectClient.get("/projects/1/protocols") assert response.status_code == 404 assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectByIdCall is None def test_LoadProtocolsReturns404WhenProtocolsAreMissing(projectClient, fakeProjectService): @@ -91,6 +190,8 @@ def test_LoadProtocolsReturnsProtocols(projectClient, fakeProjectService): assert response.status_code == 200 assert response.json() == [{"id": 11, "name": "Prot A"}] + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None def test_ListProtocolLogChannelsNormalizesStringAndDictItems(projectClient, fakeProjectService): @@ -138,6 +239,8 @@ def test_PollProtocolLogsNormalizesOffsetsAndIncludesDynamicChannels(projectClie ] } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None assert fakeProjectService.lastPollLogsCall == { "projectId": 1, "protocolId": 22, @@ -148,6 +251,12 @@ def test_PollProtocolLogsNormalizesOffsetsAndIncludesDynamicChannels(projectClie }, "maxBytes": 123, "maxLines": 45, + "mapper": fakeProjectService.lastPollLogsCall["mapper"], + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, } @@ -158,11 +267,12 @@ def test_ResolveAnalyzeViewerUnwrapsCtx(projectClient, fakeProjectService): ) assert response.status_code == 200 - assert response.json() == {"handled": True, "viewer": "volume"} + assert response.json() == {"handled": False} assert fakeProjectService.lastResolveViewerCall == { "projectId": 1, "protocolId": 22, "ctx": {"outputName": "particles", "outputClass": "SetOfParticles"}, + "mapper": fakeProjectService.lastResolveViewerCall["mapper"], } @@ -179,6 +289,72 @@ def test_ResolveAnalyzeViewerReturnsHandledFalseOnUnexpectedError(projectClient, "handled": False, "message": "viewer failed", } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastResolveViewerCall == { + "projectId": 1, + "protocolId": 22, + "ctx": {"outputName": "particles"}, + "mapper": fakeProjectService.lastResolveViewerCall["mapper"], + } + + +def test_ListMetadataTablesDelegatesMapperToService(projectClient, fakeProjectService): + fakeProjectService.metadataTablesResult = [ + { + "name": "objects", + "alias": "Particles", + "rowCount": 3, + "hasColumnId": True, + }, + { + "name": "Properties", + "alias": "Properties", + "rowCount": 2, + "hasColumnId": False, + }, + ] + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/metadata/tables" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.metadataTablesResult + assert fakeProjectService.lastListOutputMetadataTablesCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastListOutputMetadataTablesCall["mapper"], + } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_GetMetadataTableSchemaDelegatesMapperToService(projectClient, fakeProjectService): + fakeProjectService.metadataTableSchemaResult = { + "name": "objects", + "alias": "Particles", + "hasColumnId": True, + "actions": ["Particle"], + "columns": [], + } + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/schema" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.metadataTableSchemaResult + assert fakeProjectService.lastGetMetadataTableSchemaCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tableName": "objects", + "mapper": fakeProjectService.lastGetMetadataTableSchemaCall["mapper"], + } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None def test_RunMetadataTableActionRejectsMissingIds(projectClient): @@ -229,6 +405,37 @@ def test_RunMetadataTableActionUsesDefaultSubsetNameAndNormalizesServiceResult( } +def test_GetMetadataTablePageDelegatesMapperToService(projectClient, fakeProjectService): + fakeProjectService.metadataTablePageResult = { + "pageNumber": 2, + "pageSize": 50, + "totalRows": 1, + "rows": [{"id": 1, "values": ["row-1"]}], + } + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/page" + "?page=2&pageSize=50&sortBy=id&asc=false&selectionOnly=true" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.metadataTablePageResult + assert fakeProjectService.lastGetMetadataTablePageCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tableName": "objects", + "page": 2, + "pageSize": 50, + "sortBy": "id", + "asc": False, + "selectionOnly": True, + "mapper": fakeProjectService.lastGetMetadataTablePageCall["mapper"], + } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + def test_ExportMetadataTableRejectsInvalidIds(projectClient): response = projectClient.get( "/projects/1/protocols/2/outputs/out/metadata/tables/table/export?ids=1,abc,3" @@ -254,4 +461,956 @@ def test_ExportMetadataTableParsesIdsAndDelegatesToService(projectClient, fakePr "fmt": "csv", "selectionOnly": False, "ids": [1, 2, 3], - } \ No newline at end of file + "mapper": fakeProjectService.lastExportMetadataTableCall["mapper"], + } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_RenderMetadataImageCellDelegatesMapperToService(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/image" + "?rowId=7&rowIndex=3&column=stack&size=128" + "&applyTransform=true&inline=false&fmt=jpeg" + ) + + assert response.status_code == 200 + assert response.text == "image-bytes" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + call = fakeProjectService.lastRenderMetadataImageCellCall + assert str(call["rowId"]) == "7" + assert call == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tableName": "objects", + "rowId": call["rowId"], + "rowIndex": 3, + "columnName": "stack", + "size": 128, + "applyTransform": True, + "inline": False, + "fmt": "jpeg", + "mapper": call["mapper"], + } + + +def test_ListOutputVolumesUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/volumes" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.volumeItemsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListOutputVolumesCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastListOutputVolumesCall["mapper"], + } + + +def test_GetVolumeInfoUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/volumes/vol-1/info" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.volumeInfoResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetVolumeInfoCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "volumeId": "vol-1", + "mapper": fakeProjectService.lastGetVolumeInfoCall["mapper"], + } + + +def test_GetVolumeHistogramUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/volumes/vol-1/histogram?bins=32" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.volumeHistogramResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetVolumeHistogramCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "volumeId": "vol-1", + "bins": 32, + "mapper": fakeProjectService.lastGetVolumeHistogramCall["mapper"], + } + + +@pytest.mark.parametrize( + "url", + [ + "/projects/1/protocols/2/outputs/out/volumes", + "/projects/1/protocols/2/outputs/out/volumes/vol-1/info", + "/projects/1/protocols/2/outputs/out/volumes/vol-1/histogram", + ], +) +def test_VolumeReadEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + url, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get(url) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_ListOutputTiltSeriesUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/tiltseries" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.tiltSeriesResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListOutputTiltSeriesCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastListOutputTiltSeriesCall["mapper"], + } + + +def test_GetTiltSeriesFramesUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/tiltseries/TS_001/frames" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.tiltSeriesFramesResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetTiltSeriesFramesCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tiltSeriesId": "TS_001", + "mapper": fakeProjectService.lastGetTiltSeriesFramesCall["mapper"], + } + + +def test_ListCtftomoSeriesUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/ctftomo" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.ctftomoSeriesResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListOutputCtftomoSeriesCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastListOutputCtftomoSeriesCall["mapper"], + } + + +def test_GetCtftomoSeriesViewsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/ctftomo/TS_001/views" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.ctftomoSeriesViewsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetCtftomoSeriesViewsCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tiltSeriesId": "TS_001", + "mapper": fakeProjectService.lastGetCtftomoSeriesViewsCall["mapper"], + } + + +@pytest.mark.parametrize( + "url", + [ + "/projects/1/protocols/2/outputs/out/tiltseries", + "/projects/1/protocols/2/outputs/out/tiltseries/TS_001/frames", + "/projects/1/protocols/2/outputs/out/ctftomo", + "/projects/1/protocols/2/outputs/out/ctftomo/TS_001/views", + ], +) +def test_TomographyReadEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + url, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get(url) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_ListCoordinates3dTomogramsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/coords3d/tomograms" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.coords3dTomogramsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListCoordinates3dTomogramsCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastListCoordinates3dTomogramsCall["mapper"], + } + + +def test_GetCoordinates3dPointsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/coords3d/tomograms/tomo-1" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.coords3dPointsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetCoordinates3dPointsCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tomogramId": "tomo-1", + "mapper": fakeProjectService.lastGetCoordinates3dPointsCall["mapper"], + } + + +def test_GetIntegratedAnalyzeContextUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/integrated-context" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.integratedAnalyzeContextResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetIntegratedAnalyzeContextCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastGetIntegratedAnalyzeContextCall["mapper"], + } + + +@pytest.mark.parametrize( + "url", + [ + "/projects/1/protocols/2/outputs/out/coords3d/tomograms", + "/projects/1/protocols/2/outputs/out/coords3d/tomograms/tomo-1", + "/projects/1/protocols/2/outputs/out/integrated-context", + ], +) +def test_Coordinates3dReadEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + url, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get(url) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_ListProjectTagsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get("/projects/1/tags") + + assert response.status_code == 200 + assert response.json() == fakeProjectService.projectTagsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListProjectTagsCall["projectId"] == 1 + + +def test_CreateProjectTagUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.post( + "/projects/1/tags", + json={ + "title": "New tag", + "description": None, + "color": "#ff0000", + }, + ) + + assert response.status_code == 201 + assert response.json() == fakeProjectService.createProjectTagResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastCreateProjectTagCall["projectId"] == 1 + assert fakeProjectService.lastCreateProjectTagCall["payload"].title == "New tag" + + +def test_UpdateProjectTagUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.put( + "/projects/1/tags/tag-1", + json={ + "title": "Updated tag", + "description": "Updated", + "color": "#0000ff", + }, + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.updateProjectTagResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastUpdateProjectTagCall["projectId"] == 1 + assert fakeProjectService.lastUpdateProjectTagCall["tagId"] == "tag-1" + + +def test_DeleteProjectTagUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.delete("/projects/1/tags/tag-1") + + assert response.status_code == 200 + assert response.json() == {"success": True} + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastDeleteProjectTagCall["projectId"] == 1 + assert fakeProjectService.lastDeleteProjectTagCall["tagId"] == "tag-1" + + +def test_ListProtocolTagsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get("/projects/1/protocols/2/tags") + + assert response.status_code == 200 + assert response.json() == fakeProjectService.protocolTagsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListProtocolTagsCall["projectId"] == 1 + assert fakeProjectService.lastListProtocolTagsCall["protocolId"] == 2 + + +def test_SetProtocolTagsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.put( + "/projects/1/protocols/2/tags", + json={"tagIds": ["tag-1", "tag-2"]}, + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.setProtocolTagsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastSetProtocolTagsCall["projectId"] == 1 + assert fakeProjectService.lastSetProtocolTagsCall["protocolId"] == 2 + assert fakeProjectService.lastSetProtocolTagsCall["tagIds"] == ["tag-1", "tag-2"] + + +def test_GetContextMenuVisibilityUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get("/projects/1/context-menu-visibility") + + assert response.status_code == 200 + assert response.json() == fakeProjectService.contextMenuVisibilityResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetContextMenuVisibilityPolicyCall == {} + + +@pytest.mark.parametrize( + ("method", "url", "payload"), + [ + ("get", "/projects/1/tags", None), + ("post", "/projects/1/tags", {"title": "New tag"}), + ("put", "/projects/1/tags/tag-1", {"title": "Updated"}), + ("delete", "/projects/1/tags/tag-1", None), + ("get", "/projects/1/protocols/2/tags", None), + ("put", "/projects/1/protocols/2/tags", {"tagIds": ["tag-1"]}), + ("get", "/projects/1/context-menu-visibility", None), + ], +) +def test_TagAndContextMenuEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + method, + url, + payload, +): + fakeProjectService.projectDbRowResult = None + + request = getattr(projectClient, method) + if payload is None: + response = request(url) + else: + response = request(url, json=payload) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_GetFscRowsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/fsc/rows" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.fscRowsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetFscRowsCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastGetFscRowsCall["mapper"], + } + + +def test_GetFscRowsReturns404WhenProjectMissing(projectClient, fakeProjectService): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/fsc/rows" + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_GetProjectEffectiveSettingsUsesProjectDbRow( + projectClient, + fakeProjectService, +): + response = projectClient.get("/projects/1/effective-settings") + + assert response.status_code == 200 + assert response.json() == fakeProjectService.projectEffectiveSettingsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetProjectEffectiveSettingsCall == { + "mapper": fakeProjectService.lastGetProjectEffectiveSettingsCall["mapper"], + "projectId": 1, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + } + + +def test_GetProjectEffectiveSettingsReturns404WhenProjectMissing( + projectClient, + fakeProjectService, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get("/projects/1/effective-settings") + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetProjectEffectiveSettingsCall is None + + +def test_RenderCoords3dTomogramSliceUsesProjectDbRow( + projectClient, + fakeProjectService, +): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/coords3d/tomograms/tomo-1/slice" + "?index=4&axis=y&cmap=gray&normalize=zscore&scale=1.5" + "&inline=false&format=png&thumb=128&fast=false&quality=80" + ) + + assert response.status_code == 200 + assert response.content == b"slice-bytes" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderCoords3dTomogramSliceCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tomogramId": "tomo-1", + "sliceIndex": 4, + "axis": "y", + "colormap": "gray", + "normalize": "zscore", + "scale": 1.5, + "inline": False, + "fmt": "png", + "thumb": 128, + "fast": False, + "quality": 80, + "mapper": fakeProjectService.lastRenderCoords3dTomogramSliceCall["mapper"], + } + + +def test_RenderCoords3dTomogramSliceReturns404WhenProjectMissing( + projectClient, + fakeProjectService, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/coords3d/tomograms/tomo-1/slice?index=4" + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderCoords3dTomogramSliceCall is None + + +def resolveAnalyzeViewerDecision( + self, + projectId, + protocolId, + ctx, + mapper=None, +): + self.lastResolveAnalyzeViewerDecisionCall = { + "projectId": projectId, + "protocolId": protocolId, + "ctx": ctx, + "mapper": mapper, + } + return self.resolveAnalyzeViewerDecisionResult + + +def test_ResolveAnalyzeViewerUnwrapsCtxAndUsesProjectDbRow( + projectClient, + fakeProjectService, +): + ctx = { + "outputName": "out", + "objectId": "tomo-1", + "objectKind": "tomogram", + } + + response = projectClient.post( + "/projects/1/protocols/2/viewer/resolve", + json={"ctx": ctx}, + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.resolveAnalyzeViewerDecisionResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + call = ( + fakeProjectService.lastResolveAnalyzeViewerDecisionCall + or fakeProjectService.lastResolveViewerCall + ) + + assert call == { + "projectId": 1, + "protocolId": 2, + "ctx": ctx, + "mapper": call["mapper"], + } + + +def test_ResolveAnalyzeViewerReturns404WhenProjectMissing( + projectClient, + fakeProjectService, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.post( + "/projects/1/protocols/2/viewer/resolve", + json={"outputName": "out"}, + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastResolveAnalyzeViewerDecisionCall is None + + +def test_RenderVolumeSliceUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/volumes/vol-1/slice" + "?index=4&axis=y&cmap=gray&normalize=zscore&scale=1.5" + "&inline=false&format=png&thumb=128&fast=false&quality=80" + ) + + assert response.status_code == 200 + assert response.content == b"volume-slice-bytes" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderVolumeSliceCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "volumeId": "vol-1", + "sliceIndex": 4, + "axis": "y", + "colormap": "gray", + "normalize": "zscore", + "scale": 1.5, + "inline": False, + "fmt": "png", + "thumb": 128, + "fast": False, + "quality": 80, + "mapper": fakeProjectService.lastRenderVolumeSliceCall["mapper"], + } + + +def test_GetVolumeData3dUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/volumes/vol-1/data3d" + "?maxDim=96&method=stride" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.volumeData3dResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetVolumeData3dCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "volumeId": "vol-1", + "maxDim": 96, + "method": "stride", + "mapper": fakeProjectService.lastGetVolumeData3dCall["mapper"], + } + + +def test_GetVolumeSurfaceMeshUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/volumes/vol-1/surface" + "?level=0.5&maxDim=96&method=stride&maxTriangles=1234" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.volumeSurfaceMeshResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetVolumeSurfaceMeshCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "volumeId": "vol-1", + "level": 0.5, + "maxDim": 96, + "method": "stride", + "maxTriangles": 1234, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + "mapper": fakeProjectService.lastGetVolumeSurfaceMeshCall["mapper"], + } + + +@pytest.mark.parametrize( + "url", + [ + "/projects/1/protocols/2/outputs/out/volumes/vol-1/slice?index=4", + "/projects/1/protocols/2/outputs/out/volumes/vol-1/data3d", + "/projects/1/protocols/2/outputs/out/volumes/vol-1/surface", + ], +) +def test_VolumeRenderEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + url, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get(url) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderVolumeSliceCall is None + assert fakeProjectService.lastGetVolumeData3dCall is None + assert fakeProjectService.lastGetVolumeSurfaceMeshCall is None + + +def test_RenderTiltSeriesImageUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/tiltseries/TS_001/tilt" + "?index=3&size=256&fmt=webp&applyTransform=true&inline=false" + ) + + assert response.status_code == 200 + assert response.content == b"tilt-image-bytes" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderTiltSeriesImageCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tiltSeriesId": "TS_001", + "index": 3, + "size": 256, + "fmt": "webp", + "applyTransform": True, + "inline": False, + "requestHeaders": None, + "mapper": fakeProjectService.lastRenderTiltSeriesImageCall["mapper"], + } + + +def test_RenderTiltSeriesImagesBatchUsesProjectDbRow( + projectClient, + fakeProjectService, +): + response = projectClient.post( + "/projects/1/protocols/2/outputs/out/tiltseries/TS_001/tilt/batch", + json={ + "indices": [0, 2, 4], + "size": 256, + "fmt": "webp", + "applyTransform": True, + "inline": False, + }, + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.tiltSeriesBatchResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderTiltSeriesImagesBatchCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tiltSeriesId": "TS_001", + "indices": [0, 2, 4], + "size": 256, + "fmt": "webp", + "applyTransform": True, + "inline": False, + "requestHeaders": None, + "mapper": fakeProjectService.lastRenderTiltSeriesImagesBatchCall["mapper"], + } + + +@pytest.mark.parametrize( + ("method", "url", "payload"), + [ + ( + "get", + "/projects/1/protocols/2/outputs/out/tiltseries/TS_001/tilt?index=3", + None, + ), + ( + "post", + "/projects/1/protocols/2/outputs/out/tiltseries/TS_001/tilt/batch", + {"indices": [0, 2]}, + ), + ], +) +def test_TiltSeriesRenderEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + method, + url, + payload, +): + fakeProjectService.projectDbRowResult = None + + request = getattr(projectClient, method) + if payload is None: + response = request(url) + else: + response = request(url, json=payload) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderTiltSeriesImageCall is None + assert fakeProjectService.lastRenderTiltSeriesImagesBatchCall is None + + +def test_RenderCtftomoPsdImageUsesProjectDbRow( + projectClient, + fakeProjectService, +): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/ctftomo/psd" + "?spec=3%40%2Ftmp%2Fpsd.mrc&size=256&fmt=webp" + "&applyTransform=true&inline=false" + ) + + assert response.status_code == 200 + assert response.content == b"ctftomo-psd-bytes" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderCtfTomoPsdImageCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "psdPath": "3@/tmp/psd.mrc", + "size": 256, + "fmt": "webp", + "inline": False, + "index": 0, + "quality": 75, + "applyTransform": True, + "rot": None, + "shifts": None, + "mapper": fakeProjectService.lastRenderCtfTomoPsdImageCall["mapper"], + } + + +def test_RenderCtftomoPsdImageReturns404WhenProjectMissing( + projectClient, + fakeProjectService, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/ctftomo/psd" + "?spec=3%40%2Ftmp%2Fpsd.mrc" + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderCtfTomoPsdImageCall is None + + +def test_RenderCtftomoPsdImagePreservesHttpException( + projectClient, + fakeProjectService, +): + fakeProjectService.renderCtfTomoPsdImageError = HTTPException( + status_code=404, + detail="PSD image file not found", + ) + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/ctftomo/psd" + "?spec=3%40%2Ftmp%2Fpsd.mrc" + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "PSD image file not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_ListProtocolLogChannelsUsesProjectDbRow( + projectClient, + fakeProjectService, +): + response = projectClient.get("/projects/1/protocols/2/logs/channels") + + assert response.status_code == 200 + assert response.json() == { + "channels": fakeProjectService.logChannelsResult, + } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListProtocolLogChannelsCall == { + "projectId": 1, + "protocolId": 2, + "mapper": fakeProjectService.lastListProtocolLogChannelsCall["mapper"], + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + } + + +def test_PollProtocolLogsUsesProjectDbRow( + projectClient, + fakeProjectService, +): + response = projectClient.post( + "/projects/1/protocols/2/logs/chunk", + json={ + "offsets": { + "stdout": 0, + }, + "maxBytes": 32, + "maxLines": 10, + }, + ) + + assert response.status_code == 200 + assert response.json() == { + "chunks": [ + { + "channel": "stdout", + "content": "hello", + "offset": 5, + } + ] + } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastPollLogsCall == { + "projectId": 1, + "protocolId": 2, + "offsets": {"stdout": 0}, + "maxBytes": 32, + "maxLines": 10, + "mapper": fakeProjectService.lastPollLogsCall["mapper"], + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + } + + +@pytest.mark.parametrize( + ("method", "url", "payload"), + [ + ("get", "/projects/1/protocols/2/logs/channels", None), + ("post", "/projects/1/protocols/2/logs/chunk", {"offsets": {"stdout": 0}}), + ], +) +def test_ProtocolLogEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + method, + url, + payload, +): + fakeProjectService.projectDbRowResult = None + + request = getattr(projectClient, method) + response = request(url) if payload is None else request(url, json=payload) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListProtocolLogChannelsCall is None + assert fakeProjectService.lastPollLogsCall is None \ No newline at end of file diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 0fc7bf84..d39c71dc 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -23,13 +23,16 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ****************************************************************************** +import pytest from fastapi import HTTPException def patchRenameProtocolFake(fakeProjectService): # patchRenameProtocolFake - def renameProtocol(protocolId, newName, newComment=""): + def renameProtocol(mapper, projectId, protocolId, newName, newComment=""): fakeProjectService.lastRenameProtocolCall = { + "mapper": mapper, + "projectId": projectId, "protocolId": protocolId, "newName": newName, "newComment": newComment, @@ -62,6 +65,65 @@ def test_LoadProtocolReturnsParams(projectClient, fakeProjectService): assert fakeProjectService.lastGetProtocolParamsCall == { "projectId": 1, "protocolId": 10, + "mapper": fakeProjectService.lastGetProtocolParamsCall["mapper"], + } + + +def test_LoadProtocolsReturns404WhenProjectMissing( + projectClient, + fakeProjectService, + fakeProjectMapper, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get("/projects/1/protocols") + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + + assert fakeProjectService.lastGetProjectDbRowCall == { + "mapper": fakeProjectMapper, + "projectId": 1, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + } + + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_LoadProtocolsUsesProjectDbRow( + projectClient, + fakeProjectService, + fakeProjectMapper, +): + response = projectClient.get("/projects/1/protocols") + + assert response.status_code == 200 + assert response.json() == fakeProjectService.protocolsResult + + assert fakeProjectService.lastGetProjectDbRowCall == { + "mapper": fakeProjectMapper, + "projectId": 1, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + } + + assert fakeProjectService.lastGetProjectByIdCall is None + + assert fakeProjectService.lastGetProtocolsCall == { + "mapper": fakeProjectMapper, + "projectId": 1, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, } @@ -80,6 +142,65 @@ def test_LoadNewProtocolReturnsParams(projectClient, fakeProjectService): } +@pytest.mark.parametrize( + ("method", "url", "payload"), + [ + ( + "post", + "/projects/1/protocols/duplicate", + {"items": [{"id": "10", "name": "Copy 1"}]}, + ), + ( + "post", + "/projects/1/protocols/delete", + {"protocolIds": ["10"]}, + ), + ( + "post", + "/projects/1/protocols/10/restart-all", + None, + ), + ( + "post", + "/projects/1/protocols/10/continue-all", + None, + ), + ( + "post", + "/projects/1/protocols/10/reset-from", + None, + ), + ( + "post", + "/projects/1/protocols/stop", + {"protocolIds": ["10"]}, + ), + ], +) +def test_ProtocolOperationsReturn404EnvelopeWhenProjectMissing( + projectClient, + fakeProjectService, + method, + url, + payload, +): + fakeProjectService.projectByIdResult = None + + request = getattr(projectClient, method) + kwargs = {} + if payload is not None: + kwargs["json"] = payload + + response = request(url, **kwargs) + + assert response.status_code == 404 + assert response.json() == { + "status": 1, + "errors": ["Project not found"], + "workflow": [], + } + + def test_LaunchProtocolReturns404EnvelopeWhenProjectMissing(projectClient, fakeProjectService): fakeProjectService.projectByIdResult = None @@ -95,7 +216,7 @@ def test_LaunchProtocolReturns404EnvelopeWhenProjectMissing(projectClient, fakeP assert response.status_code == 404 assert response.json() == { - "status": 0, + "status": 1, "errors": ["Project not found"], "workflow": [], } @@ -129,6 +250,74 @@ def test_LaunchProtocolDelegatesToService(projectClient, fakeProjectService): } +def test_LaunchProtocolDefaultsMissingModeToLaunch(projectClient, fakeProjectService): + response = projectClient.post( + "/projects/1/launch", + json={ + "protocolId": "10", + "protocolClassName": "ProtClass", + "params": {"a": 1}, + }, + ) + + assert response.status_code == 200 + assert response.json() == { + "status": 0, + "errors": [], + "workflow": [], + } + + assert fakeProjectService.lastLaunchProtocolCall["executeMode"] is None + + +def test_LaunchProtocolReturnsSyncCounts( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeLaunchProtocol( + mapper, + projectId, + protocolId, + protocolClassName, + params, + executeMode, + ): + fakeProjectService.lastLaunchProtocolCall = { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + "protocolClassName": protocolClassName, + "params": params, + "executeMode": executeMode, + } + return { + "protocols": 2, + "dependencies": 1, + } + + monkeypatch.setattr(fakeProjectService, "launchProtocol", fakeLaunchProtocol) + + response = projectClient.post( + "/projects/1/launch", + json={ + "protocolId": "10", + "protocolClassName": "ProtClass", + "params": {"a": 1}, + "mode": "resume", + }, + ) + + assert response.status_code == 200 + assert response.json() == { + "status": 0, + "errors": [], + "workflow": [], + "protocolsCount": 2, + "dependenciesCount": 1, + } + + def test_LaunchProtocolWrapsHttpException(projectClient, fakeProjectService): fakeProjectService.launchProtocolError = HTTPException(status_code=409, detail=["conflict", "busy"]) @@ -144,12 +333,47 @@ def test_LaunchProtocolWrapsHttpException(projectClient, fakeProjectService): assert response.status_code == 409 assert response.json() == { - "status": 0, + "status": 1, "errors": ["conflict", "busy"], "workflow": [], } +def test_LaunchProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeLaunchProtocol( + mapper, + projectId, + protocolId, + protocolClassName, + params, + executeMode, + ): + raise RuntimeError("boom") + + monkeypatch.setattr(fakeProjectService, "launchProtocol", fakeLaunchProtocol) + + response = projectClient.post( + "/projects/1/launch", + json={ + "protocolId": "10", + "protocolClassName": "ProtClass", + "params": {"a": 1}, + "mode": "resume", + }, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["Internal server error"], + "workflow": [], + } + + def test_SaveProtocolReturnsSuccessWhenNoErrors(projectClient, fakeProjectService): fakeProjectService.saveProtocolResult = ({"protocolId": "10"}, []) @@ -178,6 +402,29 @@ def test_SaveProtocolReturnsSuccessWhenNoErrors(projectClient, fakeProjectServic } +def test_SaveProtocolReturns404EnvelopeWhenProjectMissing( + projectClient, + fakeProjectService, +): + fakeProjectService.projectByIdResult = None + + response = projectClient.post( + "/projects/1/save", + json={ + "protocolId": "10", + "protocolClassName": "ProtClass", + "params": {"a": 1}, + }, + ) + + assert response.status_code == 404 + assert response.json() == { + "status": 1, + "errors": ["Project not found"], + "workflow": [], + } + + def test_SaveProtocolReturnsStatusOneWhenServiceReturnsErrors(projectClient, fakeProjectService): fakeProjectService.saveProtocolResult = ( {"protocolId": "10"}, @@ -201,6 +448,62 @@ def test_SaveProtocolReturnsStatusOneWhenServiceReturnsErrors(projectClient, fak } +def test_SaveProtocolWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.saveProtocolError = HTTPException( + status_code=500, + detail="Protocol was saved in Scipion but graph sync to PostgreSQL failed", + ) + + response = projectClient.post( + "/projects/1/save", + json={ + "protocolId": "10", + "protocolClassName": "ProtClass", + "params": {"a": 1}, + }, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["Protocol was saved in Scipion but graph sync to PostgreSQL failed"], + "workflow": [], + } + + +def test_SaveProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeSaveProtocol( + mapper, + projectId, + protocolId, + protocolClassName, + params, + ): + raise RuntimeError("boom") + + monkeypatch.setattr(fakeProjectService, "saveProtocol", fakeSaveProtocol) + + response = projectClient.post( + "/projects/1/save", + json={ + "protocolId": "10", + "protocolClassName": "ProtClass", + "params": {"a": 1}, + }, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_SuggestionProtocolReturns404EnvelopeWhenProjectMissing(projectClient, fakeProjectService): fakeProjectService.projectByIdResult = None @@ -220,7 +523,51 @@ def test_SuggestionProtocolReturnsSuggestions(projectClient, fakeProjectService) assert response.status_code == 200 assert response.json() == [{"id": "next-1", "name": "Next protocol"}] - assert fakeProjectService.lastGetNextProtocolSuggestionsCall == {"protocolId": 10} + assert fakeProjectService.lastGetNextProtocolSuggestionsCall == { + "protocolId": 10, + "mapper": fakeProjectService.lastGetNextProtocolSuggestionsCall["mapper"], + "projectId": 1, + } + + +def test_SuggestionProtocolWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.nextProtocolSuggestionsError = HTTPException( + status_code=404, + detail="Protocol not found", + ) + + response = projectClient.get("/projects/1/protocols/10/suggestions/next") + + assert response.status_code == 404 + assert response.json() == { + "status": 1, + "errors": ["Protocol not found"], + "workflow": [], + } + + +def test_SuggestionProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeGetNextProtocolSuggestions(mapper, projectId, protocolId): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "getNextProtocolSuggestions", + fakeGetNextProtocolSuggestions, + ) + + response = projectClient.get("/projects/1/protocols/10/suggestions/next") + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } def test_RenameProtocolRejectsBlankName(projectClient): @@ -253,9 +600,13 @@ def test_RenameProtocolDelegatesToService(projectClient, fakeProjectService): "status": 0, "errors": [], "workflow": [], + "protocolsCount": 1, + "dependenciesCount": 0, } assert fakeProjectService.lastRenameProtocolCall == { + "mapper": fakeProjectService.lastRenameProtocolCall["mapper"], + "projectId": 1, "protocolId": 10, "newName": "Renamed protocol", "newComment": "Updated comment", @@ -270,6 +621,64 @@ def test_RenameProtocolDelegatesToService(projectClient, fakeProjectService): } +def test_RenameProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeRenameProtocol( + mapper, + projectId, + protocolId, + newName, + newComment, + ): + raise RuntimeError("boom") + + monkeypatch.setattr(fakeProjectService, "renameProtocol", fakeRenameProtocol) + + response = projectClient.put( + "/projects/1/protocols/10/rename", + json={ + "runName": "Renamed protocol", + "comment": "Updated comment", + }, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + +def test_RenameProtocolWrapsHttpException( + projectClient, + fakeProjectService, +): + patchRenameProtocolFake(fakeProjectService) + fakeProjectService.renameProtocolError = HTTPException( + status_code=404, + detail="Protocol not found", + ) + + response = projectClient.put( + "/projects/1/protocols/10/rename", + json={ + "runName": "Renamed protocol", + "comment": "Updated comment", + }, + ) + + assert response.status_code == 404 + assert response.json() == { + "status": 1, + "errors": ["Protocol not found"], + "workflow": [], + } + + def test_DuplicateProtocolRejectsMissingItems(projectClient): response = projectClient.post( "/projects/1/protocols/duplicate", @@ -312,13 +721,99 @@ def test_DuplicateProtocolDelegatesToService(projectClient, fakeProjectService): assert items[1].name is None +def test_DuplicateProtocolReturnsSyncCounts( + projectClient, + fakeProjectService, +): + fakeProjectService.duplicateProtocolResult = { + "status": 0, + "errors": [], + "duplicated": [{"sourceId": "10", "newId": "20"}], + "protocolsCount": 4, + "dependenciesCount": 3, + } + + response = projectClient.post( + "/projects/1/protocols/duplicate", + json={ + "items": [ + {"id": "10", "name": "Copy 1"}, + ] + }, + ) + + assert response.status_code == 201 + assert response.json() == { + "status": 0, + "errors": [], + "workflow": [], + "duplicated": [{"sourceId": "10", "newId": "20"}], + "protocolsCount": 4, + "dependenciesCount": 3, + } + + +def test_DuplicateProtocolWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.duplicateProtocolError = HTTPException( + status_code=500, + detail="Failed to duplicate protocols: copy failed", + ) + + response = projectClient.post( + "/projects/1/protocols/duplicate", + json={ + "items": [ + {"id": "10", "name": "Copy 1"}, + ] + }, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["Failed to duplicate protocols: copy failed"], + "workflow": [], + } + + +def test_DuplicateProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeDuplicateProtocol(mapper, projectId, items): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "duplicateProtocol", + fakeDuplicateProtocol, + ) + + response = projectClient.post( + "/projects/1/protocols/duplicate", + json={ + "items": [ + {"id": "10", "name": "Copy 1"}, + ] + }, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_DeleteProtocolRejectsMissingProtocolIds(projectClient): response = projectClient.post( "/projects/1/protocols/delete", json={"protocolIds": []}, ) - assert response.status_code == 404 + assert response.status_code == 422 assert response.json() == { "status": 1, "errors": ["Missing protocolIds"], @@ -346,6 +841,52 @@ def test_DeleteProtocolDelegatesToService(projectClient, fakeProjectService): } +def test_DeleteProtocolWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.deleteProtocolError = HTTPException( + status_code=500, + detail="Failed to delete protocols", + ) + + response = projectClient.post( + "/projects/1/protocols/delete", + json={"protocolIds": ["10"]}, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["Failed to delete protocols"], + "workflow": [], + } + + +def test_DeleteProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeDeleteProtocol(mapper, projectId, protocolIds): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "deleteProtocol", + fakeDeleteProtocol, + ) + + response = projectClient.post( + "/projects/1/protocols/delete", + json={"protocolIds": ["10"]}, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_RestartProtocolAllReturnsErrorsWhenServiceRaisesHttpException(projectClient, fakeProjectService): fakeProjectService.restartProtocolAllError = HTTPException( status_code=422, @@ -362,6 +903,30 @@ def test_RestartProtocolAllReturnsErrorsWhenServiceRaisesHttpException(projectCl } +def test_RestartProtocolAllWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeRestartProtocolAll(mapper, projectId, protocolId): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "restartProtocolAll", + fakeRestartProtocolAll, + ) + + response = projectClient.post("/projects/1/protocols/10/restart-all") + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_RestartProtocolAllReturnsSuccess(projectClient, fakeProjectService): fakeProjectService.restartProtocolAllResult = [] @@ -372,9 +937,15 @@ def test_RestartProtocolAllReturnsSuccess(projectClient, fakeProjectService): "status": 0, "errors": [], "workflow": [], + "protocolsCount": 1, + "dependenciesCount": 0, } - assert fakeProjectService.lastRestartProtocolAllCall == {"protocolId": 10} + assert fakeProjectService.lastRestartProtocolAllCall == { + "mapper": fakeProjectService.lastRestartProtocolAllCall["mapper"], + "projectId": 1, + "protocolId": 10, + } assert fakeProjectService.lastSyncProjectGraphAfterMutationCall == { "mapper": fakeProjectService.lastSyncProjectGraphAfterMutationCall["mapper"], "projectId": 1, @@ -384,6 +955,46 @@ def test_RestartProtocolAllReturnsSuccess(projectClient, fakeProjectService): } +def test_ContinueProtocolAllWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.continueProtocolAllError = HTTPException( + status_code=422, + detail=["cannot continue", "blocked"], + ) + + response = projectClient.post("/projects/1/protocols/10/continue-all") + + assert response.status_code == 422 + assert response.json() == { + "status": 1, + "errors": ["cannot continue", "blocked"], + "workflow": [], + } + + +def test_ContinueProtocolAllWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeContinueProtocolAll(mapper, projectId, protocolId, currentUser): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "continueProtocolAll", + fakeContinueProtocolAll, + ) + + response = projectClient.post("/projects/1/protocols/10/continue-all") + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_ContinueProtocolAllDelegatesToService(projectClient, fakeProjectService): response = projectClient.post("/projects/1/protocols/10/continue-all") @@ -392,6 +1003,8 @@ def test_ContinueProtocolAllDelegatesToService(projectClient, fakeProjectService "status": 0, "errors": [], "workflow": [], + "protocolsCount": 1, + "dependenciesCount": 0, } assert fakeProjectService.lastSyncProjectGraphAfterMutationCall == { @@ -414,6 +1027,46 @@ def test_ContinueProtocolAllDelegatesToService(projectClient, fakeProjectService } +def test_ResetProtocolFromWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.resetProtocolFromError = HTTPException( + status_code=422, + detail=["cannot reset", "blocked"], + ) + + response = projectClient.post("/projects/1/protocols/10/reset-from") + + assert response.status_code == 422 + assert response.json() == { + "status": 1, + "errors": ["cannot reset", "blocked"], + "workflow": [], + } + + +def test_ResetProtocolFromWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeResetProtocolFrom(mapper, projectId, protocolId): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "resetProtocolFrom", + fakeResetProtocolFrom, + ) + + response = projectClient.post("/projects/1/protocols/10/reset-from") + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_ResetProtocolFromDelegatesToService(projectClient, fakeProjectService): response = projectClient.post("/projects/1/protocols/10/reset-from") @@ -422,9 +1075,15 @@ def test_ResetProtocolFromDelegatesToService(projectClient, fakeProjectService): "status": 0, "errors": [], "workflow": [], + "protocolsCount": 1, + "dependenciesCount": 0, } - assert fakeProjectService.lastResetProtocolFromCall == {"protocolId": 10} + assert fakeProjectService.lastResetProtocolFromCall == { + "mapper": fakeProjectService.lastResetProtocolFromCall["mapper"], + "projectId": 1, + "protocolId": 10, + } assert fakeProjectService.lastSyncProjectGraphAfterMutationCall == { "mapper": fakeProjectService.lastSyncProjectGraphAfterMutationCall["mapper"], "projectId": 1, @@ -454,6 +1113,8 @@ def test_RenameProtocolReturnsErrorWhenGraphSyncFails(projectClient, fakeProject } assert fakeProjectService.lastRenameProtocolCall == { + "mapper": fakeProjectService.lastRenameProtocolCall["mapper"], + "projectId": 1, "protocolId": 10, "newName": "Renamed protocol", "newComment": "Updated comment", @@ -476,7 +1137,11 @@ def test_RestartProtocolAllReturnsErrorWhenGraphSyncFails(projectClient, fakePro "workflow": [], } - assert fakeProjectService.lastRestartProtocolAllCall == {"protocolId": 10} + assert fakeProjectService.lastRestartProtocolAllCall == { + "mapper": fakeProjectService.lastRestartProtocolAllCall["mapper"], + "projectId": 1, + "protocolId": 10, + } def test_ContinueProtocolAllReturnsErrorWhenGraphSyncFails(projectClient, fakeProjectService): @@ -521,7 +1186,57 @@ def test_ResetProtocolFromReturnsErrorWhenGraphSyncFails(projectClient, fakeProj "workflow": [], } - assert fakeProjectService.lastResetProtocolFromCall == {"protocolId": 10} + assert fakeProjectService.lastResetProtocolFromCall == { + "mapper": fakeProjectService.lastResetProtocolFromCall["mapper"], + "projectId": 1, + "protocolId": 10, + } + + +def test_StopProtocolWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.stopProtocolError = HTTPException( + status_code=422, + detail=["cannot stop", "blocked"], + ) + + response = projectClient.post( + "/projects/1/protocols/stop", + json={"protocolIds": ["10"]}, + ) + + assert response.status_code == 422 + assert response.json() == { + "status": 1, + "errors": ["cannot stop", "blocked"], + "workflow": [], + } + + +def test_StopProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeStopProtocol(mapper, projectId, protocolIds): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "stopProtocol", + fakeStopProtocol, + ) + + response = projectClient.post( + "/projects/1/protocols/stop", + json={"protocolIds": ["10"]}, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } def test_StopProtocolReturnsErrorWhenGraphSyncFails(projectClient, fakeProjectService): @@ -543,6 +1258,8 @@ def test_StopProtocolReturnsErrorWhenGraphSyncFails(projectClient, fakeProjectSe } assert fakeProjectService.lastStopProtocolCall == { + "mapper": fakeProjectService.lastStopProtocolCall["mapper"], + "projectId": 1, "protocolIds": ["10", "11"], } @@ -572,15 +1289,87 @@ def test_StopProtocolDelegatesToService(projectClient, fakeProjectService): "status": 0, "errors": [], "workflow": [], + "protocolsCount": 1, + "dependenciesCount": 0, } assert fakeProjectService.lastStopProtocolCall == { + "mapper": fakeProjectService.lastStopProtocolCall["mapper"], + "projectId": 1, "protocolIds": ["10", "11"], } + assert fakeProjectService.lastSyncProjectGraphAfterMutationCall == { "mapper": fakeProjectService.lastSyncProjectGraphAfterMutationCall["mapper"], "projectId": 1, "actionLabel": "stop protocol", "refresh": True, "checkPid": True, - } \ No newline at end of file + } + + +def test_DeleteProtocolReturnsErrorsWhenServiceRaisesHttpException( + projectClient, + fakeProjectService, +): + fakeProjectService.deleteProtocolError = HTTPException( + status_code=422, + detail=["cannot delete protocol"], + ) + + response = projectClient.post( + "/projects/1/protocols/delete", + json={"protocolIds": ["10"]}, + ) + + assert response.status_code == 422 + assert response.json() == { + "status": 1, + "errors": ["cannot delete protocol"], + "workflow": [], + } + + assert fakeProjectService.lastDeleteProtocolCall == { + "mapper": fakeProjectService.lastDeleteProtocolCall["mapper"], + "projectId": 1, + "protocolIds": ["10"], + } + + +def test_DeleteProtocolReturnsSyncCounts( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeDeleteProtocol(mapper, projectId, protocolIds): + fakeProjectService.lastDeleteProtocolCall = { + "mapper": mapper, + "projectId": projectId, + "protocolIds": protocolIds, + } + return { + "protocolsCount": 3, + "dependenciesCount": 2, + } + + monkeypatch.setattr(fakeProjectService, "deleteProtocol", fakeDeleteProtocol) + + response = projectClient.post( + "/projects/1/protocols/delete", + json={"protocolIds": ["10"]}, + ) + + assert response.status_code == 200 + assert response.json() == { + "status": 0, + "errors": [], + "workflow": [], + "protocolsCount": 3, + "dependenciesCount": 2, + } + + assert fakeProjectService.lastDeleteProtocolCall == { + "mapper": fakeProjectService.lastDeleteProtocolCall["mapper"], + "projectId": 1, + "protocolIds": ["10"], + } diff --git a/tests/unit/backend/api/routers/test_protocols_router.py b/tests/unit/backend/api/routers/test_protocols_router.py index 31b83a4e..6d7292c7 100644 --- a/tests/unit/backend/api/routers/test_protocols_router.py +++ b/tests/unit/backend/api/routers/test_protocols_router.py @@ -69,10 +69,11 @@ def getProjectById(self, mapper, projectId, currentUser): } return self.projectByIdResult - def getProtocolParams(self, projectId, protocolId): + def getProtocolParams(self, projectId, protocolId, mapper=None): self.lastGetProtocolParamsCall = { "projectId": projectId, "protocolId": protocolId, + "mapper": mapper, } return self.protocolParamsResult @@ -101,13 +102,14 @@ def saveProtocol(self, protocolId, protocolClassName, params): if self.saveError is not None: raise self.saveError - def getProtocolLogs(self, projectId, protocolId, offset, errOffset, scheduleOffset): + def getProtocolLogs(self, projectId, protocolId, offset, errOffset, scheduleOffset, mapper=None): self.lastGetProtocolLogsCall = { "projectId": projectId, "protocolId": protocolId, "offset": offset, "errOffset": errOffset, "scheduleOffset": scheduleOffset, + "mapper": mapper, } return self.protocolLogsResult @@ -131,9 +133,6 @@ def protocolClient( fakeProtocolRouterService, monkeypatch: pytest.MonkeyPatch, ) -> Iterator[TestClient]: - # protocolClient - monkeypatch.setattr(protocolRouterModule, "service", fakeProtocolRouterService) - app = FastAPI() app.include_router(protocolRouterModule.router) @@ -143,6 +142,9 @@ def protocolClient( "email": "user@example.com", "role": "user", } + app.dependency_overrides[protocolRouterModule.getProjectService] = ( + lambda: fakeProtocolRouterService + ) with TestClient(app) as client: yield client @@ -172,6 +174,7 @@ def test_LoadProtocolReturnsProtocolParams(protocolClient, fakeProtocolRouterSer assert fakeProtocolRouterService.lastGetProtocolParamsCall == { "projectId": 1, "protocolId": 10, + "mapper": fakeProtocolRouterService.lastGetProtocolParamsCall["mapper"], } @@ -199,29 +202,7 @@ def test_LoadNewProtocolReturnsProtocolTemplate(protocolClient, fakeProtocolRout } -def test_LaunchProtocolReturnsNullOnSuccess(protocolClient, fakeProtocolRouterService): - response = protocolClient.post( - "/protocols/launch", - json={ - "protocolId": "10", - "protocolClassName": "ProtClass", - "params": {"a": 1}, - }, - ) - - assert response.status_code == 200 - assert response.json() is None - - assert fakeProtocolRouterService.lastLaunchProtocolCall == { - "protocolId": "10", - "protocolClassName": "ProtClass", - "params": {"a": 1}, - } - - -def test_LaunchProtocolWrapsUnexpectedErrorAs500(protocolClient, fakeProtocolRouterService): - fakeProtocolRouterService.launchError = RuntimeError("launch failed") - +def test_LaunchProtocolEndpointIsDisabled(protocolClient): response = protocolClient.post( "/protocols/launch", json={ @@ -231,33 +212,10 @@ def test_LaunchProtocolWrapsUnexpectedErrorAs500(protocolClient, fakeProtocolRou }, ) - assert response.status_code == 500 - assert response.json()["detail"] == "launch failed" - - -def test_SaveProtocolReturnsNullOnSuccess(protocolClient, fakeProtocolRouterService): - response = protocolClient.post( - "/protocols/save", - json={ - "protocolId": "10", - "protocolClassName": "ProtClass", - "params": {"a": 1}, - }, - ) - - assert response.status_code == 200 - assert response.json() is None - - assert fakeProtocolRouterService.lastSaveProtocolCall == { - "protocolId": "10", - "protocolClassName": "ProtClass", - "params": {"a": 1}, - } - + assert response.status_code == 404 -def test_SaveProtocolWrapsUnexpectedErrorAs500(protocolClient, fakeProtocolRouterService): - fakeProtocolRouterService.saveError = RuntimeError("save failed") +def test_SaveProtocolEndpointIsDisabled(protocolClient): response = protocolClient.post( "/protocols/save", json={ @@ -267,8 +225,7 @@ def test_SaveProtocolWrapsUnexpectedErrorAs500(protocolClient, fakeProtocolRoute }, ) - assert response.status_code == 500 - assert response.json()["detail"] == "save failed" + assert response.status_code == 404 def test_GetProtocolLogsReturns404WhenProjectMissing(protocolClient, fakeProtocolRouterService): @@ -296,4 +253,5 @@ def test_GetProtocolLogsReturnsPayload(protocolClient, fakeProtocolRouterService "offset": 5, "errOffset": 7, "scheduleOffset": 9, + "mapper": fakeProtocolRouterService.lastGetProtocolLogsCall["mapper"], } \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_coords2d_service.py b/tests/unit/backend/api/services/test_coords2d_service.py new file mode 100644 index 00000000..a0172d47 --- /dev/null +++ b/tests/unit/backend/api/services/test_coords2d_service.py @@ -0,0 +1,432 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +import importlib + +import pytest +from fastapi import HTTPException + + +class FakeMapper: + pass + + +class FakeProjectService: + def __init__(self, currentProject): + self.currentProject = currentProject + self.projectRow = {"id": 1} + self.runtimeProtocolIdByDbId = {} + self.getProjectByIdCalls = [] + self.runtimeCalls = [] + + def getProjectById(self, mapper, projectId, currentUser, refresh=False, checkPid=False): + self.getProjectByIdCalls.append({ + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + "refresh": refresh, + "checkPid": checkPid, + }) + return self.projectRow + + def _getScipionProtocolForRuntime(self, mapper, projectId, protocolId): + self.runtimeCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + }) + + runtimeProtocolId = self.runtimeProtocolIdByDbId.get( + int(protocolId), + int(protocolId), + ) + return self.currentProject.protocols[int(runtimeProtocolId)] + + +class FakeCurrentProject: + def __init__(self): + self.protocols = {} + + +class FakeProtocol: + def __init__(self, objId=10): + self.objId = objId + + def getObjId(self): + return self.objId + + +class FakeMicrograph: + def __init__( + self, + objId, + location=None, + fileName=None, + micName=None, + label=None, + dims=None, + ): + self.objId = objId + self.location = location + self.fileName = fileName + self.micName = micName + self.label = label + self.dims = dims or (4096, 4096) + + def clone(self): + return FakeMicrograph( + objId=self.objId, + location=self.location, + fileName=self.fileName, + micName=self.micName, + label=self.label, + dims=self.dims, + ) + + def getObjId(self): + return self.objId + + def getLocation(self): + return self.location + + def getFileName(self): + return self.fileName + + def getMicName(self): + return self.micName + + def getObjLabel(self): + return self.label + + def getDim(self): + return self.dims + + +class FakeMicrographsSet: + def __init__(self, micrographs): + self.micrographs = list(micrographs) + + def iterItems(self, iterate=False): + return iter(self.micrographs) + + def __getitem__(self, micId): + for micrograph in self.micrographs: + if int(micrograph.getObjId()) == int(micId): + return micrograph + + raise KeyError(micId) + + +class FakeCoordinate: + def __init__( + self, + objId, + micId, + x, + y, + score=None, + classId=None, + ): + self.objId = objId + self.micId = micId + self.x = x + self.y = y + self.score = score + self.classId = classId + + def getObjId(self): + return self.objId + + def getMicId(self): + return self.micId + + def getX(self): + return self.x + + def getY(self): + return self.y + + def getScore(self): + return self.score + + def getClassId(self): + return self.classId + + def clone(self): + return FakeCoordinate( + objId=self.objId, + micId=self.micId, + x=self.x, + y=self.y, + score=self.score, + classId=self.classId, + ) + + +class FakeCoordinatesSet: + def __init__( + self, + micrographs, + coordinates, + boxSize=64, + ): + self.micrographsSet = FakeMicrographsSet(micrographs) + self.coordinates = list(coordinates) + self.boxSize = boxSize + + def getMicrographs(self): + return self.micrographsSet + + def iterItems(self, iterate=False): + return iter(self.coordinates) + + def iterCoordinates(self, micId): + return ( + coordinate + for coordinate in self.coordinates + if str(coordinate.getMicId()) == str(micId) + ) + + def getBoxSize(self): + return self.boxSize + + def getSize(self): + return len(self.coordinates) + + def getFirstItem(self): + return self.coordinates[0] if self.coordinates else None + + +@pytest.fixture +def coords2dServiceModule(authTestEnv): + return importlib.import_module("app.backend.api.services.coords2d_service") + + +@pytest.fixture +def currentProject(): + return FakeCurrentProject() + + +@pytest.fixture +def projectService(currentProject): + return FakeProjectService(currentProject) + + +@pytest.fixture +def service(coords2dServiceModule, projectService): + instance = object.__new__(coords2dServiceModule.Coords2dService) + instance.projectService = projectService + return instance + + +@pytest.fixture +def mapper(): + return FakeMapper() + + +@pytest.fixture +def currentUser(): + return {"id": 1} + + +def buildCoordinatesOutput(): + micrographA = FakeMicrograph( + objId=1, + location=(0, "/data/micrograph-a.mrc"), + micName="Micrograph A", + dims=(4096, 3072), + ) + micrographB = FakeMicrograph( + objId=2, + location="1@/data/micrograph-b.mrc", + label="Micrograph B label", + dims=(2048, 2048), + ) + + coordinateA = FakeCoordinate( + objId=101, + micId=1, + x=10.5, + y=20.5, + score=0.95, + classId="good", + ) + coordinateB = FakeCoordinate( + objId=102, + micId=1, + x=30.0, + y=40.0, + score=0.75, + classId="bad", + ) + coordinateC = FakeCoordinate( + objId=201, + micId=2, + x=15.0, + y=25.0, + score=0.60, + classId="ok", + ) + + return FakeCoordinatesSet( + micrographs=[micrographA, micrographB], + coordinates=[coordinateA, coordinateB, coordinateC], + boxSize=96, + ) + + +def test_LoadCoordinatesOutputResolvesPostgresqlProtocolId( + service, + currentProject, + projectService, + mapper, + currentUser, +): + protocol = FakeProtocol(objId=10) + coordinatesSet = buildCoordinatesOutput() + protocol.outputCoordinates = coordinatesSet + + currentProject.protocols[10] = protocol + projectService.runtimeProtocolIdByDbId[500] = 10 + + loadedProtocol, loadedCoordinatesSet = service._loadCoordinatesOutput( + mapper=mapper, + projectId=1, + currentUser=currentUser, + protocolId=500, + outputName="outputCoordinates", + ) + + assert loadedProtocol is protocol + assert loadedCoordinatesSet is coordinatesSet + + assert projectService.getProjectByIdCalls == [ + { + "mapper": mapper, + "projectId": 1, + "currentUser": currentUser, + "refresh": False, + "checkPid": False, + } + ] + + assert projectService.runtimeCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + + +def test_LoadCoordinatesOutputRaisesWhenProjectDoesNotExist( + service, + projectService, + mapper, + currentUser, +): + projectService.projectRow = None + + with pytest.raises(HTTPException) as exc: + service._loadCoordinatesOutput( + mapper=mapper, + projectId=1, + currentUser=currentUser, + protocolId=500, + outputName="outputCoordinates", + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Project not found" + assert projectService.runtimeCalls == [] + + +def test_LoadCoordinatesOutputRaisesWhenOutputIsMissing( + service, + currentProject, + projectService, + mapper, + currentUser, +): + protocol = FakeProtocol(objId=10) + + currentProject.protocols[10] = protocol + projectService.runtimeProtocolIdByDbId[500] = 10 + + with pytest.raises(HTTPException) as exc: + service._loadCoordinatesOutput( + mapper=mapper, + projectId=1, + currentUser=currentUser, + protocolId=500, + outputName="missingOutput", + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Output 'missingOutput' not found in protocol '500'" + + assert projectService.runtimeCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + + +def test_LoadCoordinatesOutputRaisesWhenOutputIsNotCoordinatesSet( + service, + currentProject, + projectService, + mapper, + currentUser, +): + protocol = FakeProtocol(objId=10) + protocol.outputVolume = object() + + currentProject.protocols[10] = protocol + projectService.runtimeProtocolIdByDbId[500] = 10 + + with pytest.raises(HTTPException) as exc: + service._loadCoordinatesOutput( + mapper=mapper, + projectId=1, + currentUser=currentUser, + protocolId=500, + outputName="outputVolume", + ) + + assert exc.value.status_code == 422 + assert exc.value.detail == "Output 'outputVolume' is not a SetOfCoordinates output" + + assert projectService.runtimeCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + diff --git a/tests/unit/backend/api/services/test_coords2d_service_postgresql.py b/tests/unit/backend/api/services/test_coords2d_service_postgresql.py new file mode 100644 index 00000000..bcf16778 --- /dev/null +++ b/tests/unit/backend/api/services/test_coords2d_service_postgresql.py @@ -0,0 +1,116 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from fastapi import HTTPException +import pytest + +from app.backend.api.services.coords2d_service import Coords2dService + + +class FakeReader: + def __init__(self, micrographs=None, coordinates=None): + self.micrographs = micrographs + self.coordinates = coordinates + self.lastSkipReason = None + + def listMicrographs(self): + return self.micrographs + + def listCoordinatesForMicrograph(self, micId): + return self.coordinates + + +def test_Coords2dServiceListMicrographsUsesPostgresqlReader(monkeypatch): + service = Coords2dService() + expected = { + "micrographs": [{"id": "10", "particles": 2}], + "totalMicrographs": 1, + "totalPicks": 2, + "boxSize": 128, + } + + monkeypatch.setattr( + service, + "_getPostgresqlCoords2dReaderIfAvailable", + lambda **kwargs: FakeReader(micrographs=expected), + ) + + payload = service.listMicrographs( + mapper=object(), + projectId=1, + currentUser={"id": 1}, + protocolId=2, + outputName="coordinates", + ) + + assert payload == expected + + +def test_Coords2dServiceListCoordinatesUsesPostgresqlReader(monkeypatch): + service = Coords2dService() + expected = { + "coordinates": [ + {"id": 1, "micId": "10", "x": 11.5, "y": 22.5} + ] + } + + monkeypatch.setattr( + service, + "_getPostgresqlCoords2dReaderIfAvailable", + lambda **kwargs: FakeReader(coordinates=expected), + ) + + payload = service.listCoordinatesForMicrograph( + mapper=object(), + projectId=1, + currentUser={"id": 1}, + protocolId=2, + outputName="coordinates", + micId="10", + ) + + assert payload == expected + + +def test_Coords2dServiceRaisesWhenPostgresqlReaderMissing(monkeypatch): + service = Coords2dService() + + monkeypatch.setattr( + service, + "_getPostgresqlCoords2dReaderIfAvailable", + lambda **kwargs: None, + ) + + with pytest.raises(HTTPException) as exc: + service.listMicrographs( + mapper=object(), + projectId=1, + currentUser={"id": 1}, + protocolId=2, + outputName="coordinates", + ) + + assert exc.value.status_code == 404 + assert "Coordinates2D output is not available in PostgreSQL metadata" in str(exc.value.detail) \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_analyze_viewer.py b/tests/unit/backend/api/services/test_project_service_analyze_viewer.py new file mode 100644 index 00000000..4ea93a87 --- /dev/null +++ b/tests/unit/backend/api/services/test_project_service_analyze_viewer.py @@ -0,0 +1,77 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +import importlib + + +class FakeDb: + def __init__(self): + self.fetchOneCalls = [] + + +class FakeMapper: + def __init__(self): + self.db = FakeDb() + + +def test_ResolveAnalyzeViewerDecisionReturnsHandledFalse(authTestEnv): + module = importlib.import_module("app.backend.api.services.project_service") + service = module.ProjectService() + mapper = FakeMapper() + + result = service.resolveAnalyzeViewerDecision( + mapper=mapper, + projectId=1, + protocolId=500, + ctx={ + "outputName": "outputVolume", + "pointerClass": "Volume", + "paramClass": "PointerParam", + }, + ) + + assert result == {"handled": False} + assert mapper.db.fetchOneCalls == [] + + +def test_ResolveAnalyzeViewerDecisionDoesNotResolveRuntimeObjects(authTestEnv): + module = importlib.import_module("app.backend.api.services.project_service") + service = module.ProjectService() + mapper = FakeMapper() + + result = service.resolveAnalyzeViewerDecision( + mapper=mapper, + projectId=1, + protocolId=500, + ctx={ + "outputName": "outputCustom", + "pointerClass": "UnknownExternalObject", + "paramClass": "PointerParam", + }, + ) + + assert result == {"handled": False} + assert mapper.db.fetchOneCalls == [] \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py new file mode 100644 index 00000000..e622465f --- /dev/null +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -0,0 +1,2995 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +import importlib + +import pytest + + +class FakeParam: + def __init__(self, value): + self.value = value + + +class FakeProtocol: + def __init__(self, status, outputs=None, steps=None, inputs=None, params=None, className="FakeProtocol"): + self.status = status + self.outputs = outputs or [] + self.steps = steps or [] + self.inputs = inputs or [] + self.params = params or {} + self.className = className + + def getClassName(self): + return self.className + + def getStatus(self): + return self.status + + def iterOutputAttributes(self): + return list(self.outputs) + + def loadSteps(self): + return list(self.steps) + + def iterInputAttributes(self): + return list(self.inputs) + + def iterParams(self): + return [ + (paramName, FakeParam(value)) + for paramName, value in self.params.items() + ] + + def getAttributeValue(self, paramName): + return self.params.get(paramName) + +class FakePointerTarget: + def __init__(self, objId, className): + self.objId = objId + self.className = className + + def getObjId(self): + return self.objId + + def getClassName(self): + return self.className + + +class FakePointer: + def __init__(self, parentProtocolId, outputName, className="SetOfParticles", objectId=100): + self.parent = FakePointerTarget(parentProtocolId, className) + self.target = FakePointerTarget(objectId, className) + self.outputName = outputName + + def getObjValue(self): + return self.parent + + def getExtended(self): + return self.outputName + + def get(self): + return self.target + + +class FakeOutput: + def __init__(self, className, itemsCount=None): + self.className = className + self.itemsCount = itemsCount + + def getClassName(self): + return self.className + + def getSize(self): + return self.itemsCount + + +class FakeValueHolder: + def __init__(self, value): + self.value = value + + def get(self): + return self.value + + +class FakeStep: + def __init__(self, index, name, status): + self.index = index + self.funcName = FakeValueHolder(name) + self.status = status + + def getIndex(self): + return self.index + + def getStatus(self): + return self.status + + def getClassName(self): + return "FakeStep" + + +class FakeParentNode: + def __init__(self, name): + self.name = name + + def getName(self): + return self.name + + +class FakeRunNode: + def __init__(self, name, status=None, parents=None): + self.name = name + self.run = FakeProtocol(status) if status is not None else None + self._parents = [FakeParentNode(parentName) for parentName in (parents or [])] + + def getName(self): + return self.name + + +class FakeRunsGraph: + def __init__(self, nodes): + self._nodesDict = nodes + + +class FakeCurrentProject: + def __init__(self, nodes): + self.nodes = nodes + self.lastRefresh = None + self.lastCheckPids = None + + def getRunsGraph(self, refresh=True, checkPids=True): + self.lastRefresh = refresh + self.lastCheckPids = checkPids + return FakeRunsGraph(self.nodes) + + +class FakeMapper: + def __init__( + self, + projectRow, + protocolRows, + adjacencyMap, + setRows=None, + treeRows=None, + stepsByProtocolId=None, + inputRefs=None, + ): + self.projectRow = projectRow + self.protocolRows = protocolRows + self.adjacencyMap = adjacencyMap + self.stepsByProtocolId = stepsByProtocolId or {} + self.inputRefs = inputRefs or [] + self.db = FakeDb( + setRows=setRows, + treeRows=treeRows, + ) + + def getProject(self, projectId, userId): + if int(projectId) != int(self.projectRow["id"]): + return None + if int(userId) != int(self.projectRow["ownerId"]): + return None + return dict(self.projectRow) + + def getProtocols(self, projectId): + return list(self.protocolRows) + + def getProjectProtocolAdjacencyMap(self, projectId): + return dict(self.adjacencyMap) + + def getProjectProtocolStepsByProtocolId(self, projectId): + return dict(self.stepsByProtocolId) + + def listProtocolInputRefs(self, projectId): + return list(self.inputRefs) + + +class FakeDb: + def __init__(self, setRows=None, treeRows=None): + self.setRows = setRows or [] + self.treeRows = treeRows or [] + self.fetchAllCalls = [] + + def fetchAll(self, query, params): + normalizedQuery = " ".join(str(query).split()) + + self.fetchAllCalls.append( + { + "query": query, + "params": params, + } + ) + + if "FROM scipion_objects o" in normalizedQuery: + return list(self.treeRows) + + if "FROM scipion_sets s" in normalizedQuery and "JOIN protocols p" in normalizedQuery: + return list(self.setRows) + + return [] + + +def makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=1, + itemsTableCount=None, + maxItemId=1, + maxItemIdFromItems=None, + columnsCount=2, + setColumnsCount=None, + rootTablesCount=1, + rootTableId=500, + rootTableItemsCount=None, + rootTableMaxItemId=None, + rootTableColumnsCount=None, + setColumnsSignature=None, + rootTableColumnsSignature=None, + itemsIdSignature="items-1-2-3", + rootTableItemsIdSignature=None, + itemsValueSignature="values-1-2-3", + rootTableItemsValueSignature=None, + propertiesPayloadSignature=None, + setPropertiesSignature=None, + propertiesPayloadCount=None, + setPropertiesCount=None, + protocolDbId=1000, + rootObjectDbId=None, + rootObjectProjectId=1, + rootObjectProtocolDbId=None, + rootObjectParentObjectId=None, + rootObjectName=None, + rootObjectPath=None, + rootObjectClassName=None, + rootObjectMissing=False, +): + defaultColumnsSignature = [ + { + "labelProperty": "_id", + "columnName": "id", + "className": "Integer", + "valueType": "int", + "position": 0, + "indexed": True, + }, + { + "labelProperty": "_enabled", + "columnName": "enabled", + "className": "Boolean", + "valueType": "bool", + "position": 1, + "indexed": False, + }, + ] + + resolvedItemsTableCount = ( + itemsCount if itemsTableCount is None else itemsTableCount + ) + resolvedMaxItemIdFromItems = ( + maxItemId if maxItemIdFromItems is None else maxItemIdFromItems + ) + resolvedSetColumnsCount = ( + columnsCount if setColumnsCount is None else setColumnsCount + ) + resolvedSetColumnsSignature = ( + defaultColumnsSignature + if setColumnsSignature is None + else setColumnsSignature + ) + + if rootTablesCount == 0: + resolvedRootTableId = None + resolvedRootTableItemsCount = 0 + resolvedRootTableMaxItemId = None + resolvedRootTableColumnsCount = 0 + resolvedRootTableColumnsSignature = [] + resolvedRootTableItemsIdSignature = None + resolvedRootTableItemsValueSignature = None + else: + resolvedRootTableId = rootTableId + resolvedRootTableItemsCount = ( + resolvedItemsTableCount + if rootTableItemsCount is None + else rootTableItemsCount + ) + resolvedRootTableMaxItemId = ( + resolvedMaxItemIdFromItems + if rootTableMaxItemId is None + else rootTableMaxItemId + ) + resolvedRootTableColumnsCount = ( + resolvedSetColumnsCount + if rootTableColumnsCount is None + else rootTableColumnsCount + ) + resolvedRootTableColumnsSignature = ( + resolvedSetColumnsSignature + if rootTableColumnsSignature is None + else rootTableColumnsSignature + ) + resolvedRootTableItemsIdSignature = ( + itemsIdSignature + if rootTableItemsIdSignature is None + else rootTableItemsIdSignature + ) + resolvedRootTableItemsValueSignature = ( + itemsValueSignature + if rootTableItemsValueSignature is None + else rootTableItemsValueSignature + ) + + defaultPropertiesSignature = [ + { + "key": "columnsCount", + "value": str(columnsCount), + }, + { + "key": "itemsCount", + "value": str(itemsCount), + }, + { + "key": "nestedTablesVersion", + "value": "14", + }, + ] + + resolvedPropertiesPayloadSignature = ( + defaultPropertiesSignature + if propertiesPayloadSignature is None + else propertiesPayloadSignature + ) + resolvedSetPropertiesSignature = ( + resolvedPropertiesPayloadSignature + if setPropertiesSignature is None + else setPropertiesSignature + ) + resolvedPropertiesPayloadCount = ( + len(resolvedPropertiesPayloadSignature) + if propertiesPayloadCount is None + else propertiesPayloadCount + ) + resolvedSetPropertiesCount = ( + len(resolvedSetPropertiesSignature) + if setPropertiesCount is None + else setPropertiesCount + ) + + if rootObjectMissing: + resolvedRootObjectDbId = None + resolvedRootObjectProjectId = None + resolvedRootObjectProtocolDbId = None + resolvedRootObjectParentObjectId = None + resolvedRootObjectName = None + resolvedRootObjectPath = None + resolvedRootObjectClassName = None + else: + resolvedRootObjectDbId = ( + 200 if rootObjectDbId is None else rootObjectDbId + ) + resolvedRootObjectProjectId = rootObjectProjectId + resolvedRootObjectProtocolDbId = ( + protocolDbId if rootObjectProtocolDbId is None else rootObjectProtocolDbId + ) + resolvedRootObjectParentObjectId = rootObjectParentObjectId + resolvedRootObjectName = ( + outputName if rootObjectName is None else rootObjectName + ) + resolvedRootObjectPath = ( + outputName if rootObjectPath is None else rootObjectPath + ) + resolvedRootObjectClassName = ( + setClassName if rootObjectClassName is None else rootObjectClassName + ) + + return { + "protocolId": str(protocolId), + "id": 100, + "objectId": 200, + "outputName": outputName, + "setClassName": setClassName, + "itemClassName": "Particle", + "properties": { + "itemsCount": itemsCount, + "maxItemId": maxItemId, + "columnsCount": columnsCount, + }, + "itemsTableCount": resolvedItemsTableCount, + "maxItemIdFromItems": resolvedMaxItemIdFromItems, + "itemsIdSignature": itemsIdSignature, + "setColumnsCount": resolvedSetColumnsCount, + "setColumnsSignature": resolvedSetColumnsSignature, + "rootTablesCount": rootTablesCount, + "rootTableId": resolvedRootTableId, + "rootTableItemsCount": resolvedRootTableItemsCount, + "rootTableMaxItemId": resolvedRootTableMaxItemId, + "rootTableItemsIdSignature": resolvedRootTableItemsIdSignature, + "rootTableColumnsCount": resolvedRootTableColumnsCount, + "rootTableColumnsSignature": resolvedRootTableColumnsSignature, + "createdAt": None, + "updatedAt": None, + "itemsValueSignature": itemsValueSignature, + "rootTableItemsValueSignature": resolvedRootTableItemsValueSignature, + "propertiesPayloadCount": resolvedPropertiesPayloadCount, + "propertiesPayloadSignature": resolvedPropertiesPayloadSignature, + "setPropertiesCount": resolvedSetPropertiesCount, + "setPropertiesSignature": resolvedSetPropertiesSignature, + "protocolDbId": protocolDbId, + "rootObjectDbId": resolvedRootObjectDbId, + "rootObjectProjectId": resolvedRootObjectProjectId, + "rootObjectProtocolDbId": resolvedRootObjectProtocolDbId, + "rootObjectParentObjectId": resolvedRootObjectParentObjectId, + "rootObjectName": resolvedRootObjectName, + "rootObjectPath": resolvedRootObjectPath, + "rootObjectClassName": resolvedRootObjectClassName, + } + + +@pytest.fixture +def projectServiceModule(authTestEnv): + return importlib.import_module("app.backend.api.services.project_service") + + +@pytest.fixture +def service(projectServiceModule): + instance = object.__new__(projectServiceModule.ProjectService) + instance.currentProject = None + instance.tomoList = {} + return instance + + +def patchRuntimeProject(service, monkeypatch, currentProject): + def loadProjectForThumbnails(dbProj): + service.currentProject = currentProject + return currentProject + + monkeypatch.setattr(service, "loadProjectForThumbnails", loadProjectForThumbnails) + + +def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode("10", status="finished", parents=["PROJECT"]), + "11": FakeRunNode("11", status="running", parents=["10"]), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=1)), + ] + currentProject.nodes["11"].run.inputs = [ + ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + + }, + protocolRows=[ + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + {"protocolId": "11", "status": "running", "protocolClassName": "FakeProtocol"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": ["11"]}, + "11": {"parents": ["10"], "children": []}, + }, + inputRefs=[ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "objectId": "100", + }, + ], + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=1, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=False, + checkPid=False, + ) + + assert result["ok"] is True + assert result["summary"] == { + "runtimeProtocols": 2, + "postgresqlProtocols": 2, + "runtimeDependencies": 1, + "postgresqlDependencies": 1, + "runtimeOutputs": 1, + "postgresqlOutputs": 1, + "runtimeSteps": 0, + "postgresqlSteps": 0, + "runtimeInputRefs": 1, + "postgresqlInputRefs": 1, + "runtimeInputRefDependencies": 1, + "postgresqlInputRefDependencies": 1, + "runtimeParams": 0, + "postgresqlParams": 0, + "issues": 0, + } + assert result["issues"] == { + "missingProtocols": [], + "extraProtocols": [], + "statusMismatches": [], + "missingDependencies": [], + "extraDependencies": [], + "missingOutputs": [], + "extraOutputs": [], + "missingSteps": [], + "extraSteps": [], + "stepMismatches": [], + "missingInputRefs": [], + "extraInputRefs": [], + "inputRefMismatches": [], + "runtimeInputRefDependenciesMissing": [], + "runtimeDependenciesWithoutInputRefs": [], + "postgresqlInputRefDependenciesMissing": [], + "postgresqlDependenciesWithoutInputRefs": [], + "missingParams": [], + "extraParams": [], + "paramValueMismatches": [], + "outputClassMismatches": [], + "outputMapperKindMismatches": [], + "outputItemsCountMismatches": [], + "postgresqlFlatSetMaxItemIdMismatches": [], + "protocolClassMismatches": [], + "postgresqlInputRefsWithMissingParentProtocols": [], + "postgresqlInputRefsWithMissingParentOutputs": [], + "postgresqlFlatSetOutputsWithIncompletePayload": [], + "postgresqlTreeOutputsWithIncompletePayload": [], + "postgresqlFlatSetItemsCountMismatches": [], + "postgresqlFlatSetRootTableMismatches": [], + "postgresqlFlatSetColumnsCountMismatches": [], + "postgresqlFlatSetPropertiesMismatches": [], + "postgresqlFlatSetRootObjectMismatches": [], + } + assert currentProject.lastRefresh is False + assert currentProject.lastCheckPids is False + + +def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode("10", status="finished", parents=["PROJECT"]), + "11": FakeRunNode("11", status="running", parents=["10"]), + } + ) + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "failed"}, + {"protocolId": "12", "status": "finished"}, + ], + adjacencyMap={ + "10": {"parents": ["99"], "children": []}, + "12": {"parents": [], "children": []}, + }, + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"] == { + "runtimeProtocols": 2, + "postgresqlProtocols": 2, + "runtimeDependencies": 1, + "postgresqlDependencies": 1, + "runtimeOutputs": 0, + "postgresqlOutputs": 0, + "runtimeSteps": 0, + "postgresqlSteps": 0, + "runtimeInputRefs": 0, + "postgresqlInputRefs": 0, + "runtimeInputRefDependencies": 0, + "postgresqlInputRefDependencies": 0, + "runtimeParams": 0, + "postgresqlParams": 0, + "issues": 7, + } + assert result["issues"] == { + "missingProtocols": [ + { + "protocolId": "11", + "runtimeStatus": "running", + } + ], + "extraProtocols": [ + { + "protocolId": "12", + "postgresqlStatus": "finished", + } + ], + "statusMismatches": [ + { + "protocolId": "10", + "runtimeStatus": "finished", + "postgresqlStatus": "failed", + } + ], + "missingDependencies": [ + { + "parentId": "10", + "childId": "11", + } + ], + "extraDependencies": [ + { + "parentId": "99", + "childId": "10", + } + ], + "missingOutputs": [], + "extraOutputs": [], + "missingSteps": [], + "extraSteps": [], + "stepMismatches": [], + "missingInputRefs": [], + "extraInputRefs": [], + "inputRefMismatches": [], + "runtimeInputRefDependenciesMissing": [], + "runtimeDependenciesWithoutInputRefs": [ + { + "parentId": "10", + "childId": "11", + } + ], + "postgresqlInputRefDependenciesMissing": [], + "postgresqlDependenciesWithoutInputRefs": [ + { + "parentId": "99", + "childId": "10", + } + ], + "missingParams": [], + "extraParams": [], + "paramValueMismatches": [], + "outputClassMismatches": [], + "outputMapperKindMismatches": [], + "outputItemsCountMismatches": [], + "postgresqlFlatSetMaxItemIdMismatches": [], + "protocolClassMismatches": [], + "postgresqlInputRefsWithMissingParentProtocols": [], + "postgresqlInputRefsWithMissingParentOutputs": [], + "postgresqlFlatSetOutputsWithIncompletePayload": [], + "postgresqlTreeOutputsWithIncompletePayload": [], + "postgresqlFlatSetItemsCountMismatches": [], + "postgresqlFlatSetRootTableMismatches": [], + "postgresqlFlatSetColumnsCountMismatches": [], + "postgresqlFlatSetPropertiesMismatches": [], + "postgresqlFlatSetRootObjectMismatches": [], + } + + +def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles")), + ("outputVolume", FakeOutput("Volume")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=12, + ), + ], + treeRows=[ + { + "protocolId": "10", + "id": 300, + "scipionObjId": 400, + "name": "outputExtra", + "path": "outputExtra", + "className": "Volume", + "value": None, + "label": None, + "comment": None, + "metadata": {}, + "createdAt": None, + "updatedAt": None, + } + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"] == { + "runtimeProtocols": 1, + "postgresqlProtocols": 1, + "runtimeDependencies": 0, + "postgresqlDependencies": 0, + "runtimeOutputs": 2, + "postgresqlOutputs": 2, + "runtimeSteps": 0, + "postgresqlSteps": 0, + "runtimeInputRefs": 0, + "postgresqlInputRefs": 0, + "runtimeInputRefDependencies": 0, + "postgresqlInputRefDependencies": 0, + "runtimeParams": 0, + "postgresqlParams": 0, + "issues": 2, + } + assert result["issues"]["missingOutputs"] == [ + { + "protocolId": "10", + "outputName": "outputVolume", + "className": "Volume", + } + ] + assert result["issues"]["extraOutputs"] == [ + { + "protocolId": "10", + "outputName": "outputExtra", + "mapperKind": "tree", + "className": "Volume", + } + ] + assert result["issues"]["missingProtocols"] == [] + assert result["issues"]["extraProtocols"] == [] + assert result["issues"]["statusMismatches"] == [] + assert result["issues"]["missingDependencies"] == [] + assert result["issues"]["extraDependencies"] == [] + assert result["issues"]["missingSteps"] == [] + assert result["issues"]["extraSteps"] == [] + assert result["issues"]["stepMismatches"] == [] + assert result["issues"]["missingInputRefs"] == [] + assert result["issues"]["extraInputRefs"] == [] + assert result["issues"]["inputRefMismatches"] == [] + assert result["issues"]["runtimeInputRefDependenciesMissing"] == [] + assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] + assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] + assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] + assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="running", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.steps = [ + FakeStep(index=1, name="importStep", status="finished"), + FakeStep(index=2, name="processStep", status="running"), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "running"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + stepsByProtocolId={ + "10": [ + { + "index": 1, + "name": "importStep", + "status": "finished", + }, + { + "index": 2, + "name": "processStep", + "status": "scheduled", + }, + { + "index": 3, + "name": "extraStep", + "status": "new", + }, + ], + }, + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"] == { + "runtimeProtocols": 1, + "postgresqlProtocols": 1, + "runtimeDependencies": 0, + "postgresqlDependencies": 0, + "runtimeOutputs": 0, + "postgresqlOutputs": 0, + "runtimeSteps": 2, + "postgresqlSteps": 3, + "runtimeInputRefs": 0, + "postgresqlInputRefs": 0, + "runtimeInputRefDependencies": 0, + "postgresqlInputRefDependencies": 0, + "runtimeParams": 0, + "postgresqlParams": 0, + "issues": 2, + } + assert result["issues"]["missingSteps"] == [] + assert result["issues"]["extraSteps"] == [ + { + "protocolId": "10", + "index": 3, + "name": "extraStep", + "status": "new", + } + ] + assert result["issues"]["stepMismatches"] == [ + { + "protocolId": "10", + "index": 2, + "fields": ["status"], + "runtimeName": "processStep", + "postgresqlName": "processStep", + "runtimeStatus": "running", + "postgresqlStatus": "scheduled", + } + ] + assert result["issues"]["missingProtocols"] == [] + assert result["issues"]["extraProtocols"] == [] + assert result["issues"]["statusMismatches"] == [] + assert result["issues"]["missingDependencies"] == [] + assert result["issues"]["extraDependencies"] == [] + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["missingInputRefs"] == [] + assert result["issues"]["extraInputRefs"] == [] + assert result["issues"]["inputRefMismatches"] == [] + assert result["issues"]["runtimeInputRefDependenciesMissing"] == [] + assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] + assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] + assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] + assert result["issues"]["missingParams"] == [] + assert result["issues"]["extraParams"] == [] + assert result["issues"]["paramValueMismatches"] == [] + assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + "11": FakeRunNode( + "11", + status="running", + parents=["10"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=1)), + ] + currentProject.nodes["11"].run.inputs = [ + ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), + ] + currentProject.nodes["10"].run.steps = [ + FakeStep(index=1, name="importStep", status="finished"), + ] + currentProject.nodes["11"].run.steps = [ + FakeStep(index=1, name="processStep", status="running"), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + {"protocolId": "11", "status": "running", "protocolClassName": "FakeProtocol"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": ["11"]}, + "11": {"parents": ["10"], "children": []}, + }, + stepsByProtocolId={ + "10": [ + { + "index": 1, + "name": "importStep", + "status": "finished", + } + ], + "11": [ + { + "index": 1, + "name": "processStep", + "status": "running", + } + ], + }, + inputRefs=[ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "objectId": "100", + }, + ], + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=1, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is True + assert result["summary"]["runtimeSteps"] == 2 + assert result["summary"]["postgresqlSteps"] == 2 + assert result["issues"]["missingSteps"] == [] + assert result["issues"]["extraSteps"] == [] + assert result["issues"]["stepMismatches"] == [] + assert result["summary"]["runtimeInputRefs"] == 1 + assert result["summary"]["postgresqlInputRefs"] == 1 + assert result["summary"]["runtimeInputRefDependencies"] == 1 + assert result["summary"]["postgresqlInputRefDependencies"] == 1 + + assert result["issues"]["runtimeInputRefDependenciesMissing"] == [] + assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] + assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] + assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] + assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + "11": FakeRunNode( + "11", + status="running", + parents=["10"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=1)), + ] + currentProject.nodes["11"].run.inputs = [ + ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), + ("inputVolume", FakePointer(parentProtocolId=10, outputName="outputVolume", className="Volume")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + {"protocolId": "11", "status": "running", "protocolClassName": "FakeProtocol"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": ["11"]}, + "11": {"parents": ["10"], "children": []}, + }, + inputRefs=[ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "wrongOutput", + "objectClassName": "SetOfParticles", + "objectId": "100", + }, + { + "protocolId": "11", + "inputName": "inputMask", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputMask", + "objectClassName": "VolumeMask", + "objectId": "101", + }, + ], + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=1, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"] == { + "runtimeProtocols": 2, + "postgresqlProtocols": 2, + "runtimeDependencies": 1, + "postgresqlDependencies": 1, + "runtimeOutputs": 1, + "postgresqlOutputs": 1, + "runtimeSteps": 0, + "postgresqlSteps": 0, + "runtimeInputRefs": 2, + "postgresqlInputRefs": 2, + "runtimeInputRefDependencies": 1, + "postgresqlInputRefDependencies": 1, + "runtimeParams": 0, + "postgresqlParams": 0, + "issues": 3, + } + + assert result["issues"]["missingInputRefs"] == [ + { + "protocolId": "11", + "inputName": "inputVolume", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputVolume", + "objectClassName": "Volume", + } + ] + + assert result["issues"]["extraInputRefs"] == [ + { + "protocolId": "11", + "inputName": "inputMask", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputMask", + "objectClassName": "VolumeMask", + } + ] + + assert result["issues"]["inputRefMismatches"] == [ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "fields": ["parentOutputName"], + "runtimeParentProtocolId": "10", + "postgresqlParentProtocolId": "10", + "runtimeParentOutputName": "outputParticles", + "postgresqlParentOutputName": "wrongOutput", + "runtimeObjectClassName": "SetOfParticles", + "postgresqlObjectClassName": "SetOfParticles", + } + ] + assert result["issues"]["postgresqlInputRefsWithMissingParentProtocols"] == [] + assert result["issues"]["postgresqlInputRefsWithMissingParentOutputs"] == [] + + assert result["issues"]["missingProtocols"] == [] + assert result["issues"]["extraProtocols"] == [] + assert result["issues"]["statusMismatches"] == [] + assert result["issues"]["missingDependencies"] == [] + assert result["issues"]["extraDependencies"] == [] + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["missingSteps"] == [] + assert result["issues"]["extraSteps"] == [] + assert result["issues"]["stepMismatches"] == [] + assert result["issues"]["runtimeInputRefDependenciesMissing"] == [] + assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] + assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] + assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] + assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + "11": FakeRunNode( + "11", + status="running", + parents=["10"], + ), + "12": FakeRunNode( + "12", + status="running", + parents=[], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=1)), + ("outputVolume", FakeOutput("Volume")), + ] + currentProject.nodes["11"].run.inputs = [ + ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), + ] + currentProject.nodes["12"].run.inputs = [ + ("inputVolume", FakePointer(parentProtocolId=10, outputName="outputVolume", className="Volume")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + {"protocolId": "11", "status": "running", "protocolClassName": "FakeProtocol"}, + {"protocolId": "12", "status": "running", "protocolClassName": "FakeProtocol"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": ["11"]}, + "11": {"parents": ["10"], "children": []}, + "12": {"parents": [], "children": []}, + }, + inputRefs=[ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "objectId": "100", + }, + { + "protocolId": "12", + "inputName": "inputVolume", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputVolume", + "objectClassName": "Volume", + "objectId": "101", + }, + ], + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=1, + ), + ], + treeRows=[ + { + "protocolId": "10", + "id": 300, + "scipionObjId": 400, + "name": "outputVolume", + "path": "outputVolume", + "className": "Volume", + "value": None, + "label": None, + "comment": None, + "metadata": {}, + "createdAt": None, + "updatedAt": None, + }, + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"] == { + "runtimeProtocols": 3, + "postgresqlProtocols": 3, + "runtimeDependencies": 1, + "postgresqlDependencies": 1, + "runtimeOutputs": 2, + "postgresqlOutputs": 2, + "runtimeSteps": 0, + "postgresqlSteps": 0, + "runtimeInputRefs": 2, + "postgresqlInputRefs": 2, + "runtimeInputRefDependencies": 2, + "postgresqlInputRefDependencies": 2, + "runtimeParams": 0, + "postgresqlParams": 0, + "issues": 2, + } + + assert result["issues"]["runtimeInputRefDependenciesMissing"] == [ + { + "parentId": "10", + "childId": "12", + } + ] + assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] + assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [ + { + "parentId": "10", + "childId": "12", + } + ] + assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] + + assert result["issues"]["missingProtocols"] == [] + assert result["issues"]["extraProtocols"] == [] + assert result["issues"]["statusMismatches"] == [] + assert result["issues"]["missingDependencies"] == [] + assert result["issues"]["extraDependencies"] == [] + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["missingSteps"] == [] + assert result["issues"]["extraSteps"] == [] + assert result["issues"]["stepMismatches"] == [] + assert result["issues"]["missingInputRefs"] == [] + assert result["issues"]["extraInputRefs"] == [] + assert result["issues"]["inputRefMismatches"] == [] + assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] + assert result["issues"]["postgresqlInputRefsWithMissingParentProtocols"] == [] + assert result["issues"]["postgresqlInputRefsWithMissingParentOutputs"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsParamMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.params = { + "boxSize": 128, + "threshold": 0.5, + } + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + "params": { + "boxSize": 256, + "extraParam": "abc", + }, + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeParams"] == 2 + assert result["summary"]["postgresqlParams"] == 2 + assert result["issues"]["missingParams"] == [ + { + "protocolId": "10", + "paramName": "threshold", + "value": 0.5, + } + ] + assert result["issues"]["extraParams"] == [ + { + "protocolId": "10", + "paramName": "extraParam", + "value": "abc", + } + ] + assert result["issues"]["paramValueMismatches"] == [ + { + "protocolId": "10", + "paramName": "boxSize", + "runtimeValue": 128, + "postgresqlValue": 256, + } + ] + assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsOutputClassMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="Volume", + itemsCount=12, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeOutputs"] == 1 + assert result["summary"]["postgresqlOutputs"] == 1 + assert result["summary"]["issues"] == 1 + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["outputClassMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "runtimeClassName": "SetOfParticles", + "postgresqlClassName": "Volume", + "mapperKind": "flat_set", + } + ] + assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsOutputMapperKindMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + treeRows=[ + { + "protocolId": "10", + "id": 300, + "scipionObjId": 400, + "name": "outputParticles", + "path": "outputParticles", + "className": "SetOfParticles", + "value": None, + "label": None, + "comment": None, + "metadata": {}, + "createdAt": None, + "updatedAt": None, + }, + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeOutputs"] == 1 + assert result["summary"]["postgresqlOutputs"] == 1 + assert result["summary"]["issues"] == 1 + + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "className": "SetOfParticles", + "expectedMapperKind": "flat_set", + "postgresqlMapperKind": "tree", + } + ] + assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsOutputItemsCountMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=12)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=10, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeOutputs"] == 1 + assert result["summary"]["postgresqlOutputs"] == 1 + assert result["summary"]["issues"] == 1 + + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "className": "SetOfParticles", + "runtimeItemsCount": 12, + "postgresqlItemsCount": 10, + "mapperKind": "flat_set", + } + ] + assert result["issues"]["protocolClassMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsFlatSetItemsTableCountMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=12)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=12, + itemsTableCount=10, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeOutputs"] == 1 + assert result["summary"]["postgresqlOutputs"] == 1 + assert result["summary"]["issues"] == 1 + + assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["postgresqlFlatSetItemsCountMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "itemsCount": 12, + "itemsTableCount": 10, + "itemClassName": "Particle", + } + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsProtocolClassMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.className = "RuntimeProtocolClass" + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + "protocolClassName": "PostgresqlProtocolClass", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeProtocols"] == 1 + assert result["summary"]["postgresqlProtocols"] == 1 + assert result["summary"]["issues"] == 1 + assert result["issues"]["protocolClassMismatches"] == [ + { + "protocolId": "10", + "runtimeClassName": "RuntimeProtocolClass", + "postgresqlClassName": "PostgresqlProtocolClass", + } + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsInputRefMissingParentProtocol( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "11": FakeRunNode("11", status="running", parents=["PROJECT"]), + } + ) + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "11", "status": "running", "protocolClassName": "FakeProtocol"}, + ], + adjacencyMap={ + "11": {"parents": [], "children": []}, + }, + inputRefs=[ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "99", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "objectId": "100", + }, + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=False, + checkPid=False, + ) + + assert result["ok"] is False + assert result["issues"]["postgresqlInputRefsWithMissingParentProtocols"] == [ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "99", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "missingParentProtocolId": "99", + } + ] + assert result["issues"]["postgresqlInputRefsWithMissingParentOutputs"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsInputRefMissingParentOutput( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode("10", status="finished", parents=["PROJECT"]), + "11": FakeRunNode("11", status="running", parents=["10"]), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles")), + ] + currentProject.nodes["11"].run.inputs = [ + ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + {"protocolId": "11", "status": "running", "protocolClassName": "FakeProtocol"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": ["11"]}, + "11": {"parents": ["10"], "children": []}, + }, + inputRefs=[ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "objectId": "100", + }, + ], + setRows=[], + treeRows=[], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=False, + checkPid=False, + ) + + assert result["ok"] is False + assert result["issues"]["postgresqlInputRefsWithMissingParentProtocols"] == [] + assert result["issues"]["postgresqlInputRefsWithMissingParentOutputs"] == [ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "missingParentOutputName": "outputParticles", + } + ] + +def test_ValidateProjectPostgresqlConsistencyReportsIncompletePostgresqlOutputPayloads( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=None)), + ("outputVolume", FakeOutput("Volume")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=None, + ), + ], + treeRows=[ + { + "protocolId": "10", + "id": None, + "scipionObjId": 400, + "name": "outputVolume", + "path": "outputVolume", + "className": "Volume", + "value": None, + "label": None, + "comment": None, + "metadata": {}, + "createdAt": None, + "updatedAt": None, + }, + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeOutputs"] == 2 + assert result["summary"]["postgresqlOutputs"] == 2 + assert result["summary"]["issues"] == 2 + + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] + + assert result["issues"]["postgresqlFlatSetOutputsWithIncompletePayload"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "missingFields": ["itemsCount"], + "setId": 100, + "rootObjectId": 200, + "itemsCount": None, + "itemClassName": "Particle", + } + ] + + assert result["issues"]["postgresqlTreeOutputsWithIncompletePayload"] == [ + { + "protocolId": "10", + "outputName": "outputVolume", + "mapperKind": "tree", + "className": "Volume", + "missingFields": ["rootObjectId"], + "rootObjectId": None, + "scipionObjId": 400, + } + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsFlatSetMaxItemIdMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=20, + maxItemIdFromItems=30, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetItemsCountMismatches"] == [] + assert result["issues"]["postgresqlFlatSetMaxItemIdMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "maxItemId": 20, + "maxItemIdFromItems": 30, + "itemClassName": "Particle", + } + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsMissingFlatSetRootTable( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + rootTablesCount=0, + rootTableId=None, + rootTableItemsCount=0, + rootTableMaxItemId=None, + rootTableColumnsCount=0, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "rootTableId": None, + "fields": ["rootTableMissing"], + "rootTablesCount": 0, + "itemsTableCount": 3, + "rootTableItemsCount": 0, + "maxItemIdFromItems": 30, + "rootTableMaxItemId": None, + "setColumnsCount": 2, + "rootTableColumnsCount": 0, + "itemClassName": "Particle", + } + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsFlatSetColumnsCountMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + columnsCount=5, + setColumnsCount=4, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeOutputs"] == 1 + assert result["summary"]["postgresqlOutputs"] == 1 + assert result["summary"]["issues"] == 1 + + assert result["issues"]["postgresqlFlatSetColumnsCountMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "columnsCount": 5, + "setColumnsCount": 4, + "itemClassName": "Particle", + } + ] + assert result["issues"]["postgresqlFlatSetItemsCountMismatches"] == [] + assert result["issues"]["postgresqlFlatSetMaxItemIdMismatches"] == [] + assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsRootTableColumnsCountMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + columnsCount=5, + setColumnsCount=5, + rootTablesCount=1, + rootTableId=500, + rootTableItemsCount=3, + rootTableMaxItemId=30, + rootTableColumnsCount=4, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "rootTableId": 500, + "fields": ["rootTableColumnsCount"], + "rootTablesCount": 1, + "itemsTableCount": 3, + "rootTableItemsCount": 3, + "maxItemIdFromItems": 30, + "rootTableMaxItemId": 30, + "setColumnsCount": 5, + "rootTableColumnsCount": 4, + "itemClassName": "Particle", + } + ] + assert result["issues"]["postgresqlFlatSetColumnsCountMismatches"] == [] + assert result["issues"]["postgresqlFlatSetItemsCountMismatches"] == [] + assert result["issues"]["postgresqlFlatSetMaxItemIdMismatches"] == [] + +def test_ValidateProjectPostgresqlConsistencyReportsRootTableColumnSchemaMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + setColumnsSignature = [ + { + "labelProperty": "_id", + "columnName": "id", + "className": "Integer", + "valueType": "int", + "position": 0, + "indexed": True, + }, + { + "labelProperty": "_enabled", + "columnName": "enabled", + "className": "Boolean", + "valueType": "bool", + "position": 1, + "indexed": False, + }, + ] + + rootTableColumnsSignature = [ + { + "labelProperty": "_id", + "columnName": "id", + "className": "Integer", + "valueType": "int", + "position": 0, + "indexed": True, + }, + { + "labelProperty": "_enabled", + "columnName": "wrong_enabled", + "className": "Boolean", + "valueType": "bool", + "position": 1, + "indexed": False, + }, + ] + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + columnsCount=2, + setColumnsCount=2, + rootTablesCount=1, + rootTableId=500, + rootTableItemsCount=3, + rootTableMaxItemId=30, + rootTableColumnsCount=2, + setColumnsSignature=setColumnsSignature, + rootTableColumnsSignature=rootTableColumnsSignature, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "rootTableId": 500, + "fields": ["rootTableColumnsSignature"], + "rootTablesCount": 1, + "itemsTableCount": 3, + "rootTableItemsCount": 3, + "maxItemIdFromItems": 30, + "rootTableMaxItemId": 30, + "setColumnsCount": 2, + "rootTableColumnsCount": 2, + "setColumnsSignature": setColumnsSignature, + "rootTableColumnsSignature": rootTableColumnsSignature, + "itemClassName": "Particle", + } + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsRootTableItemsIdSignatureMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + rootTablesCount=1, + rootTableId=500, + rootTableItemsCount=3, + rootTableMaxItemId=30, + itemsIdSignature="items-10-20-30", + rootTableItemsIdSignature="items-10-25-30", + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "rootTableId": 500, + "fields": ["rootTableItemsIdSignature"], + "rootTablesCount": 1, + "itemsTableCount": 3, + "rootTableItemsCount": 3, + "maxItemIdFromItems": 30, + "rootTableMaxItemId": 30, + "setColumnsCount": 2, + "rootTableColumnsCount": 2, + "itemsIdSignature": "items-10-20-30", + "rootTableItemsIdSignature": "items-10-25-30", + "itemClassName": "Particle", + } + ] + +def test_ValidateProjectPostgresqlConsistencyReportsRootTableItemsValueSignatureMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + rootTablesCount=1, + rootTableId=500, + rootTableItemsCount=3, + rootTableMaxItemId=30, + itemsIdSignature="items-10-20-30", + rootTableItemsIdSignature="items-10-20-30", + itemsValueSignature="values-original", + rootTableItemsValueSignature="values-corrupted", + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "rootTableId": 500, + "fields": ["rootTableItemsValueSignature"], + "rootTablesCount": 1, + "itemsTableCount": 3, + "rootTableItemsCount": 3, + "maxItemIdFromItems": 30, + "rootTableMaxItemId": 30, + "setColumnsCount": 2, + "rootTableColumnsCount": 2, + "itemsValueSignature": "values-original", + "rootTableItemsValueSignature": "values-corrupted", + "itemClassName": "Particle", + } + ] + +def test_ValidateProjectPostgresqlConsistencyReportsFlatSetPropertiesMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + propertiesPayloadSignature = [ + { + "key": "columnsCount", + "value": "2", + }, + { + "key": "itemsCount", + "value": "3", + }, + { + "key": "nestedTablesVersion", + "value": "14", + }, + ] + setPropertiesSignature = [ + { + "key": "columnsCount", + "value": "2", + }, + { + "key": "itemsCount", + "value": "4", + }, + { + "key": "nestedTablesVersion", + "value": "14", + }, + ] + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + propertiesPayloadSignature=propertiesPayloadSignature, + setPropertiesSignature=setPropertiesSignature, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetPropertiesMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "fields": ["setPropertiesSignature"], + "propertiesPayloadCount": 3, + "setPropertiesCount": 3, + "propertiesPayloadSignature": propertiesPayloadSignature, + "setPropertiesSignature": setPropertiesSignature, + "itemClassName": "Particle", + } + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsFlatSetRootObjectMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "id": 1000, + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + protocolDbId=1000, + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + rootObjectProjectId=2, + rootObjectProtocolDbId=9999, + rootObjectParentObjectId=123, + rootObjectName="wrongName", + rootObjectPath="wrongPath", + rootObjectClassName="WrongSetClass", + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootObjectMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "fields": [ + "rootObjectProjectId", + "rootObjectProtocolDbId", + "rootObjectParentObjectId", + "rootObjectName", + "rootObjectPath", + "rootObjectClassName", + ], + "protocolDbId": 1000, + "rootObjectDbId": 200, + "rootObjectProjectId": 2, + "rootObjectProtocolDbId": 9999, + "rootObjectParentObjectId": 123, + "rootObjectName": "wrongName", + "rootObjectPath": "wrongPath", + "rootObjectClassName": "WrongSetClass", + "itemClassName": "Particle", + } + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsMissingFlatSetRootObject( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "id": 1000, + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + protocolDbId=1000, + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + rootObjectMissing=True, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootObjectMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "fields": ["rootObjectMissing"], + "protocolDbId": 1000, + "rootObjectDbId": None, + "rootObjectProjectId": None, + "rootObjectProtocolDbId": None, + "rootObjectParentObjectId": None, + "rootObjectName": None, + "rootObjectPath": None, + "rootObjectClassName": None, + "itemClassName": "Particle", + } + ] \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_coords3d.py b/tests/unit/backend/api/services/test_project_service_coords3d.py index c57627d0..4907f44b 100644 --- a/tests/unit/backend/api/services/test_project_service_coords3d.py +++ b/tests/unit/backend/api/services/test_project_service_coords3d.py @@ -300,6 +300,132 @@ def test_ListCoordinates3dTomogramsServiceBuildsTomogramList(service, tmp_path): assert service.tomoList["TS_002"] is tomo2 +def test_GetPostgresqlCoords3dReaderIfAvailableUsesResolvedProtocolDbId( + service, + monkeypatch, +): + createdReaders = [] + + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakePostgresqlCoords3dReader: + # fakePostgresqlCoords3dReader + def __init__(self, db, projectId, protocolId, outputName): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + createdReaders.append(self) + + def hasOutput(self): + return True + + readerModule = importlib.import_module( + "app.backend.viewers.postgresql_coords3d_reader" + ) + + monkeypatch.setattr( + readerModule, + "PostgresqlCoords3dReader", + FakePostgresqlCoords3dReader, + ) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 987, + ) + + mapper = FakeMapper() + + reader = service._getPostgresqlCoords3dReaderIfAvailable( + mapper=mapper, + projectId=1, + protocolId=10, + outputName="outputCoords3d", + ) + + assert reader is createdReaders[0] + assert createdReaders[0].db is mapper.db + assert createdReaders[0].projectId == 1 + assert createdReaders[0].protocolId == 987 + assert createdReaders[0].outputName == "outputCoords3d" + + +@pytest.mark.parametrize( + "serviceCall, expectedDetail", + [ + ( + lambda service, mapper: service.listCoordinates3dTomogramsService( + projectId=1, + protocolId=10, + outputName="outputCoords3d", + mapper=mapper, + ), + "Coordinates3D output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.getCoordinates3dPointsService( + projectId=1, + protocolId=10, + outputName="outputCoords3d", + tomogramId="TS_001", + mapper=mapper, + ), + "Coordinates3D points output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.renderCoords3dTomogramSliceService( + projectId=1, + protocolId=10, + outputName="outputCoords3d", + tomogramId="TS_001", + sliceIndex=0, + axis="z", + colormap=None, + normalize="minmax", + scale=1.0, + inline=True, + fmt="png", + thumb=None, + fast=True, + quality=75, + mapper=mapper, + ), + "Coordinates3D tomogram slice output is not available in PostgreSQL metadata", + ), + ], +) +def test_Coordinates3dServicesRequirePostgresqlWhenMapperIsPresent( + service, + monkeypatch, + serviceCall, + expectedDetail, +): + monkeypatch.setattr( + service, + "_getPostgresqlCoords3dReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy Coordinates3D fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForCoordinates3d", failRuntimeFallback) + + with pytest.raises(HTTPException) as exc: + serviceCall(service, object()) + + assert exc.value.status_code == 404 + assert expectedDetail in exc.value.detail + + def test_GetCoordinates3dPointsServiceBuildsPointPayload(service, tmp_path): tomoPath = tmp_path / "tomo1.mrc" tomoPath.write_text("placeholder", encoding="utf-8") @@ -482,16 +608,14 @@ def test_CreateCoords3dOutputFromPointsServiceCreatesNewOutput(projectServiceMod payload=payload, ) - assert result == { - "success": True, - "outputName": "outputCoords3d_edited", - "message": "Created new coords3d output 'outputCoords3d_edited'", - "data": { - "sourceOutputName": "outputCoords3d", - "replacedPoints": 2, - "copiedPoints": 0, - }, - } + assert result["success"] is True + assert result["outputName"] == "outputCoords3d_edited" + assert result["message"] == "Created new coords3d output 'outputCoords3d_edited'" + assert result["data"]["sourceOutputName"] == "outputCoords3d" + assert result["data"]["replacedPoints"] == 2 + assert result["data"]["copiedPoints"] == 0 + assert result["data"]["postgresqlStored"] is False + assert result["data"]["postgresqlError"] is None assert "outputCoords3d_edited" in protocol.definedOutputs createdSet = protocol.definedOutputs["outputCoords3d_edited"] diff --git a/tests/unit/backend/api/services/test_project_service_ctftomo.py b/tests/unit/backend/api/services/test_project_service_ctftomo.py index e37f624f..a1589b4e 100644 --- a/tests/unit/backend/api/services/test_project_service_ctftomo.py +++ b/tests/unit/backend/api/services/test_project_service_ctftomo.py @@ -372,6 +372,148 @@ def test_ListOutputCtftomoSeriesServiceBuildsSummaries(service, tmp_path): ] +def test_GetCtftomoSeriesViewsServiceRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + monkeypatch.setattr( + service, + "_getPostgresqlCtftomoReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy CTFTomo views fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForCtftomoSeries", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service.getCtftomoSeriesViewsService( + projectId=1, + protocolId=10, + outputName="outputCtftomo", + tiltSeriesId="TS_001", + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "CTFTomo views output is not available in PostgreSQL metadata" in exc.value.detail + assert "reader_not_available" in exc.value.detail + + +def test_ListOutputCtftomoSeriesServiceRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + monkeypatch.setattr( + service, + "_getPostgresqlCtftomoReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy CTFTomo fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForCtftomoSeries", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service.listOutputCtftomoSeriesService( + projectId=1, + protocolId=10, + outputName="outputCtftomo", + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "CTFTomo output is not available in PostgreSQL metadata" in exc.value.detail + assert "reader_not_available" in exc.value.detail + + +def test_RenderCtfTomoPsdImageServiceRequiresPostgresqlPathWhenMapperIsPresent( + service, + monkeypatch, +): + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy CTFTomo PSD fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForCtftomoSeries", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service.renderCtfTomoPsdImageService( + projectId=1, + protocolId=10, + outputName="outputCtftomo", + psdPath="relative/psd.mrc", + size=512, + fmt="png", + inline=True, + index=0, + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "CTFTomo PSD output is not available in PostgreSQL metadata" in exc.value.detail + assert "psd_file_not_available" in exc.value.detail + + +def test_GetPostgresqlCtftomoReaderIfAvailableUsesResolvedProtocolDbId( + service, + monkeypatch, +): + createdReaders = [] + + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakePostgresqlCtftomoReader: + # fakePostgresqlCtftomoReader + def __init__(self, db, projectId, protocolId, outputName): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + createdReaders.append(self) + + def hasOutput(self): + return True + + readerModule = importlib.import_module( + "app.backend.viewers.postgresql_ctftomo_reader" + ) + + monkeypatch.setattr( + readerModule, + "PostgresqlCtftomoReader", + FakePostgresqlCtftomoReader, + ) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 654, + ) + + mapper = FakeMapper() + + reader = service._getPostgresqlCtftomoReaderIfAvailable( + mapper=mapper, + projectId=1, + protocolId=10, + outputName="outputCtftomo", + ) + + assert reader is createdReaders[0] + assert createdReaders[0].db is mapper.db + assert createdReaders[0].projectId == 1 + assert createdReaders[0].protocolId == 654 + assert createdReaders[0].outputName == "outputCtftomo" + + def test_GetCtftomoSeriesViewsServiceBuildsFrames(service, tmp_path): associatedTs = FakeAssociatedTiltSeries( items={ @@ -541,6 +683,116 @@ def test_CreateNewSetOfCtftomoSeriesServiceReturnsEmptyWhenEverythingExcluded(se } +def test_CreateNewSetOfCtftomoSeriesServiceStoresSetWithResolvedProtocolDbId( + service, + monkeypatch, +): + storedCalls = [] + + class FakeDb: + # fakeDb + def fetchOne(self, *args, **kwargs): + return None + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakeScipionSetPostgresqlMapper: + # fakeScipionSetPostgresqlMapper + def __init__(self, db): + self.db = db + + def storeSet(self, projectId, protocolDbId, outputName, scipionSet): + storedCalls.append( + { + "projectId": projectId, + "protocolDbId": protocolDbId, + "outputName": outputName, + "scipionSet": scipionSet, + } + ) + return { + "stored": True, + "protocolDbId": protocolDbId, + "outputName": outputName, + } + + scipionSetMapperModule = importlib.import_module( + "app.backend.mapper.scipion_set_mapper" + ) + monkeypatch.setattr( + scipionSetMapperModule, + "ScipionSetPostgresqlMapper", + FakeScipionSetPostgresqlMapper, + ) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 654, + ) + + associatedTs = FakeAssociatedTiltSeries() + ctf1 = FakeCtfMeasurement( + objId=100, + index=1, + defocusU=12000.0, + defocusV=11000.0, + defocusAngle=45.0, + resolution=3.2, + phaseShift=0.15, + acquisitionOrder=1, + psdFile="psd1.mrc", + enabled=True, + ) + inputSeries = FakeCtftomoSeries( + tsId="TS_001", + label="Series 1", + tiltSeries=associatedTs, + items=[ctf1], + ) + inputSet = FakeCtftomoOutputSet( + seriesList=[inputSeries], + associatedTiltSeriesSet=associatedTs, + ) + protocol = FakeProtocol("outputCtftomo", inputSet, "/tmp/fake-protocol") + service.currentProject = FakeCurrentProject(protocol) + + mapper = FakeMapper() + + result = service.createNewSetOfCtftomoSeriesService( + projectId=1, + protocolId=10, + outputName="outputCtftomo", + exclusions={ + "TS_001": { + "excluded": False, + "tiltimages": [], + } + }, + restack=False, + mapper=mapper, + ) + + assert result["status"] == 0 + assert result["outputName"] == "CTFTomoSeries_0" + assert result["postgresqlSync"] == { + "stored": True, + "protocolDbId": 654, + "outputName": "CTFTomoSeries_0", + } + assert result["postgresqlError"] is None + assert storedCalls == [ + { + "projectId": 1, + "protocolDbId": 654, + "outputName": "CTFTomoSeries_0", + "scipionSet": protocol._definedOutputs["CTFTomoSeries_0"], + } + ] + + def test_CreateNewSetOfCtftomoSeriesServiceCreatesFilteredSeries(service, tmp_path): associatedTs = FakeAssociatedTiltSeries() ctf1 = FakeCtfMeasurement( @@ -590,12 +842,12 @@ def test_CreateNewSetOfCtftomoSeriesServiceCreatesFilteredSeries(service, tmp_pa restack=False, ) - assert result == { - "status": 0, - "outputName": "CTFTomoSeries_0", - "createdSeries": 1, - "restack": False, - } + assert result["status"] == 0 + assert result["outputName"] == "CTFTomoSeries_0" + assert result["createdSeries"] == 1 + assert result["restack"] is False + assert result["postgresqlSync"] is None + assert result["postgresqlError"] is None assert "CTFTomoSeries_0" in protocol._definedOutputs createdSet = protocol._definedOutputs["CTFTomoSeries_0"] assert createdSet.isEmpty() is False diff --git a/tests/unit/backend/api/services/test_project_service_fsc.py b/tests/unit/backend/api/services/test_project_service_fsc.py new file mode 100644 index 00000000..da035d13 --- /dev/null +++ b/tests/unit/backend/api/services/test_project_service_fsc.py @@ -0,0 +1,131 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +import importlib + +import pytest +from fastapi import HTTPException + + +class FakeFsc: + def __init__(self, label="FSC 1"): + self.label = label + + def clone(self): + return self + + def getObjLabel(self): + return self.label + + def getData(self): + return [ + [0.01, 0.02, 0.03], + [0.9, 0.5, 0.1], + ] + + def calculateResolution(self, threshold): + return 3.5 + + +class FakeFscOutput: + def __init__(self, items=None): + self.items = items or [] + + def iterItems(self, iterate=False): + return list(self.items) + + +class FakeProtocol: + def __init__(self, outputName, output): + setattr(self, outputName, output) + + +class FakeCurrentProject: + def __init__(self, protocol): + self.protocol = protocol + self.lastGetProtocolId = None + + def getProtocol(self, protocolId): + self.lastGetProtocolId = protocolId + return self.protocol + + +@pytest.fixture +def projectServiceModule(authTestEnv): + return importlib.import_module("app.backend.api.services.project_service") + + +@pytest.fixture +def service(projectServiceModule): + instance = object.__new__(projectServiceModule.ProjectService) + instance.currentProject = None + return instance + + +def test_GetFscRowsServiceRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy FSC fallback should not be used") + + monkeypatch.setattr(service, "_getScipionProtocolForRuntime", failRuntimeFallback) + + with pytest.raises(HTTPException) as exc: + service.getFscRowsService( + projectId=1, + protocolId=10, + outputName="outputFsc", + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "FSC output is not available in PostgreSQL metadata" in exc.value.detail + assert "reader_not_available" in exc.value.detail + + +def test_GetFscRowsServiceBuildsRowsWithoutMapper(service): + fscOutput = FakeFscOutput(items=[FakeFsc(label="Half maps")]) + protocol = FakeProtocol("outputFsc", fscOutput) + service.currentProject = FakeCurrentProject(protocol) + + result = service.getFscRowsService( + projectId=1, + protocolId=10, + outputName="outputFsc", + ) + + assert result == { + "threshold": 0.143, + "rows": [ + { + "label": "Half maps", + "resolution": 3.5, + "x": [0.01, 0.02, 0.03], + "y": [0.9, 0.5, 0.1], + } + ], + } \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_io_thumbnails.py b/tests/unit/backend/api/services/test_project_service_io_thumbnails.py index 88d56cad..c43a14a6 100644 --- a/tests/unit/backend/api/services/test_project_service_io_thumbnails.py +++ b/tests/unit/backend/api/services/test_project_service_io_thumbnails.py @@ -48,6 +48,33 @@ def __init__(self, protocolId, outputName=None, output=None): if outputName is not None: setattr(self, outputName, output) +class FakeDb: + def __init__(self, runtimeProtocolIdByDbId=None): + self.runtimeProtocolIdByDbId = runtimeProtocolIdByDbId or {} + self.fetchCalls = [] + + def fetchOne(self, query, params): + self.fetchCalls.append({ + "query": query, + "params": params, + }) + + if len(params) < 3: + return None + + protocolDbId = params[1] + runtimeProtocolId = self.runtimeProtocolIdByDbId.get(int(protocolDbId)) + if runtimeProtocolId is None: + return None + + return { + "protocolId": runtimeProtocolId, + } + + +class FakeMapper: + def __init__(self, runtimeProtocolIdByDbId=None): + self.db = FakeDb(runtimeProtocolIdByDbId=runtimeProtocolIdByDbId) class FakeCurrentProject: # fakeCurrentProject @@ -212,6 +239,49 @@ def test_OutputPreviewDelegatesToOutputsPreview(projectServiceModule, service, m } +def test_OutputPreviewResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, + tmp_path, +): + FakeOutputsPreview.instances = [] + + outputFile = tmp_path / "output.sqlite" + outputFile.write_text("placeholder", encoding="utf-8") + + output = FakeOutput(str(outputFile)) + protocol = FakeProtocol(protocolId=10, outputName="outputMetadata", output=output) + service.currentProject = FakeCurrentProject(protocols={10: protocol}) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + monkeypatch.setattr(projectServiceModule, "OutputsPreview", FakeOutputsPreview) + monkeypatch.setattr(service, "_createObjectManager", lambda: {"manager": "fresh"}) + + result = service.outputPreview( + protocolId=500, + outputName="outputMetadata", + requestHeaders={"x-preview-colormap": "viridis"}, + colormap="plasma", + mapper=mapper, + projectId=1, + ) + + assert result == { + "preview": True, + "protocolId": 10, + "outputPath": str(outputFile), + "colormap": "plasma", + } + assert FakeOutputsPreview.instances[0].lastPreviewCall == { + "protocolId": 10, + "outputPath": str(outputFile), + "objMgr": {"manager": "fresh"}, + } + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + def test_BuildProtocolThumbnailDelegatesToThumbnailService(projectServiceModule, service, monkeypatch): FakeThumbnailService.instances = [] monkeypatch.setattr(projectServiceModule, "ThumbnailService", FakeThumbnailService) @@ -230,6 +300,42 @@ def test_BuildProtocolThumbnailDelegatesToThumbnailService(projectServiceModule, ] +def test_BuildProtocolThumbnailResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, +): + FakeThumbnailService.instances = [] + monkeypatch.setattr(projectServiceModule, "ThumbnailService", FakeThumbnailService) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.buildProtocolThumbnail( + protocolId=500, + force=True, + size=400, + outputName="outputA", + mapper=mapper, + projectId=1, + ) + + assert result == { + "kind": "protocol", + "protocolId": 10, + "outputName": "outputA", + } + assert FakeThumbnailService.instances[0].calls == [ + { + "method": "buildProtocolThumbnail", + "protocolId": 10, + "force": True, + "size": 400, + "outputName": "outputA", + } + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + def test_BuildProjectThumbnailDelegatesToThumbnailService(projectServiceModule, service, monkeypatch): FakeThumbnailService.instances = [] monkeypatch.setattr(projectServiceModule, "ThumbnailService", FakeThumbnailService) @@ -265,6 +371,42 @@ def test_BuildProtocolOutputThumbnailDelegatesToThumbnailService(projectServiceM ] +def test_BuildProtocolOutputThumbnailResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, +): + FakeThumbnailService.instances = [] + monkeypatch.setattr(projectServiceModule, "ThumbnailService", FakeThumbnailService) + + mapper = FakeMapper(runtimeProtocolIdByDbId={501: 11}) + + result = service.buildProtocolOutputThumbnail( + protocolId=501, + outputName="outputVol", + force=False, + size=256, + mapper=mapper, + projectId=1, + ) + + assert result == { + "kind": "output", + "protocolId": 11, + "outputName": "outputVol", + } + assert FakeThumbnailService.instances[0].calls == [ + { + "method": "buildProtocolOutputThumbnail", + "protocolId": 11, + "outputName": "outputVol", + "force": False, + "size": 256, + } + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 501, "501") + + def test_ListProjectThumbnailItemsDelegatesToThumbnailService(projectServiceModule, service, monkeypatch): FakeThumbnailService.instances = [] monkeypatch.setattr(projectServiceModule, "ThumbnailService", FakeThumbnailService) @@ -291,6 +433,82 @@ def test_ListProjectThumbnailItemsDelegatesToThumbnailService(projectServiceModu ] +def test_ListProjectThumbnailItemsResolvesPostgresqlProtocolIdsWhenFilteringOutputs( + projectServiceModule, + service, + monkeypatch, +): + class FakeThumbnailServiceWithItems: + # fakeThumbnailServiceWithItems + instances = [] + + def __init__(self, currentProject): + self.currentProject = currentProject + FakeThumbnailServiceWithItems.instances.append(self) + + def listProtocolThumbnailItems( + self, + projectId, + force=False, + size=320, + maxProtocols=12, + maxOutputsPerProtocol=4, + inlineImages=False, + ): + return [ + { + "projectId": projectId, + "protocolId": 500, + "outputs": [ + {"outputName": "outputVol"}, + {"outputName": "missingOutput"}, + ], + }, + { + "projectId": projectId, + "protocolId": 999, + "outputs": [ + {"outputName": "outputVol"}, + ], + }, + ] + + output = FakeOutput("volume.mrc") + protocol = FakeProtocol(protocolId=10, outputName="outputVol", output=output) + service.currentProject = FakeCurrentProject(protocols={10: protocol}) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + monkeypatch.setattr( + projectServiceModule, + "ThumbnailService", + FakeThumbnailServiceWithItems, + ) + + result = service.listProjectThumbnailItems( + projectId=1, + force=False, + size=320, + maxProtocols=12, + maxOutputsPerProtocol=4, + inlineImages=False, + mapper=mapper, + ) + + assert result == [ + { + "projectId": 1, + "protocolId": 500, + "outputs": [ + {"outputName": "outputVol"}, + ], + } + ] + + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + assert mapper.db.fetchCalls[1]["params"] == (1, 999, "999") + + def test_NormalizeExportJsonContentAcceptsJsonString(service): content = service._normalizeExportJsonContent('[{"id": 10}]') assert json.loads(content) == [{"id": 10}] @@ -341,7 +559,11 @@ def test_ExportProtocolsServiceWritesJsonFile(service, monkeypatch, tmp_path): exportPayload=[{"protocolId": 10}, {"protocolId": 11}], ) - monkeypatch.setattr(service, "_resolveFsRootForWrite", lambda protocolId: rootPath) + monkeypatch.setattr( + service, + "_resolveFsRootForWrite", + lambda protocolId, mapper=None, projectId=None: rootPath, + ) payload = FakePayload( protocolIds=[10, 11], @@ -402,7 +624,11 @@ def test_WriteRemoteFileServiceWritesContent(service, monkeypatch, tmp_path): rootPath = tmp_path / "browser-root" rootPath.mkdir(parents=True, exist_ok=True) - monkeypatch.setattr(service, "_resolveFsRootForWrite", lambda protocolId: rootPath) + monkeypatch.setattr( + service, + "_resolveFsRootForWrite", + lambda protocolId, mapper=None, projectId=None: rootPath, + ) payload = FakePayload( path="exports/result.json", @@ -421,3 +647,198 @@ def test_WriteRemoteFileServiceWritesContent(service, monkeypatch, tmp_path): "size": targetPath.stat().st_size, "mimeType": "application/json", } + + +def test_GetProtocolOutputThumbnailsBatchResolvesPostgresqlProtocolId( + service, + monkeypatch, + tmp_path, +): + projectRouterModule = importlib.import_module("app.backend.api.routers.project_router") + + outputFile = tmp_path / "output.sqlite" + outputFile.write_text("placeholder", encoding="utf-8") + + thumbnailFile = tmp_path / "thumbnail.png" + thumbnailFile.write_bytes(b"fake-thumbnail") + + output = FakeOutput(str(outputFile)) + protocol = FakeProtocol(protocolId=10, outputName="outputVol", output=output) + service.currentProject = FakeCurrentProject(protocols={10: protocol}) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + monkeypatch.setattr( + service, + "getProjectDbRow", + lambda mapper, projectId, currentUser: {"id": projectId, "name": str(tmp_path)}, + ) + monkeypatch.setattr( + service, + "loadProjectForThumbnails", + lambda dbProj: service.currentProject, + ) + + buildCalls = [] + + def fakeBuildProtocolOutputThumbnail( + protocolId, + outputName, + force=False, + size=320, + mapper=None, + projectId=None, + ): + buildCalls.append({ + "protocolId": protocolId, + "outputName": outputName, + "force": force, + "size": size, + "mapper": mapper, + "projectId": projectId, + }) + return { + "absolutePath": str(thumbnailFile), + "exists": True, + "cached": False, + } + + monkeypatch.setattr( + service, + "buildProtocolOutputThumbnail", + fakeBuildProtocolOutputThumbnail, + ) + + payload = FakePayload( + outputs=[ + FakePayload(protocolId=500, outputName="outputVol"), + ], + size=256, + inlineImages=False, + ) + + response = projectRouterModule.getProtocolOutputThumbnailsBatch( + projectId=1, + payload=payload, + currentUser={"id": 1}, + mapper=mapper, + service=service, + ) + + payloadJson = json.loads(response.body.decode("utf-8")) + + assert payloadJson == { + "projectId": 1, + "size": 256, + "items": [ + { + "protocolId": 500, + "outputName": "outputVol", + "outputClassName": "FakeOutput", + "exists": True, + "cached": False, + "thumbnailUrl": "/projects/1/protocols/500/outputs/outputVol/thumbnail", + "thumbnailDataUrl": None, + "error": None, + } + ], + } + + assert buildCalls == [ + { + "protocolId": 500, + "outputName": "outputVol", + "force": False, + "size": 256, + "mapper": mapper, + "projectId": 1, + } + ] + + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_ExportProtocolsServiceResolvesPostgresqlProtocolIdsAndWritesJsonFile( + service, + monkeypatch, + tmp_path, +): + rootPath = tmp_path / "browser-root" + rootPath.mkdir(parents=True, exist_ok=True) + + protocol10 = FakeProtocol(protocolId=10) + protocol11 = FakeProtocol(protocolId=11) + + service.currentProject = FakeCurrentProject( + protocols={ + 10: protocol10, + 11: protocol11, + }, + exportPayload=[ + {"protocolId": 10}, + {"protocolId": 11}, + ], + ) + + exportedProtocolLists = [] + + def fakeGetProtocolsJson(protocolList): + exportedProtocolLists.append(protocolList) + return [ + {"protocolId": 10}, + {"protocolId": 11}, + ] + + service.currentProject.getProtocolsJson = fakeGetProtocolsJson + + mapper = FakeMapper(runtimeProtocolIdByDbId={ + 500: 10, + 501: 11, + }) + + monkeypatch.setattr( + service, + "_resolveFsRootForWrite", + lambda protocolId, mapper=None, projectId=None: rootPath, + ) + + payload = FakePayload( + protocolIds=["500", "501"], + directoryPath="exports", + filename="workflow-export", + ) + + result = service.exportProtocolsService( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=payload, + ) + + exportedPath = rootPath / "exports" / "workflow-export.json" + + assert exportedProtocolLists == [[protocol10, protocol11]] + assert exportedPath.exists() is True + + exportedText = exportedPath.read_text(encoding="utf-8") + assert exportedText.startswith("ScipionWeb metadata format: scipionweb.workflow.metadata") + assert "ScipionWeb metadata version: 1" in exportedText + assert "Scipion required plugins:" in exportedText + + exportedJsonText = service._extractWorkflowJsonText(exportedText) + assert json.loads(exportedJsonText) == [ + {"protocolId": 10}, + {"protocolId": 11}, + ] + + assert result == { + "success": True, + "path": str(exportedPath.resolve()), + "filename": "workflow-export.json", + "size": exportedPath.stat().st_size, + "mimeType": "application/json", + "protocolIds": ["500", "501"], + } + + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + assert mapper.db.fetchCalls[1]["params"] == (1, 501, "501") \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_logs_fs.py b/tests/unit/backend/api/services/test_project_service_logs_fs.py index 95f1d19f..ebbf1602 100644 --- a/tests/unit/backend/api/services/test_project_service_logs_fs.py +++ b/tests/unit/backend/api/services/test_project_service_logs_fs.py @@ -51,6 +51,34 @@ def getStderrLog(self): def getScheduleLog(self): return self._scheduleLog +class FakeDb: + def __init__(self, runtimeProtocolIdByDbId=None): + self.runtimeProtocolIdByDbId = runtimeProtocolIdByDbId or {} + self.fetchCalls = [] + + def fetchOne(self, query, params): + self.fetchCalls.append({ + "query": query, + "params": params, + }) + + if len(params) < 3: + return None + + protocolDbId = params[1] + runtimeProtocolId = self.runtimeProtocolIdByDbId.get(int(protocolDbId)) + if runtimeProtocolId is None: + return None + + return { + "protocolId": runtimeProtocolId, + } + + +class FakeMapper: + def __init__(self, runtimeProtocolIdByDbId=None): + self.db = FakeDb(runtimeProtocolIdByDbId=runtimeProtocolIdByDbId) + class FakeCurrentProject: # fakeCurrentProject @@ -116,6 +144,35 @@ def previewProtocolImageFile(self, protocolId, path, inline): "inline": inline, } +class FakePgDb: + def __init__(self, projectPath, runtimeProtocolIdByDbId=None): + self.projectPath = projectPath + self.runtimeProtocolIdByDbId = runtimeProtocolIdByDbId or {} + self.fetchCalls = [] + + def fetchOne(self, query, params): + self.fetchCalls.append({"query": query, "params": params}) + + if "FROM projects" in query: + return {"name": str(self.projectPath)} + + if "FROM protocols" in query: + protocolDbId = params[1] + runtimeProtocolId = self.runtimeProtocolIdByDbId.get(int(protocolDbId)) + if runtimeProtocolId is None: + return None + return {"protocolId": runtimeProtocolId} + + return None + + +class FakePgMapper: + def __init__(self, projectPath, runtimeProtocolIdByDbId=None): + self.db = FakePgDb( + projectPath=projectPath, + runtimeProtocolIdByDbId=runtimeProtocolIdByDbId, + ) + @pytest.fixture def projectServiceModule(authTestEnv): @@ -255,6 +312,121 @@ def test_GetProtocolLogsReadsAllChannelsFromOffsets(service, tmp_path): } +def test_ListProtocolLogChannelsServiceResolvesPostgresqlProtocolId(service, tmp_path): + stdoutLog = tmp_path / "stdout.log" + stderrLog = tmp_path / "stderr.log" + scheduleLog = tmp_path / "schedule.log" + + stdoutLog.write_text("hello\n", encoding="utf-8") + stderrLog.write_text("error\n", encoding="utf-8") + scheduleLog.write_text("schedule\n", encoding="utf-8") + + service.currentProject.protocols[10] = FakeProtocol( + stdoutLog=str(stdoutLog), + stderrLog=str(stderrLog), + scheduleLog=str(scheduleLog), + ) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.listProtocolLogChannelsService( + projectId=1, + protocolId=500, + mapper=mapper, + ) + + assert result["projectId"] == 1 + assert result["protocolId"] == 10 + assert result["channels"] == [ + {"id": "stdout", "label": "Output", "order": 1}, + {"id": "stderr", "label": "Errors", "order": 2}, + {"id": "schedule", "label": "Schedule", "order": 3}, + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_PollProtocolLogsServiceResolvesPostgresqlProtocolId(service, tmp_path): + stdoutLog = tmp_path / "stdout.log" + scheduleLog = tmp_path / "schedule.log" + + stdoutLog.write_text("line1\nline2\nline3\n", encoding="utf-8") + scheduleLog.write_text("sched1\nsched2\n", encoding="utf-8") + + service.currentProject.protocols[10] = FakeProtocol( + stdoutLog=str(stdoutLog), + stderrLog=str(tmp_path / "missing-stderr.log"), + scheduleLog=str(scheduleLog), + ) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.pollProtocolLogsService( + projectId=1, + protocolId=500, + offsets={ + "stdoutLog": 6, + "err": 0, + "schedule": 7, + }, + maxBytes=64, + maxLines=1, + mapper=mapper, + ) + + assert result["projectId"] == 1 + assert result["protocolId"] == 10 + assert result["channels"]["stdout"] == { + "content": "line2\n", + "offset": 12, + } + assert result["channels"]["stderr"] == { + "content": "", + "offset": 0, + } + assert result["channels"]["schedule"] == { + "content": "sched2\n", + "offset": 14, + } + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_GetProtocolLogsResolvesPostgresqlProtocolId(service, tmp_path): + stdoutLog = tmp_path / "stdout.log" + stderrLog = tmp_path / "stderr.log" + scheduleLog = tmp_path / "schedule.log" + + stdoutLog.write_text("abc\ndef\n", encoding="utf-8") + stderrLog.write_text("ERR1\nERR2\n", encoding="utf-8") + scheduleLog.write_text("SCH1\nSCH2\n", encoding="utf-8") + + service.currentProject.protocols[10] = FakeProtocol( + stdoutLog=str(stdoutLog), + stderrLog=str(stderrLog), + scheduleLog=str(scheduleLog), + ) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.getProtocolLogs( + projectId=1, + protocolId=500, + offset=4, + errOffset=5, + scheduleOffset=5, + mapper=mapper, + ) + + assert result == { + "stdoutLog": "def\n", + "stderrLog": "ERR2\n", + "stdoutOffset": 8, + "stderrOffset": 10, + "scheduleLog": "SCH2\n", + "scheduleOffset": 10, + } + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + def test_GetProtocolPathReturnsGlobalBrowserPayload(service, monkeypatch, tmp_path): browserRoot = tmp_path / "browser-root" browserRoot.mkdir(parents=True, exist_ok=True) @@ -427,4 +599,333 @@ def test_PreviewProtocolImageFileDelegatesToProtocolBrowser(projectServiceModule } assert FakeFileHandlers.lastInstance.calls == [ ("previewProtocolImageFile", "10", "preview.webp", False), - ] \ No newline at end of file + ] + + +def test_GetProtocolPathResolvesPostgresqlProtocolId(service, tmp_path): + protocolPath = tmp_path / "DemoProject" / "Runs" / "000010_ProtImport" + protocolPath.mkdir(parents=True, exist_ok=True) + + service.currentProject.protocols[10] = FakeProtocol(protocolPath=str(protocolPath)) + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.getProtocolPath( + protocolId=500, + mapper=mapper, + projectId=1, + ) + + assert result == { + "rootAbs": str((tmp_path / "DemoProject").resolve()), + "startPath": "Runs/000010_ProtImport", + "protocolRoot": "Runs/000010_ProtImport", + } + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_ListProtocolDirResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "FileHandlers", FakeFileHandlers) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.listProtocolDir( + protocolId="500", + path="extra", + mapper=mapper, + projectId=1, + ) + + assert result == { + "mode": "protocol", + "protocolId": "10", + "path": "extra", + } + assert FakeFileHandlers.lastInstance.calls == [ + ("listProtocolDir", "10", "extra"), + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_PreviewProtocolTextFileResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "FileHandlers", FakeFileHandlers) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.previewProtocolTextFile( + protocolId="500", + path="notes.txt", + mapper=mapper, + projectId=1, + ) + + assert result == { + "mode": "protocol-text", + "protocolId": "10", + "path": "notes.txt", + } + assert FakeFileHandlers.lastInstance.calls == [ + ("previewProtocolTextFile", "10", "notes.txt"), + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_PreviewRemoteEntryResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "FileHandlers", FakeFileHandlers) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.previewRemoteEntry( + protocolId="500", + path="image.png", + mapper=mapper, + projectId=1, + ) + + assert result == { + "mode": "protocol-preview", + "protocolId": "10", + "path": "image.png", + } + assert FakeFileHandlers.lastInstance.calls == [ + ("previewProtocolRemoteEntry", "10", "image.png"), + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_PreviewProtocolImageFileResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "FileHandlers", FakeFileHandlers) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.previewProtocolImageFile( + protocolId="500", + path="preview.webp", + inline=False, + mapper=mapper, + projectId=1, + ) + + assert result == { + "mode": "protocol-image", + "protocolId": "10", + "path": "preview.webp", + "inline": False, + } + assert FakeFileHandlers.lastInstance.calls == [ + ("previewProtocolImageFile", "10", "preview.webp", False), + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_WriteRemoteFileServiceResolvesPostgresqlProtocolId(service, tmp_path): + protocolPath = tmp_path / "DemoProject" / "Runs" / "000010_ProtImport" + protocolPath.mkdir(parents=True, exist_ok=True) + + service.currentProject.protocols[10] = FakeProtocol(protocolPath=str(protocolPath)) + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + class FakePayload: + path = "exports/result.json" + content = '{"ok": true}' + mimeType = "application/json" + + result = service.writeRemoteFileService( + protocolId=500, + payload=FakePayload(), + mapper=mapper, + projectId=1, + ) + + targetPath = tmp_path / "DemoProject" / "exports" / "result.json" + + assert targetPath.exists() is True + assert targetPath.read_text(encoding="utf-8") == '{"ok": true}' + assert result == { + "success": True, + "path": str(targetPath.resolve()), + "size": targetPath.stat().st_size, + "mimeType": "application/json", + } + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_GetProtocolLogsNormalizesNegativeOffsets(service, tmp_path): + stdoutLog = tmp_path / "stdout.log" + stderrLog = tmp_path / "stderr.log" + scheduleLog = tmp_path / "schedule.log" + + stdoutLog.write_text("abc\n", encoding="utf-8") + stderrLog.write_text("ERR\n", encoding="utf-8") + scheduleLog.write_text("SCH\n", encoding="utf-8") + + service.currentProject.protocols[10] = FakeProtocol( + stdoutLog=str(stdoutLog), + stderrLog=str(stderrLog), + scheduleLog=str(scheduleLog), + ) + + result = service.getProtocolLogs( + projectId=1, + protocolId=10, + offset=-10, + errOffset=-5, + scheduleOffset=-1, + ) + + assert result == { + "stdoutLog": "abc\n", + "stderrLog": "ERR\n", + "stdoutOffset": 4, + "stderrOffset": 4, + "scheduleLog": "SCH\n", + "scheduleOffset": 4, + } + + +def test_ListProtocolLogChannelsServiceUsesPostgresqlPathsBeforeRuntime( + service, + tmp_path, +): + projectPath = tmp_path / "DemoProject" + protocolPath = projectPath / "Runs" / "000010_ProtImport" + logsPath = protocolPath / "logs" + logsPath.mkdir(parents=True, exist_ok=True) + + (logsPath / "run.stdout").write_text("hello\n", encoding="utf-8") + (logsPath / "run.stderr").write_text("error\n", encoding="utf-8") + (logsPath / "schedule.log").write_text("schedule\n", encoding="utf-8") + + mapper = FakePgMapper( + projectPath=projectPath, + runtimeProtocolIdByDbId={500: 10}, + ) + + def failRuntime(*args, **kwargs): + raise AssertionError("runtime should not be used") + + service._getScipionProtocolByRuntimeId = failRuntime + + result = service.listProtocolLogChannelsService( + projectId=1, + protocolId=500, + mapper=mapper, + ) + + assert result == { + "projectId": 1, + "protocolId": 10, + "channels": [ + {"id": "stdout", "label": "Output", "order": 1}, + {"id": "stderr", "label": "Errors", "order": 2}, + {"id": "schedule", "label": "Schedule", "order": 3}, + ], + } + + +def test_PollProtocolLogsServiceUsesPostgresqlPathsBeforeRuntime( + service, + tmp_path, +): + projectPath = tmp_path / "DemoProject" + protocolPath = projectPath / "Runs" / "000010_ProtImport" + logsPath = protocolPath / "logs" + logsPath.mkdir(parents=True, exist_ok=True) + + (logsPath / "run.stdout").write_text("line1\nline2\nline3\n", encoding="utf-8") + (logsPath / "run.stderr").write_text("err1\nerr2\n", encoding="utf-8") + (logsPath / "schedule.log").write_text("sched1\nsched2\n", encoding="utf-8") + + mapper = FakePgMapper( + projectPath=projectPath, + runtimeProtocolIdByDbId={500: 10}, + ) + + def failRuntime(*args, **kwargs): + raise AssertionError("runtime should not be used") + + service._getScipionProtocolByRuntimeId = failRuntime + + result = service.pollProtocolLogsService( + projectId=1, + protocolId=500, + offsets={ + "stdoutLog": 6, + "err": 0, + "schedule": 7, + }, + maxBytes=64, + maxLines=1, + mapper=mapper, + ) + + assert result == { + "projectId": 1, + "protocolId": 10, + "channels": { + "stdout": { + "content": "line2\n", + "offset": 12, + }, + "stderr": { + "content": "err1\n", + "offset": 5, + }, + "schedule": { + "content": "sched2\n", + "offset": 14, + }, + }, + } + + +def test_ProtocolLogsFallbackToRuntimeWhenPostgresqlLogsAreMissing( + service, + tmp_path, +): + projectPath = tmp_path / "DemoProject" + protocolPath = projectPath / "Runs" / "000010_ProtImport" + protocolPath.mkdir(parents=True, exist_ok=True) + + runtimeStdout = tmp_path / "runtime.stdout" + runtimeStdout.write_text("runtime\n", encoding="utf-8") + + service.currentProject.protocols[10] = FakeProtocol( + stdoutLog=str(runtimeStdout), + stderrLog=str(tmp_path / "missing.stderr"), + scheduleLog=str(tmp_path / "missing.schedule"), + ) + + mapper = FakePgMapper( + projectPath=projectPath, + runtimeProtocolIdByDbId={500: 10}, + ) + + result = service.pollProtocolLogsService( + projectId=1, + protocolId=500, + offsets={"stdout": 0}, + maxBytes=64, + maxLines=10, + mapper=mapper, + ) + + assert result["channels"]["stdout"] == { + "content": "runtime\n", + "offset": len("runtime\n"), + } \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_metadata.py b/tests/unit/backend/api/services/test_project_service_metadata.py index 2b2b2d10..b6f35f29 100644 --- a/tests/unit/backend/api/services/test_project_service_metadata.py +++ b/tests/unit/backend/api/services/test_project_service_metadata.py @@ -25,7 +25,6 @@ # ****************************************************************************** import importlib -from pathlib import Path import numpy as np import pytest @@ -192,8 +191,9 @@ def setSortingAsc(self, value): class FakeDao: # fakeDao - def __init__(self): - self._objectsType = {"create subset": "SetOfParticles"} + def __init__(self, objectsType=None, actionAliases=None): + self._objectsType = objectsType or {"create subset": "SetOfParticles"} + self._actionAliases = actionAliases or {} self.fillTableCalls = [] def fillTable(self, table, objMgr): @@ -204,14 +204,18 @@ def fillTable(self, table, objMgr): } ) + def _getActionAliasForTableName(self, tableName): + return self._actionAliases.get(tableName, "") + class FakeObjectManager: # fakeObjectManager - def __init__(self, tables, rowsByTable, rowCounts=None, dao=None): + def __init__(self, tables, rowsByTable, rowCounts=None, dao=None, fileName=""): self._tables = tables self._rowsByTable = rowsByTable self._rowCounts = rowCounts or {} self._dao = dao or FakeDao() + self._fileName = fileName def getTables(self): return self._tables @@ -260,13 +264,19 @@ def newProtocol(self, *args, **kwargs): class FakeCurrentProject: # fakeCurrentProject - def __init__(self, protocol): + def __init__(self, protocol, projectPath=None): self._protocol = protocol + self._projectPath = projectPath self.launchedProtocols = [] def getProtocol(self, protocolId): return self._protocol + def getPath(self): + if self._projectPath is None: + raise AttributeError("Project path is not set") + return str(self._projectPath) + def newProtocol(self, *args, **kwargs): return self._protocol.newProtocol(*args, **kwargs) @@ -290,11 +300,50 @@ def service(projectServiceModule, tmp_path): protocol = FakeProtocol("outputParticles", output) instance = object.__new__(projectServiceModule.ProjectService) - instance.currentProject = FakeCurrentProject(protocol) + instance.currentProject = FakeCurrentProject(protocol, projectPath=tmp_path) instance.tomoList = {} return instance +def patchOpenMetadataTable(service, monkeypatch, objMgr, table, calls=None): + # patchOpenMetadataTable + def openMetadataTable(projectId, protocolId, outputName, tableName, mapper=None): + if calls is not None: + calls.append( + { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tableName": tableName, + "mapper": mapper, + } + ) + return objMgr, table + + monkeypatch.setattr(service, "_openMetadataTable", openMetadataTable) + + +def patchObjectManagerForOutput(service, monkeypatch, objMgr, calls=None): + # patchObjectManagerForOutput + def getMetadataObjectManagerForOutput(projectId, protocolId, outputName, mapper=None): + if calls is not None: + calls.append( + { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + ) + return objMgr + + monkeypatch.setattr( + service, + "_getMetadataObjectManagerForOutput", + getMetadataObjectManagerForOutput, + ) + + def test_ListOutputMetadataTablesServiceReturnsSummaries(service, monkeypatch): tables = { "objects": FakeTable( @@ -315,20 +364,26 @@ def test_ListOutputMetadataTablesServiceReturnsSummaries(service, monkeypatch): rowsByTable={"objects": [], "classes": []}, rowCounts={"objects": 11, "classes": 3}, ) + mapper = object() + calls = [] - monkeypatch.setattr( - service, - "_resolveOutputForMetadata", - lambda protocolId, outputName: (None, None, "/tmp/fake.sqlite"), - ) - monkeypatch.setattr(service, "_getMetadataObjectManager", lambda metaPath: objMgr) + patchObjectManagerForOutput(service, monkeypatch, objMgr, calls=calls) result = service.listOutputMetadataTablesService( projectId=1, protocolId=10, outputName="outputParticles", + mapper=mapper, ) + assert calls == [ + { + "projectId": 1, + "protocolId": 10, + "outputName": "outputParticles", + "mapper": mapper, + } + ] assert result == [ { "name": "objects", @@ -345,6 +400,101 @@ def test_ListOutputMetadataTablesServiceReturnsSummaries(service, monkeypatch): ] +def test_GetPostgresqlDAOIfAvailableUsesResolvedProtocolDbId( + service, + monkeypatch, +): + createdDaos = [] + + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakePostgresqlDAO: + # fakePostgresqlDAO + def __init__(self, db, projectId, protocolId, outputName): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + createdDaos.append(self) + + def hasOutput(self): + return True + + daoModule = importlib.import_module( + "app.backend.viewers.postgresql_dao" + ) + + monkeypatch.setattr( + daoModule, + "PostgresqlDAO", + FakePostgresqlDAO, + ) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 852, + ) + + mapper = FakeMapper() + + dao = service._getPostgresqlDAOIfAvailable( + projectId=1, + protocolId=10, + outputName="outputParticles", + mapper=mapper, + ) + + assert dao is createdDaos[0] + assert createdDaos[0].db is mapper.db + assert createdDaos[0].projectId == 1 + assert createdDaos[0].protocolId == 852 + assert createdDaos[0].outputName == "outputParticles" + + +def test_GetMetadataObjectManagerForOutputRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + monkeypatch.setattr( + service, + "_getPostgresqlDAOIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy metadata fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForMetadata", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service._getMetadataObjectManagerForOutput( + projectId=1, + protocolId=10, + outputName="outputParticles", + mapper=FakeMapper(), + ) + + assert exc.value.status_code == 404 + assert "Metadata output is not available in PostgreSQL metadata" in exc.value.detail + assert "dao_not_available" in exc.value.detail + + def test_GetMetadataTableSchemaServiceBuildsColumns(service, monkeypatch): columns = [ FakeColumn("id", "Id", IntRenderer(), sortable=True), @@ -358,16 +508,32 @@ def test_GetMetadataTableSchemaServiceBuildsColumns(service, monkeypatch): columns=columns, hasColumnId=True, ) + objMgr = FakeObjectManager( + tables={"objects": table}, + rowsByTable={"objects": []}, + ) + mapper = object() + calls = [] - monkeypatch.setattr(service, "_openMetadataTable", lambda protocolId, outputName, tableName: (object(), table)) + patchOpenMetadataTable(service, monkeypatch, objMgr, table, calls=calls) result = service.getMetadataTableSchemaService( projectId=1, protocolId=10, outputName="outputParticles", tableName="objects", + mapper=mapper, ) + assert calls == [ + { + "projectId": 1, + "protocolId": 10, + "outputName": "outputParticles", + "tableName": "objects", + "mapper": mapper, + } + ] assert result["name"] == "objects" assert result["alias"] == "Particles" assert result["hasColumnId"] is True @@ -416,6 +582,120 @@ def test_GetMetadataTableSchemaServiceBuildsColumns(service, monkeypatch): ] +def test_GetMetadataTableSchemaServiceReturnsActionsForChildTables(service, monkeypatch): + table = FakeTable( + name="Class001_Objects", + alias="Class001_Particle", + columns=[], + actions=[FakeAction("Particle"), FakeAction("Particle")], + ) + objMgr = FakeObjectManager( + tables={"Class001_Objects": table}, + rowsByTable={"Class001_Objects": []}, + ) + + patchOpenMetadataTable(service, monkeypatch, objMgr, table) + + result = service.getMetadataTableSchemaService( + projectId=1, + protocolId=10, + outputName="outputParticles", + tableName="Class001_Objects", + mapper=object(), + ) + + assert result["name"] == "Class001_Objects" + assert result["alias"] == "Class001_Particle" + assert result["actions"] == ["Particle"] + + +def test_GetMetadataTableSchemaServiceDoesNotReturnActionsForProperties(service, monkeypatch): + table = FakeTable( + name="Properties", + alias="Properties", + columns=[FakeColumn("key", "key", StrRenderer())], + actions=[FakeAction("Particle")], + ) + objMgr = FakeObjectManager( + tables={"Properties": table}, + rowsByTable={"Properties": []}, + ) + + patchOpenMetadataTable(service, monkeypatch, objMgr, table) + + result = service.getMetadataTableSchemaService( + projectId=1, + protocolId=10, + outputName="outputParticles", + tableName="Properties", + mapper=object(), + ) + + assert result["name"] == "Properties" + assert result["alias"] == "Properties" + assert result["actions"] == [] + + +def test_ResolveMetadataActionOutputClassNameReturnsSetOfVolumesForClass3D(service): + table = FakeTable( + name="objects", + alias="SetOfClasses3D", + columns=[], + actions=[FakeAction("Volumes")], + ) + dao = FakeDao( + objectsType={}, + actionAliases={"objects": "Class3D"}, + ) + + result = service._resolveMetadataActionOutputClassName( + dao=dao, + table=table, + action="Volumes", + ) + + assert result == "SetOfVolumes" + + +def test_ResolveMetadataActionOutputClassNameReturnsSetOfAveragesForClass2D(service): + table = FakeTable( + name="objects", + alias="SetOfClasses2D", + columns=[], + actions=[FakeAction("Averages")], + ) + dao = FakeDao( + objectsType={}, + actionAliases={"objects": "Class2D"}, + ) + + result = service._resolveMetadataActionOutputClassName( + dao=dao, + table=table, + action="Averages", + ) + + assert result == "SetOfAverages" + + +def test_ResolveMetadataActionOutputClassNameUsesDaoObjectsType(service): + table = FakeTable( + name="objects", + alias="Particles", + columns=[], + actions=[FakeAction("Particle")], + ) + dao = FakeDao(objectsType={"Particle": "SetOfParticles"}) + + result = service._resolveMetadataActionOutputClassName( + dao=dao, + table=table, + action="Particle", + ) + + assert result == "SetOfParticles" + + def test_GetMetadataTablePageServiceConvertsCells(service, monkeypatch): columns = [ FakeColumn("id", "Id", IntRenderer()), @@ -431,7 +711,7 @@ def test_GetMetadataTablePageServiceConvertsCells(service, monkeypatch): rowCounts={"objects": 1}, ) - monkeypatch.setattr(service, "_openMetadataTable", lambda protocolId, outputName, tableName: (objMgr, table)) + patchOpenMetadataTable(service, monkeypatch, objMgr, table) result = service.getMetadataTablePageService( projectId=1, @@ -443,6 +723,7 @@ def test_GetMetadataTablePageServiceConvertsCells(service, monkeypatch): sortBy="id", asc=True, selectionOnly=False, + mapper=object(), ) assert result == { @@ -480,7 +761,7 @@ def test_GetMetadataTableWindowServiceReturnsOffsetWindow(service, monkeypatch): rowCounts={"objects": 3}, ) - monkeypatch.setattr(service, "_openMetadataTable", lambda protocolId, outputName, tableName: (objMgr, table)) + patchOpenMetadataTable(service, monkeypatch, objMgr, table) result = service.getMetadataTableWindowService( projectId=1, @@ -492,6 +773,7 @@ def test_GetMetadataTableWindowServiceReturnsOffsetWindow(service, monkeypatch): selectionOnly=False, sortBy="label", asc=False, + mapper=object(), ) assert table.sortBy == "label" @@ -533,7 +815,7 @@ def test_ExportMetadataTableServiceReturnsCsv(service, monkeypatch): rowCounts={"objects": 2}, ) - monkeypatch.setattr(service, "_openMetadataTable", lambda protocolId, outputName, tableName: (objMgr, table)) + patchOpenMetadataTable(service, monkeypatch, objMgr, table) response = service.exportMetadataTableService( projectId=1, @@ -543,6 +825,7 @@ def test_ExportMetadataTableServiceReturnsCsv(service, monkeypatch): fmt="csv", selectionOnly=False, ids=None, + mapper=object(), ) text = response.body.decode("utf-8") @@ -560,14 +843,10 @@ def test_RenderMetadataImageCellServiceReturnsPlaceholderWhenRowMissing(service, tables={"objects": table}, rowsByTable={"objects": []}, rowCounts={"objects": 0}, + fileName="postgresql://project/1/protocol/10/output/outputParticles", ) - monkeypatch.setattr( - service, - "_resolveOutputForMetadata", - lambda protocolId, outputName: (None, None, "/tmp/fake.sqlite"), - ) - monkeypatch.setattr(service, "_openMetadataTable", lambda protocolId, outputName, tableName: (objMgr, table)) + patchOpenMetadataTable(service, monkeypatch, objMgr, table) response = service.renderMetadataImageCellService( projectId=1, @@ -581,6 +860,7 @@ def test_RenderMetadataImageCellServiceReturnsPlaceholderWhenRowMissing(service, applyTransform=False, inline=True, fmt="png", + mapper=object(), ) assert response.status_code == 200 @@ -588,6 +868,106 @@ def test_RenderMetadataImageCellServiceReturnsPlaceholderWhenRowMissing(service, assert response.media_type == "image/png" +def test_RenderMetadataImageCellServiceResolvesProtocolIdForRelativeImagePaths( + service, + monkeypatch, + tmp_path, +): + protocolPath = tmp_path / "Runs" / "000010_Prot" + protocolPath.mkdir(parents=True) + + imagePath = protocolPath / "thumb.png" + + try: + from PIL import Image + + Image.new("L", (4, 4), 128).save(imagePath) + except Exception: + imagePath.write_bytes(b"") + + class PathRenderer: + # pathRenderer + def render(self, rawValue, rowValues): + return rawValue + + class FakeProtocolWithPath: + # fakeProtocolWithPath + def getPath(self): + return str(protocolPath) + + class FakeProjectWithoutProtocolLookup: + # fakeProjectWithoutProtocolLookup + def getPath(self): + return str(tmp_path) + + def getProtocol(self, protocolId): + raise AssertionError("currentProject.getProtocol should not be used directly") + + columns = [FakeColumn("image", "Image", PathRenderer())] + table = FakeTable(name="objects", alias="Particles", columns=columns) + objMgr = FakeObjectManager( + tables={"objects": table}, + rowsByTable={"objects": [FakeRow(1, ["thumb.png"])]}, + rowCounts={"objects": 1}, + fileName="postgresql://project/1/protocol/500/output/outputParticles", + ) + + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + mapper = FakeMapper() + resolverCalls = [] + + def getScipionProtocolForRuntime(mapper, projectId, protocolId): + resolverCalls.append( + { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + } + ) + return FakeProtocolWithPath() + + service.currentProject = FakeProjectWithoutProtocolLookup() + patchOpenMetadataTable(service, monkeypatch, objMgr, table) + monkeypatch.setattr( + service, + "_getScipionProtocolForRuntime", + getScipionProtocolForRuntime, + ) + + response = service.renderMetadataImageCellService( + projectId=1, + protocolId=500, + outputName="outputParticles", + tableName="objects", + rowId=1, + rowIndex=None, + columnName="image", + size=64, + applyTransform=False, + inline=True, + fmt="png", + mapper=mapper, + ) + + assert response.media_type == "image/png" + assert response.headers.get("x-image-placeholder") is None + assert resolverCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + + def test_RunMetadataTableActionServiceLaunchesSubsetProtocol( service, projectServiceModule, @@ -599,7 +979,7 @@ def test_RunMetadataTableActionServiceLaunchesSubsetProtocol( output = FakeOutput(str(outputFile)) protocol = FakeProtocol("outputParticles", output) - service.currentProject = FakeCurrentProject(protocol) + service.currentProject = FakeCurrentProject(protocol, projectPath=tmp_path) table = FakeTable( name="objects", @@ -607,21 +987,55 @@ def test_RunMetadataTableActionServiceLaunchesSubsetProtocol( columns=[], actions=[FakeAction("create subset")], ) - dao = FakeDao() + dao = FakeDao(objectsType={"create subset": "SetOfParticles"}) objMgr = FakeObjectManager( tables={"objects": table}, rowsByTable={"objects": []}, dao=dao, ) - monkeypatch.setattr(service, "_getMetadataObjectManager", lambda metaPath: objMgr) + class FakeDb: + # fakeDb + def fetchOne(self, *args, **kwargs): + return None + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + mapper = FakeMapper() + calls = [] + syncCalls = [] + syncResult = { + "protocols": 2, + "dependencies": 1, + } + + def syncProjectProtocolsAndDependencies( + mapperArg, + projectIdArg, + refresh=True, + checkPid=True, + ): + syncCalls.append( + { + "mapper": mapperArg, + "projectId": projectIdArg, + "refresh": refresh, + "checkPid": checkPid, + } + ) + return syncResult + + patchOpenMetadataTable(service, monkeypatch, objMgr, table, calls=calls) monkeypatch.setattr(projectServiceModule, "OBJECT_TABLE", "objects") monkeypatch.setattr(projectServiceModule, "ProtUserSubSet", object()) - - oldCwd = Path.cwd() - monkeypatch.chdir(tmp_path) - logsDir = tmp_path / "Logs" - logsDir.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + syncProjectProtocolsAndDependencies, + ) result = service.runMetadataTableActionService( projectId=1, @@ -632,15 +1046,210 @@ def test_RunMetadataTableActionServiceLaunchesSubsetProtocol( subsetName="subset A", ids=[3, 5, 7], currentUser={"id": 1}, - mapper=object(), + mapper=mapper, ) - assert result is True + assert result == { + "success": True, + "message": "Subset protocol was launched successfully", + "postgresqlSync": syncResult, + "postgresqlError": None, + } + assert syncCalls == [ + { + "mapper": mapper, + "projectId": 1, + "refresh": True, + "checkPid": True, + } + ] + assert calls == [ + { + "projectId": 1, + "protocolId": 10, + "outputName": "outputParticles", + "tableName": "objects", + "mapper": mapper, + } + ] assert len(protocol.newProtocolCalls) == 1 call = protocol.newProtocolCalls[0] assert call["kwargs"]["inputObject"] is output assert call["kwargs"]["outputClassName"] == "SetOfParticles" assert call["kwargs"]["label"] == "subset A" + assert call["kwargs"]["sqliteFile"].startswith("Logs/selection_") + assert call["kwargs"]["sqliteFile"].endswith(".txt,") assert service.currentProject.launchedProtocols == [ {"batchProtocol": True, "kwargs": call["kwargs"]} - ] \ No newline at end of file + ] + + selectionFiles = list((tmp_path / "Logs").glob("selection_*.txt")) + assert len(selectionFiles) == 1 + assert selectionFiles[0].read_text(encoding="utf-8") == "3 5 7 " + + +def test_RunMetadataTableActionServiceUsesRuntimeResolverWithMapper( + service, + projectServiceModule, + monkeypatch, + tmp_path, +): + outputFile = tmp_path / "metadata.sqlite" + outputFile.write_text("placeholder", encoding="utf-8") + + output = FakeOutput(str(outputFile)) + protocol = FakeProtocol("outputParticles", output) + service.currentProject = FakeCurrentProject(protocol, projectPath=tmp_path) + + table = FakeTable( + name="objects", + alias="Particles", + columns=[], + actions=[FakeAction("create subset")], + ) + dao = FakeDao(objectsType={"create subset": "SetOfParticles"}) + objMgr = FakeObjectManager( + tables={"objects": table}, + rowsByTable={"objects": []}, + dao=dao, + ) + + class FakeDb: + # fakeDb + def fetchOne(self, *args, **kwargs): + return None + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + mapper = FakeMapper() + resolverCalls = [] + openTableCalls = [] + + def getScipionProtocolForRuntime(mapper, projectId, protocolId): + resolverCalls.append( + { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + } + ) + return protocol + + def syncProjectProtocolsAndDependencies( + mapperArg, + projectIdArg, + refresh=True, + checkPid=True, + ): + return { + "protocols": 2, + "dependencies": 1, + } + + patchOpenMetadataTable( + service, + monkeypatch, + objMgr, + table, + calls=openTableCalls, + ) + + monkeypatch.setattr(projectServiceModule, "OBJECT_TABLE", "objects") + monkeypatch.setattr(projectServiceModule, "ProtUserSubSet", object()) + monkeypatch.setattr( + service, + "_getScipionProtocolForRuntime", + getScipionProtocolForRuntime, + ) + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + syncProjectProtocolsAndDependencies, + ) + + result = service.runMetadataTableActionService( + projectId=1, + protocolId=500, + outputName="outputParticles", + tableName="objects", + action="create subset", + subsetName="subset A", + ids=[3, 5, 7], + currentUser={"id": 1}, + mapper=mapper, + ) + + assert result["success"] is True + assert resolverCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + assert openTableCalls == [ + { + "projectId": 1, + "protocolId": 500, + "outputName": "outputParticles", + "tableName": "objects", + "mapper": mapper, + } + ] + assert len(protocol.newProtocolCalls) == 1 + + +def test_RunMetadataTableActionServiceBuildsChildTableSelectionArgument( + service, + projectServiceModule, + monkeypatch, + tmp_path, +): + outputFile = tmp_path / "metadata.sqlite" + outputFile.write_text("placeholder", encoding="utf-8") + + output = FakeOutput(str(outputFile)) + protocol = FakeProtocol("outputParticles", output) + service.currentProject = FakeCurrentProject(protocol, projectPath=tmp_path) + + table = FakeTable( + name="Class001_Objects", + alias="Class001_Particle", + columns=[], + actions=[FakeAction("Particle")], + ) + dao = FakeDao( + objectsType={"Particle": "SetOfParticles"}, + actionAliases={"Class001_Objects": "Class001_Particle"}, + ) + objMgr = FakeObjectManager( + tables={"Class001_Objects": table}, + rowsByTable={"Class001_Objects": []}, + dao=dao, + ) + + patchOpenMetadataTable(service, monkeypatch, objMgr, table) + monkeypatch.setattr(projectServiceModule, "OBJECT_TABLE", "objects") + monkeypatch.setattr(projectServiceModule, "ProtUserSubSet", object()) + + result = service.runMetadataTableActionService( + projectId=1, + protocolId=10, + outputName="outputParticles", + tableName="Class001_Objects", + action="Particle", + subsetName="class particles", + ids=[1, 2], + currentUser={"id": 1}, + mapper=object(), + ) + + assert result["success"] is True + assert len(protocol.newProtocolCalls) == 1 + call = protocol.newProtocolCalls[0] + assert call["kwargs"]["outputClassName"] == "SetOfParticles" + assert call["kwargs"]["sqliteFile"].startswith("Logs/selection_") + assert call["kwargs"]["sqliteFile"].endswith(".txt,Class001") \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_projects.py b/tests/unit/backend/api/services/test_project_service_projects.py index d6bee593..d3032b12 100644 --- a/tests/unit/backend/api/services/test_project_service_projects.py +++ b/tests/unit/backend/api/services/test_project_service_projects.py @@ -93,6 +93,8 @@ class FakeMapper: def __init__(self): self.projectsById = {} self.projectsListResult = [] + self.projectProtocolCounts = {} + self.lastCountProjectProtocolsCall = None self.insertProjectResult = 101 self.updateProjectResult = { "id": 1, @@ -137,6 +139,10 @@ def listProjects(self, ownerId=None): self.lastListProjectsCall = {"ownerId": ownerId} return self.projectsListResult + def countProjectProtocols(self, projectId): + self.lastCountProjectProtocolsCall = {"projectId": projectId} + return self.projectProtocolCounts.get(projectId, 0) + def insertProject(self, ownerId, name, description, status): self.lastInsertProjectCall = { "ownerId": ownerId, @@ -488,7 +494,8 @@ def test_ListProjectsBuildsComputedFields(service, mapper, currentUser, monkeypa ] monkeypatch.setattr(service, "getProjectSize", lambda path: 3 * 1024 ** 3) - monkeypatch.setattr(service, "countProtocols", lambda path: 7) + mapper.projectProtocolCounts[1] = 7 + monkeypatch.setattr(service, "countProtocols", lambda path: 999) monkeypatch.setattr(service, "_buildProjectThumbnailVersion", lambda **kwargs: "thumb-v1") result = service.listProjects(mapper, currentUser) @@ -513,6 +520,42 @@ def test_ListProjectsBuildsComputedFields(service, mapper, currentUser, monkeypa "thumbnailVersion": "thumb-v1", } ] + assert mapper.lastCountProjectProtocolsCall == {"projectId": 1} + + +def test_ListProjectsFallsBackToFilesystemProtocolCountWhenPostgresqlCountFails( + service, + mapper, + currentUser, + monkeypatch, +): + storedPath = Path(service.manager.PROJECTS) / "demo-project" + storedPath.mkdir(parents=True, exist_ok=True) + + mapper.projectsListResult = [ + { + "id": 1, + "name": str(storedPath), + "description": "demo description", + "createdAt": "2026-04-15T10:00:00", + "updatedAt": "2026-04-15T11:00:00", + "status": "active", + "ownerId": 1, + } + ] + + def failCountProjectProtocols(projectId): + raise RuntimeError("postgresql count failed") + + mapper.countProjectProtocols = failCountProjectProtocols + + monkeypatch.setattr(service, "getProjectSize", lambda path: 0) + monkeypatch.setattr(service, "countProtocols", lambda path: 7) + monkeypatch.setattr(service, "_buildProjectThumbnailVersion", lambda **kwargs: "thumb-v1") + + result = service.listProjects(mapper, currentUser) + + assert result[0]["protocolsCount"] == "7" def test_ListProjectsKeepsSharedFlagsFromMapper(service, mapper, currentUser, monkeypatch): diff --git a/tests/unit/backend/api/services/test_project_service_protocol_steps.py b/tests/unit/backend/api/services/test_project_service_protocol_steps.py index b6fcc2c5..78d4ced6 100644 --- a/tests/unit/backend/api/services/test_project_service_protocol_steps.py +++ b/tests/unit/backend/api/services/test_project_service_protocol_steps.py @@ -37,13 +37,43 @@ def getProtocol(self, protocolId): return self.protocols[int(protocolId)] +class FakeDb: + def __init__(self): + self.runtimeProtocolIdByDbId = {} + self.fetchOneCalls = [] + + def fetchOne(self, query, params): + self.fetchOneCalls.append({ + "query": query, + "params": params, + }) + + if len(params) < 3: + return None + + protocolDbId = params[1] + runtimeProtocolId = self.runtimeProtocolIdByDbId.get(int(protocolDbId)) + if runtimeProtocolId is None: + return None + + return { + "protocolId": runtimeProtocolId, + } + + class FakeMapper: def __init__(self): + self.db = FakeDb() self.listProtocolStepsResult = [] + self.listProtocolStepsCalls = [] self.updateProtocolStepStatusCalls = [] self.updateProtocolStepStatusResult = None def listProtocolSteps(self, projectId, protocolId): + self.listProtocolStepsCalls.append({ + "projectId": projectId, + "protocolId": protocolId, + }) return self.listProtocolStepsResult def updateProtocolStepStatus(self, **kwargs): @@ -133,6 +163,30 @@ def test_ListProtocolStepsDelegatesToMapper(service, mapper): assert result == [{"index": 1, "name": "resumeStep", "status": "finished"}] +def test_ListProtocolStepsResolvesPostgresqlProtocolId(service, mapper): + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.listProtocolStepsResult = [ + {"index": 1, "name": "resumeStep", "status": "finished"}, + ] + + result = service.listProtocolStepsService( + mapper=mapper, + projectId=1, + protocolId=500, + ) + + assert result == [ + {"index": 1, "name": "resumeStep", "status": "finished"}, + ] + assert mapper.listProtocolStepsCalls == [ + { + "projectId": 1, + "protocolId": 10, + } + ] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + def test_UpdateProtocolStepStatusUpdatesScipionAndPostgres( projectServiceModule, service, @@ -173,6 +227,48 @@ def test_UpdateProtocolStepStatusUpdatesScipionAndPostgres( ] +def test_UpdateProtocolStepStatusResolvesPostgresqlProtocolId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "STATUS_FINISHED", "finished") + + step = FakeStep(index=2, objId=102) + protocol = FakeProtocolWithSteps([step]) + + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.updateProtocolStepStatusResult = { + "index": 2, + "name": "processStep", + "status": "finished", + } + + result = service.updateProtocolStepStatusService( + mapper=mapper, + projectId=1, + protocolId=500, + stepIndex=2, + stepStatus="finished", + ) + + assert result == mapper.updateProtocolStepStatusResult + assert protocol.updateStepsCalls == [{"where": "id='102'"}] + assert step.status == "finished" + + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + assert mapper.updateProtocolStepStatusCalls == [ + { + "projectId": 1, + "protocolId": 10, + "stepIndex": 2, + "stepStatus": "finished", + }, + ] + + def test_UpdateProtocolStepStatusAcceptsObjIdHolderFallback( projectServiceModule, service, @@ -230,7 +326,7 @@ def test_UpdateProtocolStepStatusRaisesWhenCurrentProjectIsMissing(service, mapp ) assert exc.value.status_code == 500 - assert exc.value.detail == "No current project loaded" + assert exc.value.detail == "No current Scipion project loaded" def test_UpdateProtocolStepStatusRaisesWhenProtocolIsMissing(service, mapper): @@ -244,7 +340,7 @@ def test_UpdateProtocolStepStatusRaisesWhenProtocolIsMissing(service, mapper): ) assert exc.value.status_code == 404 - assert exc.value.detail == "Protocol not found: 99" + assert str(exc.value.detail).startswith("Protocol not found in Scipion runtime: 99") def test_UpdateProtocolStepStatusRaisesWhenStepIsMissing(service, mapper): diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index 638a5fe7..25801e11 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -25,6 +25,7 @@ # ****************************************************************************** import importlib +import json import pytest from fastapi import HTTPException @@ -86,6 +87,44 @@ class FakeCsvList(FakeBaseParam): pass +class FakePointerParam(FakeBaseParam): + def __init__(self, label="Pointer", choices=None, validationErrors=None, allowsNull=True, condition=None): + super().__init__( + label=label, + choices=choices, + validationErrors=validationErrors, + allowsNull=allowsNull, + condition=condition, + ) + self.default = FakeValueHolder(None) + + +class FakeMultiPointerParam(FakeBaseParam): + pass + + +class FakeRelationParam(FakeBaseParam): + pass + + +class FakePointerAttribute: + def __init__(self): + self.extended = None + + def setExtended(self, extended): + self.extended = extended + + +class FakePointerList(list): + def isEmpty(self): + return len(self) == 0 + + +class FakePointer: + def __init__(self, protocol, extended=None): + self.protocol = protocol + self.extended = extended + class FakeProtocol: # fakeProtocol def __init__(self, objId=None, className="ProtClass", useQueueFlag=False, validateErrors=None): @@ -112,6 +151,12 @@ def addParam(self, name, param): def getParam(self, name): return self._params.get(name) + def getPlugin(self): + return None + + def getClassName(self): + return self._className + def setAttributeValue(self, name, value): self.attributeValues[name] = value @@ -174,6 +219,10 @@ def newProtocol(self, protClass): def getProtocol(self, protocolId): return self.protocols[int(protocolId)] + def _fixProtParamsConfiguration(self, protocol): + self.fixedProtocolParams = getattr(self, "fixedProtocolParams", []) + self.fixedProtocolParams.append(protocol) + def _storeProtocol(self, protocol): self.storedProtocols.append(protocol) @@ -212,9 +261,34 @@ def resetWorkFlow(self, workflowProtocolList): return self.resetWorkflowResult +class FakeDb: + def __init__(self): + self.runtimeProtocolIdByDbId = {} + self.fetchOneCalls = [] + + def fetchOne(self, query, params): + self.fetchOneCalls.append({ + "query": query, + "params": params, + }) + + if len(params) < 3: + return None + + protocolDbId = params[1] + runtimeProtocolId = self.runtimeProtocolIdByDbId.get(int(protocolDbId)) + if runtimeProtocolId is None: + return None + + return { + "protocolId": runtimeProtocolId, + } + + class FakeMapper: # fakeMapper def __init__(self): + self.db = FakeDb() self.dbProtocolsByProtocolId = {} self.savedProtocolContexts = [] self.deleteProtocolCalls = [] @@ -272,147 +346,816 @@ def assertSuccessEnvelope(result): assert result["errors"] == [] -def test_CastParamValueSupportsEnumLookup(projectServiceModule, service, monkeypatch): - monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) +def test_BuildMissingOutputSyncItemsClassifiesMissingOutputs(service): + declaredOutputs = [ + { + "outputName": "persistedOutput", + "outputClassName": "SetOfParticles", + }, + { + "outputName": "skippedOutput", + "outputClassName": "UnsupportedOutput", + }, + { + "outputName": "erroredOutput", + "outputClassName": "SetOfBad", + }, + { + "outputName": "missingOutput", + "outputClassName": "SetOfMissing", + }, + ] + persistedOutputs = [ + { + "outputName": "persistedOutput", + "outputClassName": "SetOfParticles", + } + ] + skippedOutputs = [ + { + "outputName": "skippedOutput", + "outputClassName": "UnsupportedOutput", + "reason": "unsupported_output_type", + } + ] + outputErrors = [ + { + "outputName": "erroredOutput", + "outputClassName": "SetOfBad", + "error": "broken output", + } + ] - param = FakeEnumParam(choices=["Continue", "Restart"]) + result = service._buildMissingOutputSyncItems( + protocolId=10, + declaredOutputs=declaredOutputs, + persistedOutputs=persistedOutputs, + skippedOutputs=skippedOutputs, + outputErrors=outputErrors, + ) - assert service.castParamValue(param, "Restart") == 1 - assert service.castParamValue(param, "restart") == 1 - assert service.castParamValue(param, 0) == 0 + assert result == [ + { + "protocolId": "10", + "outputName": "skippedOutput", + "outputClassName": "UnsupportedOutput", + "reason": "unsupported_output_type", + }, + { + "protocolId": "10", + "outputName": "erroredOutput", + "outputClassName": "SetOfBad", + "reason": "persistence_error", + "error": "broken output", + }, + { + "protocolId": "10", + "outputName": "missingOutput", + "outputClassName": "SetOfMissing", + "reason": "not_persisted", + }, + ] + +def test_RegisterOutputReturnsPersistenceReport( + projectServiceModule, + service, + monkeypatch, +): + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakeSetOutput: + # fakeSetOutput + def getClassName(self): + return "SetOfParticles" + + class FakeObjectOutput: + # fakeObjectOutput + def getClassName(self): + return "Volume" + + class FakeBadSetOutput: + # fakeBadSetOutput + def getClassName(self): + return "SetOfBadThings" + + class FakeUnsupportedOutput: + # fakeUnsupportedOutput + pass + + class FakeProtocolWithOutputs: + # fakeProtocolWithOutputs + def getObjId(self): + return 10 + + def iterOutputAttributes(self): + return [ + ("outputParticles", FakeSetOutput()), + ("outputVolume", FakeObjectOutput()), + ("badSet", FakeBadSetOutput()), + ("emptyOutput", None), + ("unsupportedOutput", FakeUnsupportedOutput()), + ] + + class FakeScipionSetPostgresqlMapper: + # fakeScipionSetPostgresqlMapper + def __init__(self, db): + self.db = db + + def storeSet(self, projectId, protocolDbId, outputName, scipionSet): + if outputName == "badSet": + raise RuntimeError("broken set output") + + return { + "projectId": projectId, + "protocolDbId": protocolDbId, + "stored": True, + } + class FakeScipionObjectPostgresqlMapper: + # fakeScipionObjectPostgresqlMapper + def __init__(self, db): + self.db = db + + def storeObjectTree( + self, + projectId, + protocolDbId, + outputName, + scipionObj, + includeNestedProperties, + ): + return { + "projectId": projectId, + "protocolDbId": protocolDbId, + "stored": True, + "includeNestedProperties": includeNestedProperties, + } -def test_CastParamValueSupportsPrimitiveTypes(projectServiceModule, service, monkeypatch): - monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) - monkeypatch.setattr(projectServiceModule, "FloatParam", FakeFloatParam) - monkeypatch.setattr(projectServiceModule, "BooleanParam", FakeBooleanParam) - monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) - monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) - monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) + mapperPackage = importlib.import_module("app.backend.mapper") - assert service.castParamValue(FakeIntParam(), "7") == 7 - assert service.castParamValue(FakeFloatParam(), "3.5") == 3.5 - assert service.castParamValue(FakeBooleanParam(), "yes") is True - assert service.castParamValue(FakeBooleanParam(), "no") is False - assert service.castParamValue(FakeStringParam(), 123) == "123" - assert service.castParamValue(FakeCsvList(), "item") == ["item"] + monkeypatch.setattr( + mapperPackage, + "ScipionSetPostgresqlMapper", + FakeScipionSetPostgresqlMapper, + ) + monkeypatch.setattr( + mapperPackage, + "ScipionObjectPostgresqlMapper", + FakeScipionObjectPostgresqlMapper, + ) + monkeypatch.setattr( + service, + "_resolveProtocolDbIdForOutputPersistence", + lambda db, projectId, protocol: 500, + ) + report = service.registerOutput( + projectId=1, + protocol=FakeProtocolWithOutputs(), + mapper=FakeMapper(), + returnReport=True, + ) + + assert report["declared"] == [ + { + "outputName": "outputParticles", + "outputClassName": "SetOfParticles", + }, + { + "outputName": "outputVolume", + "outputClassName": "Volume", + }, + { + "outputName": "badSet", + "outputClassName": "SetOfBadThings", + }, + { + "outputName": "emptyOutput", + "outputClassName": "", + }, + { + "outputName": "unsupportedOutput", + "outputClassName": "FakeUnsupportedOutput", + }, + ] + + assert report["persisted"] == [ + { + "projectId": 1, + "protocolDbId": 500, + "stored": True, + "mapperKind": "flat_set", + "outputName": "outputParticles", + "outputClassName": "SetOfParticles", + }, + { + "projectId": 1, + "protocolDbId": 500, + "stored": True, + "includeNestedProperties": True, + "mapperKind": "tree", + "outputName": "outputVolume", + "outputClassName": "Volume", + }, + ] + + assert report["skipped"] == [ + { + "outputName": "emptyOutput", + "outputClassName": "", + "reason": "empty_output", + }, + { + "outputName": "unsupportedOutput", + "outputClassName": "FakeUnsupportedOutput", + "reason": "unsupported_output_type", + }, + ] + + assert report["errors"] == [ + { + "outputName": "badSet", + "outputClassName": "SetOfBadThings", + "error": "broken set output", + } + ] + + +def test_SyncProjectProtocolsAndDependenciesReportsOutputPersistence( + projectServiceModule, + service, + monkeypatch, +): + service.syncProjectProtocolsAndDependencies = ( + projectServiceModule.ProjectService.syncProjectProtocolsAndDependencies.__get__( + service, + projectServiceModule.ProjectService, + ) + ) + + class FakeProtocolNode: + # fakeProtocolNode + def __init__(self, protocol): + self.run = protocol + self._parents = [] + + class FakeRunsGraph: + # fakeRunsGraph + def __init__(self, protocol): + self._nodesDict = { + "PROJECT": object(), + "10": FakeProtocolNode(protocol), + } + + class FakeCurrentProjectForSync: + # fakeCurrentProjectForSync + def __init__(self, protocol): + self.protocol = protocol + + def getRunsGraph(self, refresh=False, checkPids=False): + return FakeRunsGraph(self.protocol) + + class FakeProtocolForSync: + # fakeProtocolForSync + def getObjId(self): + return 10 + + def iterInputAttributes(self): + return [] + + class FakeSyncMapper: + # fakeSyncMapper + def __init__(self): + self.savedProtocolContexts = [] + self.deletedProtocolIds = None + self.savedEdges = None + self.savedInputRefs = None + + def saveProtocol(self, protocolContext): + self.savedProtocolContexts.append(protocolContext) + return 500 + + def deleteProjectProtocolsNotInProtocolIds(self, projectId, protocolIds): + self.deletedProtocolIds = { + "projectId": projectId, + "protocolIds": protocolIds, + } + + def replaceProjectProtocolDependencies(self, projectId, edges): + self.savedEdges = { + "projectId": projectId, + "edges": edges, + } + return len(edges) + + def replaceProjectProtocolInputRefs(self, projectId, inputRefs): + self.savedInputRefs = { + "projectId": projectId, + "inputRefs": inputRefs, + } + return len(inputRefs) + + protocol = FakeProtocolForSync() + service.currentProject = FakeCurrentProjectForSync(protocol) -def test_SaveProtocolCreatesNewProtocolAndPersistsContext(projectServiceModule, service, mapper, monkeypatch): - monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) - monkeypatch.setattr(projectServiceModule, "FloatParam", FakeFloatParam) - monkeypatch.setattr(projectServiceModule, "BooleanParam", FakeBooleanParam) - monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) - monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) - monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) - monkeypatch.setattr(service, "applyParamsToProtocol", lambda protocol, params: []) monkeypatch.setattr( service, "_buildProtocolContext", lambda projectId, protocol: { "projectId": projectId, "protocolId": protocol.getObjId(), - "label": protocol._label, - "runName": protocol.runName.get(), - "comment": protocol._objComment.get(), }, ) - - def fakeSyncProjectProtocolsAndDependencies(mapperObj, projectId, refresh=False, checkPid=False): - for protocolObj in service.currentProject.setupProtocols: - mapperObj.saveProtocol(service._buildProtocolContext(projectId, protocolObj)) - return {"protocols": len(service.currentProject.setupProtocols), "dependencies": 0} - monkeypatch.setattr( service, - "syncProjectProtocolsAndDependencies", - fakeSyncProjectProtocolsAndDependencies, + "_shouldRegisterProtocolOutputs", + lambda protocol: True, + ) + monkeypatch.setattr( + service, + "registerOutput", + lambda projectId, protocol, mapper, returnReport=False: { + "declared": [ + { + "outputName": "outputParticles", + "outputClassName": "SetOfParticles", + }, + { + "outputName": "outputVolume", + "outputClassName": "Volume", + }, + { + "outputName": "unsupportedOutput", + "outputClassName": "UnsupportedOutput", + }, + { + "outputName": "badOutput", + "outputClassName": "SetOfBad", + }, + { + "outputName": "orphanOutput", + "outputClassName": "SetOfOrphan", + }, + ], + "persisted": [ + { + "mapperKind": "flat_set", + "outputName": "outputParticles", + }, + { + "mapperKind": "tree", + "outputName": "outputVolume", + }, + ], + "skipped": [ + { + "outputName": "unsupportedOutput", + "outputClassName": "UnsupportedOutput", + "reason": "unsupported_output_type", + } + ], + "errors": [ + { + "outputName": "badOutput", + "outputClassName": "SetOfBad", + "error": "boom", + } + ], + }, ) - def buildProtocol(): - protocol = FakeProtocol(objId=None, className="ProtClass") - protocol.addParam("runName", FakeStringParam(label="Run name")) - protocol.addParam("iterations", FakeIntParam(label="Iterations")) - return protocol - - service.currentProject.protocolFactories["ProtClass"] = buildProtocol + mapper = FakeSyncMapper() - protocol, errors = service.saveProtocol( + result = service.syncProjectProtocolsAndDependencies( mapper=mapper, projectId=1, - protocolId=None, - protocolClassName="ProtClass", - params={ - "runName": "My protocol", - "iterations": "5", - "_objComment": "comment", - }, + refresh=True, + checkPid=True, ) - assert errors == [] - assert protocol.getObjId() == 999 - assert protocol._label is None - assert protocol.runName.get() == "My protocol" - assert protocol.attributeValues["runName"] == "My protocol" - assert protocol.attributeValues["iterations"] == 5 - assert protocol._objComment.get() == "comment" - assert len(service.currentProject.setupProtocols) == 1 + assert result == { + "protocols": 1, + "dependencies": 0, + "inputRefs": 0, + "steps": 0, + "stepsProtocols": 0, + "stepErrors": [], + "outputsDeclared": 5, + "outputs": 2, + "outputsMissing": 3, + "outputsByKind": { + "flat_set": 1, + "tree": 1, + }, + "outputMissing": [ + { + "protocolId": "10", + "outputName": "unsupportedOutput", + "outputClassName": "UnsupportedOutput", + "reason": "unsupported_output_type", + }, + { + "protocolId": "10", + "outputName": "badOutput", + "outputClassName": "SetOfBad", + "reason": "persistence_error", + "error": "boom", + }, + { + "protocolId": "10", + "outputName": "orphanOutput", + "outputClassName": "SetOfOrphan", + "reason": "not_persisted", + }, + ], + "outputErrors": [ + { + "protocolId": "10", + "outputName": "unsupportedOutput", + "outputClassName": "UnsupportedOutput", + "reason": "unsupported_output_type", + }, + { + "protocolId": "10", + "outputName": "badOutput", + "outputClassName": "SetOfBad", + "error": "boom", + }, + ], + } + assert mapper.savedProtocolContexts == [ { "projectId": 1, - "protocolId": 999, - "label": None, - "runName": "My protocol", - "comment": "comment", + "protocolId": 10, } ] + assert mapper.deletedProtocolIds == { + "projectId": 1, + "protocolIds": ["10"], + } -def test_SaveProtocolAggregatesValidationAndPointerErrors(projectServiceModule, service, mapper, monkeypatch): - monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) - monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) - monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) - monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) +def test_GetPostgresqlIntegratedAnalyzeContextUsesResolvedProtocolId( + service, + monkeypatch, +): + readerCalls = [] + resolverCalls = [] - protocol = FakeProtocol(objId=10, className="ProtClass") - protocol.addParam( - "iterations", - FakeIntParam(label="Iterations", validationErrors=["must be greater than zero"]), + class FakeDb: + pass + + class FakeMapper: + def __init__(self): + self.db = FakeDb() + + class FakePostgresqlIntegratedContextReader: + def __init__(self, db, projectId, protocolId, outputName): + readerCalls.append({ + "db": db, + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + }) + + def getContext(self): + return { + "root": { + "projectId": 1, + "protocolId": 321, + "outputName": "outputTiltSeries", + } + } + + readerModule = importlib.import_module( + "app.backend.viewers.postgresql_integrated_context_reader" ) - service.currentProject.protocols[10] = protocol - mapper.dbProtocolsByProtocolId[(10, 1)] = {"id": 500, "protocolId": 10} - monkeypatch.setattr(service, "applyParamsToProtocol", lambda protocolObj, params: ["pointer error"]) + monkeypatch.setattr( + readerModule, + "PostgresqlIntegratedContextReader", + FakePostgresqlIntegratedContextReader, + ) - def fakeSyncProjectProtocolsAndDependencies(mapperObj, projectId, refresh=False, checkPid=False): - for protocolObj in service.currentProject.storedProtocols: - mapperObj.saveProtocol(service._buildProtocolContext(projectId, protocolObj)) - return {"protocols": len(service.currentProject.storedProtocols), "dependencies": 0} + def fakeResolvePostgresqlReaderProtocolId(mapper, projectId, protocolId): + resolverCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + }) + return 321 monkeypatch.setattr( service, - "syncProjectProtocolsAndDependencies", - fakeSyncProjectProtocolsAndDependencies, + "_resolvePostgresqlReaderProtocolId", + fakeResolvePostgresqlReaderProtocolId, ) - _, errors = service.saveProtocol( + mapper = FakeMapper() + + result = service._getPostgresqlIntegratedAnalyzeContextIfAvailable( mapper=mapper, projectId=1, protocolId=10, - protocolClassName="ProtClass", - params={"iterations": "3"}, + outputName="outputTiltSeries", ) - assert errors == [ - "**Iterations** must be greater than zero", - "pointer error", + assert result == { + "root": { + "projectId": 1, + "protocolId": 321, + "outputName": "outputTiltSeries", + } + } + + assert resolverCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 10, + } ] - assert len(service.currentProject.storedProtocols) == 1 + assert readerCalls == [ + { + "db": mapper.db, + "projectId": 1, + "protocolId": 321, + "outputName": "outputTiltSeries", + } + ] -def test_LaunchProtocolRejectsUnknownExecuteMode(service, mapper): - with pytest.raises(HTTPException) as exc: - service.launchProtocol( + +def test_GetIntegratedAnalyzeContextRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + class FakeMapper: + pass + + class RuntimeShouldNotBeUsed: + def getProtocol(self, protocolId): + raise AssertionError("Runtime fallback should not be used") + + service.currentProject = RuntimeShouldNotBeUsed() + + monkeypatch.setattr( + service, + "_getPostgresqlIntegratedAnalyzeContextIfAvailable", + lambda mapper, projectId, protocolId, outputName: None, + ) + + with pytest.raises(HTTPException) as exc: + service.getIntegratedAnalyzeContextService( + mapper=FakeMapper(), + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == ( + "Integrated Analyze Context output is not available in PostgreSQL metadata: " + "context_not_available" + ) + + +def test_GetIntegratedAnalyzeContextKeepsLegacyRuntimeFallbackWithoutMapper( + service, + monkeypatch, +): + class FakeTiltSeriesOutput: + def getClassName(self): + return "SetOfTiltSeries" + + def getObjId(self): + return 22 + + def getSize(self): + return 1 + + def getTSIds(self): + return ["TS_001"] + + def getSamplingRate(self): + return 1.5 + + def getDimensions(self): + return (100, 100, 40) + + def iterItems(self): + return iter([]) + + class FakeProtocol: + def __init__(self): + self.outputTiltSeries = FakeTiltSeriesOutput() + + def iterInputAttributes(self): + return [] + + def iterOutputAttributes(self): + return [("outputTiltSeries", self.outputTiltSeries)] + + class FakeCurrentProjectForIntegratedContext: + def __init__(self): + self.protocol = FakeProtocol() + + def getProtocol(self, protocolId): + assert protocolId == 10 + return self.protocol + + service.currentProject = FakeCurrentProjectForIntegratedContext() + + monkeypatch.setattr( + service, + "_getPostgresqlIntegratedAnalyzeContextIfAvailable", + lambda mapper, projectId, protocolId, outputName: None, + ) + + result = service.getIntegratedAnalyzeContextService( + mapper=None, + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + ) + + assert result["root"] == { + "projectId": 1, + "protocolId": 10, + "outputName": "outputTiltSeries", + "outputClass": "SetOfTiltSeries", + } + + assert result["links"]["tiltSeries"] == { + "protocolId": 10, + "outputName": "outputTiltSeries", + "itemId": 22, + "label": "outputTiltSeries", + "status": "available", + } + + assert result["summaries"]["tiltSeries"]["objectClass"] == "SetOfTiltSeries" + assert result["summaries"]["tiltSeries"]["objectId"] == 22 + assert result["summaries"]["tiltSeries"]["size"] == 1 + + +def test_CastParamValueSupportsEnumLookup(projectServiceModule, service, monkeypatch): + monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) + + param = FakeEnumParam(choices=["Continue", "Restart"]) + + assert service.castParamValue(param, "Restart") == 1 + assert service.castParamValue(param, "restart") == 1 + assert service.castParamValue(param, 0) == 0 + + +def test_CastParamValueSupportsPrimitiveTypes(projectServiceModule, service, monkeypatch): + monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) + monkeypatch.setattr(projectServiceModule, "FloatParam", FakeFloatParam) + monkeypatch.setattr(projectServiceModule, "BooleanParam", FakeBooleanParam) + monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) + monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) + monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) + + assert service.castParamValue(FakeIntParam(), "7") == 7 + assert service.castParamValue(FakeFloatParam(), "3.5") == 3.5 + assert service.castParamValue(FakeBooleanParam(), "yes") is True + assert service.castParamValue(FakeBooleanParam(), "no") is False + assert service.castParamValue(FakeStringParam(), 123) == "123" + assert service.castParamValue(FakeCsvList(), "item") == ["item"] + + +def test_SaveProtocolCreatesNewProtocolAndPersistsContext(projectServiceModule, service, mapper, monkeypatch): + monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) + monkeypatch.setattr(projectServiceModule, "FloatParam", FakeFloatParam) + monkeypatch.setattr(projectServiceModule, "BooleanParam", FakeBooleanParam) + monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) + monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) + monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) + monkeypatch.setattr( + service, + "applyParamsToProtocol", + lambda mapper=None, projectId=None, protocol=None, params=None: [], + ) + monkeypatch.setattr( + service, + "_buildProtocolContext", + lambda projectId, protocol: { + "projectId": projectId, + "protocolId": protocol.getObjId(), + "label": protocol._label, + "runName": protocol.runName.get(), + "comment": protocol._objComment.get(), + }, + ) + + def fakeSyncProjectProtocolsAndDependencies(mapperObj, projectId, refresh=False, checkPid=False): + for protocolObj in service.currentProject.setupProtocols: + mapperObj.saveProtocol(service._buildProtocolContext(projectId, protocolObj)) + return {"protocols": len(service.currentProject.setupProtocols), "dependencies": 0} + + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + fakeSyncProjectProtocolsAndDependencies, + ) + + def buildProtocol(): + protocol = FakeProtocol(objId=None, className="ProtClass") + protocol.addParam("runName", FakeStringParam(label="Run name")) + protocol.addParam("iterations", FakeIntParam(label="Iterations")) + return protocol + + service.currentProject.protocolFactories["ProtClass"] = buildProtocol + + protocol, errors = service.saveProtocol( + mapper=mapper, + projectId=1, + protocolId=None, + protocolClassName="ProtClass", + params={ + "runName": "My protocol", + "iterations": "5", + "_objComment": "comment", + }, + ) + + assert errors == [] + assert protocol.getObjId() == 999 + assert protocol._label is None + assert protocol.runName.get() == "My protocol" + assert protocol.attributeValues["runName"] == "My protocol" + assert protocol.attributeValues["iterations"] == 5 + assert protocol._objComment.get() == "comment" + assert len(service.currentProject.setupProtocols) == 1 + assert mapper.savedProtocolContexts == [ + { + "projectId": 1, + "protocolId": 999, + "label": None, + "runName": "My protocol", + "comment": "comment", + } + ] + + +def test_SaveProtocolAggregatesValidationAndPointerErrors(projectServiceModule, service, mapper, monkeypatch): + monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) + monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) + monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) + monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) + + protocol = FakeProtocol(objId=10, className="ProtClass") + protocol.addParam( + "iterations", + FakeIntParam(label="Iterations", validationErrors=["must be greater than zero"]), + ) + service.currentProject.protocols[10] = protocol + mapper.dbProtocolsByProtocolId[(10, 1)] = {"id": 500, "protocolId": 10} + + monkeypatch.setattr( + service, + "applyParamsToProtocol", + lambda mapper=None, projectId=None, protocol=None, params=None: ["pointer error"], + ) + + def fakeSyncProjectProtocolsAndDependencies(mapperObj, projectId, refresh=False, checkPid=False): + for protocolObj in service.currentProject.storedProtocols: + mapperObj.saveProtocol(service._buildProtocolContext(projectId, protocolObj)) + return {"protocols": len(service.currentProject.storedProtocols), "dependencies": 0} + + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + fakeSyncProjectProtocolsAndDependencies, + ) + + _, errors = service.saveProtocol( + mapper=mapper, + projectId=1, + protocolId=10, + protocolClassName="ProtClass", + params={"iterations": "3"}, + ) + + assert errors == [ + "**Iterations** must be greater than zero", + "pointer error", + ] + assert len(service.currentProject.storedProtocols) == 1 + + +def test_LaunchProtocolRejectsUnknownExecuteMode(service, mapper): + with pytest.raises(HTTPException) as exc: + service.launchProtocol( mapper=mapper, projectId=1, protocolId="10", @@ -428,9 +1171,28 @@ def test_LaunchProtocolRejectsUnknownExecuteMode(service, mapper): def test_LaunchProtocolStopDelegatesToStopProtocol(service, mapper, monkeypatch): calls = [] - monkeypatch.setattr(service, "stopProtocol", lambda protocolIds: calls.append(protocolIds)) + def fakeStopProtocol(mapper, projectId, protocolIds): + calls.append(protocolIds) - service.launchProtocol( + def fakeSyncProjectProtocolsAndDependencies( + mapper, + projectId, + refresh=False, + checkPid=False, + ): + return { + "protocols": 1, + "dependencies": 0, + } + + monkeypatch.setattr(service, "stopProtocol", fakeStopProtocol) + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + fakeSyncProjectProtocolsAndDependencies, + ) + + result = service.launchProtocol( mapper=mapper, projectId=1, protocolId="10", @@ -440,6 +1202,10 @@ def test_LaunchProtocolStopDelegatesToStopProtocol(service, mapper, monkeypatch) ) assert calls == [["10"]] + assert result == { + "protocols": 1, + "dependencies": 0, + } def test_LaunchProtocolRaises422WhenValidationFails(service, mapper, monkeypatch): @@ -463,7 +1229,9 @@ def test_LaunchProtocolRaises422WhenValidationFails(service, mapper, monkeypatch @pytest.mark.parametrize( "executeMode, expectedRunMode", [ + (None, "resume-mode"), ("launch", "resume-mode"), + ("resume", "resume-mode"), ("restart", "restart-mode"), ], ) @@ -519,7 +1287,13 @@ def test_RenameProtocolStoresAnnotation(service): protocol = FakeProtocol(objId=10) service.currentProject.protocols[10] = protocol - result = service.renameProtocol(10, "Renamed protocol", "Updated comment") + result = service.renameProtocol( + mapper=None, + projectId=None, + protocolId=10, + newName="Renamed protocol", + newComment="Updated comment", + ) assertSuccessEnvelope(result) assert protocol._label is None @@ -624,7 +1398,11 @@ def test_RestartProtocolAllReturnsCollectedErrors(service): service.currentProject.restartWorkflowInjectedErrors = ["cannot restart", "blocked"] with pytest.raises(HTTPException) as exc: - service.restartProtocolAll(10) + service.restartProtocolAll( + mapper=None, + projectId=None, + protocolId=10, + ) assert exc.value.status_code == 422 assert exc.value.detail == ["cannot restart", "blocked"] @@ -656,7 +1434,11 @@ def test_ResetProtocolFromReturnsSuccessWhenWorkflowResets(service): service.currentProject.protocols[10] = protocol service.currentProject.resetWorkflowResult = [] - result = service.resetProtocolFrom(10) + result = service.resetProtocolFrom( + mapper=None, + projectId=None, + protocolId=10, + ) assertSuccessEnvelope(result) @@ -667,7 +1449,1004 @@ def test_StopProtocolStopsEachProtocol(service): service.currentProject.protocols[10] = protocolA service.currentProject.protocols[11] = protocolB - result = service.stopProtocol(["10", "11"]) + result = service.stopProtocol( + mapper=None, + projectId=None, + protocolIds=["10", "11"], + ) + + assertSuccessEnvelope(result) + assert service.currentProject.stoppedProtocols == [protocolA, protocolB] + + +def test_RenameProtocolResolvesPostgresqlProtocolId(service, mapper): + protocol = FakeProtocol(objId=10) + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + result = service.renameProtocol( + mapper=mapper, + projectId=1, + protocolId=500, + newName="Renamed protocol", + newComment="Updated comment", + ) + + assertSuccessEnvelope(result) + assert protocol.runName.get() == "Renamed protocol" + assert protocol._objComment == "Updated comment" + assert service.currentProject.storedProtocols == [protocol] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_DuplicateProtocolResolvesPostgresqlProtocolIds(service, mapper, monkeypatch): + protocolA = FakeProtocol(objId=10) + protocolB = FakeProtocol(objId=11) + copiedA = FakeProtocol(objId=110) + copiedB = FakeProtocol(objId=111) + + service.currentProject.protocols[10] = protocolA + service.currentProject.protocols[11] = protocolB + service.currentProject.copiedProtocolOutputs = [copiedA, copiedB] + + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.db.runtimeProtocolIdByDbId[501] = 11 + + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + lambda mapper, projectId, refresh=False, checkPid=False: { + "protocols": 2, + "dependencies": 0, + }, + ) + + class DuplicateItem: + def __init__(self, itemId): + self.id = itemId + + result = service.duplicateProtocol( + mapper=mapper, + projectId=1, + protocols=[DuplicateItem("500"), DuplicateItem("501")], + ) assertSuccessEnvelope(result) - assert service.currentProject.stoppedProtocols == [protocolA, protocolB] \ No newline at end of file + assert service.currentProject.copiedProtocolInputs == [[protocolA, protocolB]] + assert result["duplicated"] == [ + {"sourceId": "500", "newId": "110"}, + {"sourceId": "501", "newId": "111"}, + ] + + +def test_DeleteProtocolResolvesPostgresqlProtocolIds(service, mapper, monkeypatch): + protocolA = FakeProtocol(objId=10) + protocolB = FakeProtocol(objId=11) + + service.currentProject.protocols[10] = protocolA + service.currentProject.protocols[11] = protocolB + + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.db.runtimeProtocolIdByDbId[501] = 11 + + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + lambda mapper, projectId, refresh=False, checkPid=False: { + "protocols": 0, + "dependencies": 0, + }, + ) + + result = service.deleteProtocol( + mapper=mapper, + projectId=1, + protocols=["500", "501"], + ) + + assert result["status"] == 0 + assert service.currentProject.deleteProtocolCalls == [[protocolA, protocolB]] + assert mapper.deleteProtocolCalls == [ + { + "projectId": 1, + "protocolList": [protocolA, protocolB], + } + ] + +def test_StopProtocolResolvesPostgresqlProtocolIds(service, mapper): + protocolA = FakeProtocol(objId=10) + protocolB = FakeProtocol(objId=11) + + service.currentProject.protocols[10] = protocolA + service.currentProject.protocols[11] = protocolB + + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.db.runtimeProtocolIdByDbId[501] = 11 + + result = service.stopProtocol( + mapper=mapper, + projectId=1, + protocolIds=["500", "501"], + ) + + assertSuccessEnvelope(result) + assert service.currentProject.stoppedProtocols == [protocolA, protocolB] + + +def test_RestartProtocolAllResolvesPostgresqlProtocolId(service, mapper, monkeypatch): + protocol = FakeProtocol(objId=10) + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + subworkflowCalls = [] + + def fakeGetSubworkflow(protocolObj): + subworkflowCalls.append(protocolObj) + return [protocol], [] + + service.currentProject._getSubworkflow = fakeGetSubworkflow + + cleanupCalls = [] + + monkeypatch.setattr( + service, + "_deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql", + lambda mapper, projectId, protocols: cleanupCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocols": protocols, + }) or { + "protocolsCount": len(protocols), + "setsDeleted": 0, + "objectsDeleted": 0, + "items": [], + }, + ) + + result = service.restartProtocolAll( + mapper=mapper, + projectId=1, + protocolId=500, + ) + + assertSuccessEnvelope(result) + assert subworkflowCalls == [protocol] + assert cleanupCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocols": [protocol], + } + ] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_ContinueProtocolAllResolvesPostgresqlProtocolId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "MODE_RESUME", "resume-mode") + + protocol = FakeProtocol(objId=10) + activeProtocol = FakeProtocol(objId=20) + + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + subworkflowCalls = [] + + def fakeGetSubworkflow(protocolObj): + subworkflowCalls.append(protocolObj) + return [protocol], [activeProtocol] + + service.currentProject._getSubworkflow = fakeGetSubworkflow + + result = service.continueProtocolAll( + mapper=mapper, + projectId=1, + protocolId=500, + currentUser={"id": 1}, + ) + + assertSuccessEnvelope(result) + assert subworkflowCalls == [protocol] + assert activeProtocol.runMode.get() == "resume-mode" + assert service.currentProject.launchedProtocols == [activeProtocol] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_ResetProtocolFromResolvesPostgresqlProtocolId(service, mapper, monkeypatch): + protocol = FakeProtocol(objId=10) + + service.currentProject.protocols[10] = protocol + service.currentProject.resetWorkflowResult = [] + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + subworkflowCalls = [] + + def fakeGetSubworkflow(protocolObj): + subworkflowCalls.append(protocolObj) + return [protocol], [] + + service.currentProject._getSubworkflow = fakeGetSubworkflow + + cleanupCalls = [] + + monkeypatch.setattr( + service, + "_deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql", + lambda mapper, projectId, protocols: cleanupCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocols": protocols, + }) or { + "protocolsCount": len(protocols), + "setsDeleted": 0, + "objectsDeleted": 0, + "items": [], + }, + ) + + result = service.resetProtocolFrom( + mapper=mapper, + projectId=1, + protocolId=500, + ) + + assertSuccessEnvelope(result) + assert subworkflowCalls == [protocol] + assert cleanupCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocols": [protocol], + } + ] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_GetNextProtocolSuggestionsResolvesPostgresqlProtocolId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + protocol = FakeProtocol(objId=10, className="ProtImportMovies") + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + calledUrls = [] + + class FakeResponse: + def read(self): + return json.dumps([ + [ + "ProtLowerScore", + 3, + "Lower score protocol", + "scipion-em-lower", + "Lower score help", + ], + [ + "ProtHigherScore", + 9, + "Higher score protocol", + "scipion-em-higher", + "Higher score help", + ], + ]).encode("utf-8") + + def fakeUrlopen(url): + calledUrls.append(url) + return FakeResponse() + + class FakeConfig: + SCIPION_STATS_SUGGESTION = "https://example.test/suggestions/%s" + + @staticmethod + def getDomain(): + return FakeDomain({}) + + monkeypatch.setattr(projectServiceModule, "Config", FakeConfig) + monkeypatch.setattr(projectServiceModule, "urlopen", fakeUrlopen) + + result = service.getNextProtocolSuggestions( + mapper=mapper, + projectId=1, + protocolId=500, + ) + + assert calledUrls == [ + "https://example.test/suggestions/ProtImportMovies", + ] + + assert result == [ + { + "protocolName": "Higher score protocol", + "protocolClass": "ProtHigherScore", + "help": "Higher score help", + "installed": "Missing. Available in scipion-em-higher plugin.", + }, + { + "protocolName": "Lower score protocol", + "protocolClass": "ProtLowerScore", + "help": "Lower score help", + "installed": "Missing. Available in scipion-em-lower plugin.", + }, + ] + + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_GetProtocolParamsResolvesPostgresqlProtocolId(service, mapper, monkeypatch): + protocol = FakeProtocol(objId=10, className="ProtClass") + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + buildContextCalls = [] + + def fakeBuildProtocolContext(projectId, protocolObj): + buildContextCalls.append({ + "projectId": projectId, + "protocol": protocolObj, + }) + return { + "info": { + "projectId": projectId, + "protocolId": protocolObj.getObjId(), + "protocolClassName": "ProtClass", + }, + "form": { + "sections": [], + }, + "values": {}, + } + + monkeypatch.setattr(service, "_buildProtocolContext", fakeBuildProtocolContext) + + result = service.getProtocolParams( + mapper=mapper, + projectId=1, + protocolId=500, + ) + + assert result == { + "info": { + "projectId": 1, + "protocolId": 10, + "protocolClassName": "ProtClass", + }, + "form": { + "sections": [], + }, + "values": {}, + } + + assert service.currentProject.fixedProtocolParams == [protocol] + assert buildContextCalls == [ + { + "projectId": 1, + "protocol": protocol, + } + ] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + +def test_SaveProtocolResolvesPostgresqlProtocolIdForExistingProtocol( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) + monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) + monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) + monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) + + protocol = FakeProtocol(objId=10, className="ProtClass") + protocol.addParam("runName", FakeStringParam(label="Run name")) + protocol.addParam("iterations", FakeIntParam(label="Iterations")) + + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + monkeypatch.setattr( + service, + "applyParamsToProtocol", + lambda mapper=None, projectId=None, protocol=None, params=None: [], + ) + + savedProtocol, errors = service.saveProtocol( + mapper=mapper, + projectId=1, + protocolId=500, + protocolClassName="ProtClass", + params={ + "runName": "Edited protocol", + "iterations": "7", + }, + setToSave=False, + ) + + assert errors == [] + assert savedProtocol is protocol + assert protocol.runName.get() == "Edited protocol" + assert protocol.attributeValues["iterations"] == 7 + assert service.currentProject.storedProtocols == [protocol] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_LaunchProtocolLaunchResolvesPostgresqlProtocolId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) + monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) + monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) + monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) + monkeypatch.setattr(projectServiceModule, "MODE_RESUME", "resume-mode") + + protocol = FakeProtocol(objId=10, className="ProtClass", validateErrors=[]) + protocol.addParam("runName", FakeStringParam(label="Run name")) + protocol.addParam("iterations", FakeIntParam(label="Iterations")) + + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + monkeypatch.setattr( + service, + "applyParamsToProtocol", + lambda mapper=None, projectId=None, protocol=None, params=None: [], + ) + + service.launchProtocol( + mapper=mapper, + projectId=1, + protocolId=500, + protocolClassName="ProtClass", + params={ + "runName": "Launch protocol", + "iterations": "7", + }, + executeMode="launch", + ) + + assert protocol.runName.get() == "Launch protocol" + assert protocol.attributeValues["iterations"] == 7 + assert protocol.runMode.get() == "resume-mode" + assert service.currentProject.storedProtocols == [protocol] + assert service.currentProject.launchedProtocols == [protocol] + assert service.currentProject.scheduledProtocols == [] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_LaunchProtocolRestartResolvesPostgresqlProtocolId( + projectServiceModule, + service, + mapper, + monkeypatch, + ): + monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) + monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) + monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) + monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) + monkeypatch.setattr(projectServiceModule, "MODE_RESTART", "restart-mode") + + protocol = FakeProtocol(objId=10, className="ProtClass", validateErrors=[]) + protocol.addParam("runName", FakeStringParam(label="Run name")) + protocol.addParam("iterations", FakeIntParam(label="Iterations")) + + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + monkeypatch.setattr( + service, + "applyParamsToProtocol", + lambda mapper=None, projectId=None, protocol=None, params=None: [], + ) + + cleanupCalls = [] + + monkeypatch.setattr( + service, + "_deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql", + lambda mapper, projectId, protocols: cleanupCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocols": protocols, + }) or { + "protocolsCount": len(protocols), + "setsDeleted": 0, + "objectsDeleted": 0, + "items": [], + }, + ) + + service.launchProtocol( + mapper=mapper, + projectId=1, + protocolId=500, + protocolClassName="ProtClass", + params={ + "runName": "Restart protocol", + "iterations": "9", + }, + executeMode="restart", + ) + + assert protocol.runName.get() == "Restart protocol" + assert protocol.attributeValues["iterations"] == 9 + assert protocol.runMode.get() == "restart-mode" + assert service.currentProject.storedProtocols == [protocol] + assert service.currentProject.launchedProtocols == [protocol] + assert service.currentProject.scheduledProtocols == [] + assert cleanupCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocols": [protocol], + } + ] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + +def test_LaunchProtocolScheduleResolvesPostgresqlProtocolId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) + monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) + monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) + monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) + + protocol = FakeProtocol(objId=10, className="ProtClass", validateErrors=[]) + protocol.addParam("runName", FakeStringParam(label="Run name")) + protocol.addParam("iterations", FakeIntParam(label="Iterations")) + + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + monkeypatch.setattr( + service, + "applyParamsToProtocol", + lambda mapper=None, projectId=None, protocol=None, params=None: [], + ) + + service.launchProtocol( + mapper=mapper, + projectId=1, + protocolId=500, + protocolClassName="ProtClass", + params={ + "runName": "Schedule protocol", + "iterations": "11", + }, + executeMode="schedule", + ) + + assert protocol.runName.get() == "Schedule protocol" + assert protocol.attributeValues["iterations"] == 11 + assert protocol.runMode.get() is None + assert service.currentProject.storedProtocols == [protocol] + assert service.currentProject.scheduledProtocols == [protocol] + assert service.currentProject.launchedProtocols == [] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_ExportWorkflowProtocolsResolvesPostgresqlProtocolIds(service, mapper): + protocolA = FakeProtocol(objId=10, className="ProtA") + protocolB = FakeProtocol(objId=11, className="ProtB") + + service.currentProject.protocols[10] = protocolA + service.currentProject.protocols[11] = protocolB + + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.db.runtimeProtocolIdByDbId[501] = 11 + + exportedProtocolLists = [] + + def fakeGetProtocolsJson(protocolList): + exportedProtocolLists.append(protocolList) + return [ + { + "protocol": "exported-a", + }, + { + "protocol": "exported-b", + }, + ] + + service.currentProject.getProtocolsJson = fakeGetProtocolsJson + + class FakeExportPayload: + includeUpstream = False + protocolIds = ["500", "501"] + + result = service.exportWorkflowProtocolsService( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=FakeExportPayload(), + ) + + assert exportedProtocolLists == [[protocolA, protocolB]] + assert result["sourceProjectId"] == 1 + assert result["protocolIds"] == ["500", "501"] + assert result["workflow"] == [ + { + "protocol": "exported-a", + }, + { + "protocol": "exported-b", + }, + ] + + assert result["scipionWeb"]["sourceProjectId"] == 1 + assert result["scipionWeb"]["sourceProtocolIds"] == ["500", "501"] + assert result["scipionWeb"]["protocolPlugins"][0]["protocolId"] == "10" + assert result["scipionWeb"]["protocolPlugins"][1]["protocolId"] == "11" + + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + assert mapper.db.fetchOneCalls[1]["params"] == (1, 501, "501") + + +def test_ExportWorkflowProtocolsRaisesWhenPostgresqlProtocolIdCannotBeResolved(service, mapper): + protocol = FakeProtocol(objId=10, className="ProtA") + service.currentProject.protocols[10] = protocol + + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + class FakeExportPayload: + includeUpstream = False + protocolIds = ["500", "999"] + + with pytest.raises(HTTPException) as exc: + service.exportWorkflowProtocolsService( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=FakeExportPayload(), + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Protocol(s) not found: 999" + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + assert mapper.db.fetchOneCalls[1]["params"] == (1, 999, "999") + + +def test_ImportWorkflowProtocolsSanitizesExternalReferences(service, mapper, monkeypatch): + loadedJsonPayloads = [] + + def fakeLoadProtocols(jsonStr): + loadedJsonPayloads.append(json.loads(jsonStr)) + return None + + service.currentProject.loadProtocols = fakeLoadProtocols + + workflowIds = [ + {"10"}, + {"10", "20", "21"}, + ] + + monkeypatch.setattr( + service, + "_getCurrentWorkflowProtocolIds", + lambda: workflowIds.pop(0), + ) + + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + lambda mapper, projectId, refresh=False, checkPid=False: { + "protocols": 3, + "dependencies": 2, + }, + ) + + class FakeImportPayload: + mode = "append" + sourceProjectId = 999 + workflow = [ + { + "object.id": "1", + "inputFromCopiedProtocol": "1.outputParticles", + "inputFromExternalProtocol": "99.outputParticles", + "params": { + "validPointer": "2.outputVolume", + "invalidPointer": "100.outputCoordinates", + }, + }, + { + "object.id": "2", + "inputFromFirstProtocol": "1.outputParticles", + }, + ] + + result = service.importWorkflowProtocolsService( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=FakeImportPayload(), + ) + + assert result == { + "status": 0, + "errors": [], + "workflow": [], + "created": [ + {"newId": "20"}, + {"newId": "21"}, + ], + "protocolsCount": 3, + "dependenciesCount": 2, + } + + assert loadedJsonPayloads == [ + [ + { + "object.id": "1", + "inputFromCopiedProtocol": "1.outputParticles", + "params": { + "validPointer": "2.outputVolume", + }, + }, + { + "object.id": "2", + "inputFromFirstProtocol": "1.outputParticles", + }, + ] + ] + + +def test_ImportWorkflowProtocolsKeepsReferencesForSameProject(service, mapper, monkeypatch): + loadedJsonPayloads = [] + + def fakeLoadProtocols(jsonStr): + loadedJsonPayloads.append(json.loads(jsonStr)) + return None + + service.currentProject.loadProtocols = fakeLoadProtocols + + workflowIds = [ + {"10"}, + {"10", "20"}, + ] + + monkeypatch.setattr( + service, + "_getCurrentWorkflowProtocolIds", + lambda: workflowIds.pop(0), + ) + + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + lambda mapper, projectId, refresh=False, checkPid=False: { + "protocols": 2, + "dependencies": 1, + }, + ) + + class FakeImportPayload: + mode = "append" + sourceProjectId = 1 + workflow = [ + { + "object.id": "1", + "inputFromExistingSameProjectProtocol": "99.outputParticles", + "inputFromCopiedProtocol": "1.outputParticles", + }, + ] + + result = service.importWorkflowProtocolsService( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=FakeImportPayload(), + ) + + assert result == { + "status": 0, + "errors": [], + "workflow": [], + "created": [ + {"newId": "20"}, + ], + "protocolsCount": 2, + "dependenciesCount": 1, + } + + assert loadedJsonPayloads == [ + [ + { + "object.id": "1", + "inputFromExistingSameProjectProtocol": "99.outputParticles", + "inputFromCopiedProtocol": "1.outputParticles", + }, + ] + ] + + +def test_ApplyParamsToProtocolResolvesPostgresqlPointerParentId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "PointerParam", FakePointerParam) + monkeypatch.setattr(projectServiceModule, "MultiPointerParam", FakeMultiPointerParam) + monkeypatch.setattr(projectServiceModule, "RelationParam", FakeRelationParam) + + parentProtocol = FakeProtocol(objId=10, className="ParentProtocol") + parentProtocol.outputParticles = object() + + protocol = FakeProtocol(objId=20, className="ChildProtocol") + pointerParam = FakePointerParam(label="Input particles") + protocol.addParam("inputParticles", pointerParam) + protocol.inputParticles = FakePointerAttribute() + + service.currentProject.protocols[10] = parentProtocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + errors = service.applyParamsToProtocol( + mapper=mapper, + projectId=1, + protocol=protocol, + params={ + "inputParticles": "500.outputParticles", + }, + ) + + assert errors == [] + assert pointerParam.get() == "10.outputParticles" + assert pointerParam.default.get() == "10.outputParticles" + assert protocol.attributeValues["inputParticles"] is parentProtocol + assert protocol.inputParticles.extended == "outputParticles" + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + +def test_SetPointerParamResolvesPostgresqlPointerParentId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "PointerParam", FakePointerParam) + + parentProtocol = FakeProtocol(objId=10, className="ParentProtocol") + protocol = FakeProtocol(objId=20, className="ChildProtocol") + + pointerParam = FakePointerParam(label="Input volume") + protocol.addParam("inputVolume", pointerParam) + + service.currentProject.protocols[10] = parentProtocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + service.setPointerParam( + mapper=mapper, + projectId=1, + protocol=protocol, + key="inputVolume", + value={ + "editableValue": "500.outputVolume", + }, + parentId=500, + ) + + assert pointerParam.get() == "10.outputVolume" + assert pointerParam.default.get() == "10.outputVolume" + assert protocol.attributeValues["inputVolume"] is parentProtocol + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + +def test_ApplyParamsToProtocolResolvesPostgresqlMultiPointerParentIds( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "PointerParam", FakePointerParam) + monkeypatch.setattr(projectServiceModule, "MultiPointerParam", FakeMultiPointerParam) + monkeypatch.setattr(projectServiceModule, "RelationParam", FakeRelationParam) + monkeypatch.setattr(projectServiceModule, "PointerList", FakePointerList) + monkeypatch.setattr(projectServiceModule, "Pointer", FakePointer) + + parentProtocolA = FakeProtocol(objId=10, className="ParentProtocolA") + parentProtocolB = FakeProtocol(objId=11, className="ParentProtocolB") + + protocol = FakeProtocol(objId=20, className="ChildProtocol") + multiPointerParam = FakeMultiPointerParam(label="Input sets") + protocol.addParam("inputSets", multiPointerParam) + + service.currentProject.protocols[10] = parentProtocolA + service.currentProject.protocols[11] = parentProtocolB + + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.db.runtimeProtocolIdByDbId[501] = 11 + + errors = service.applyParamsToProtocol( + mapper=mapper, + projectId=1, + protocol=protocol, + params={ + "inputSets": [ + "500.outputParticles", + "501.outputClasses", + ], + }, + ) + + assert errors == [] + assert len(protocol.attributeValues["inputSets"]) == 2 + assert protocol.attributeValues["inputSets"][0].protocol is parentProtocolA + assert protocol.attributeValues["inputSets"][0].extended == "outputParticles" + assert protocol.attributeValues["inputSets"][1].protocol is parentProtocolB + assert protocol.attributeValues["inputSets"][1].extended == "outputClasses" + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + assert mapper.db.fetchOneCalls[1]["params"] == (1, 501, "501") + + +def test_GetNewProtocolParamsCacheIsScopedByProject( + projectServiceModule, + service, + monkeypatch, +): + projectServiceModule._newProtocolCache.clear() + + class FakeProtocolClass: + pass + + service.currentProject.protocolFactories = { + "ProtClass": FakeProtocolClass, + } + + buildCalls = [] + + def fakeBuildProtocolContext(projectId, protocol): + buildCalls.append(projectId) + return { + "info": { + "projectId": projectId, + "protocolClassName": "ProtClass", + }, + "form": {}, + "values": {}, + } + + monkeypatch.setattr( + service, + "_buildProtocolContext", + fakeBuildProtocolContext, + ) + + first = service.getNewProtocolParams(1, "ProtClass") + second = service.getNewProtocolParams(2, "ProtClass") + + assert first["info"]["projectId"] == 1 + assert second["info"]["projectId"] == 2 + assert buildCalls == [1, 2] + +def test_DuplicateProtocolWrapsCopyErrorAsHttpException(service, mapper): + protocol = FakeProtocol(objId=10) + service.currentProject.protocols[10] = protocol + + def fakeCopyProtocol(protocols): + raise RuntimeError("copy failed") + + service.currentProject.copyProtocol = fakeCopyProtocol + + class DuplicateItem: + def __init__(self, itemId): + self.id = itemId + + with pytest.raises(HTTPException) as exc: + service.duplicateProtocol( + mapper=mapper, + projectId=1, + protocols=[DuplicateItem("10")], + ) + + assert exc.value.status_code == 500 + assert exc.value.detail == "Failed to duplicate protocols: copy failed" \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_tags.py b/tests/unit/backend/api/services/test_project_service_tags.py new file mode 100644 index 00000000..2a290233 --- /dev/null +++ b/tests/unit/backend/api/services/test_project_service_tags.py @@ -0,0 +1,493 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +import importlib + +import pytest +from fastapi import HTTPException + + +class FakeDb: + def __init__(self): + self.fetchOneCalls = [] + self.protocolRows = [] + + def addProtocol(self, projectId, protocolDbId, protocolId): + self.protocolRows.append({ + "projectId": int(projectId), + "id": int(protocolDbId), + "protocolId": str(protocolId), + }) + + def fetchOne(self, query, params=None): + self.fetchOneCalls.append({ + "query": query, + "params": params, + }) + + if params is None: + return None + + queryText = " ".join(str(query).split()) + + if "FROM protocols" not in queryText: + return None + + # ProjectService._resolvePostgresqlProtocolDbId: + # SELECT id FROM protocols WHERE "projectId" = %s + # AND (id = %s OR "protocolId" = %s) + if len(params) == 3: + projectId, protocolDbIdCandidate, rawProtocolId = params + projectId = int(projectId) + rawProtocolId = str(rawProtocolId) + + for row in self.protocolRows: + if row["projectId"] != projectId: + continue + + if row["id"] == int(protocolDbIdCandidate): + return {"id": row["id"]} + + if row["protocolId"] == rawProtocolId: + return {"id": row["id"]} + + return None + + # ProjectService._resolvePostgresqlProtocolDbId for non-integer values: + # SELECT id FROM protocols WHERE "projectId" = %s + # AND "protocolId" = %s + if len(params) == 2 and '"projectId" = %s' in queryText and '"protocolId" = %s' in queryText: + projectId, rawProtocolId = params + projectId = int(projectId) + rawProtocolId = str(rawProtocolId) + + for row in self.protocolRows: + if row["projectId"] == projectId and row["protocolId"] == rawProtocolId: + return {"id": row["id"]} + + return None + + def getRuntimeProtocolId(self, projectId, protocolDbId): + for row in self.protocolRows: + if row["projectId"] == int(projectId) and row["id"] == int(protocolDbId): + return row["protocolId"] + + return str(protocolDbId) + + +class FakeMapper: + def __init__(self): + self.db = FakeDb() + self.protocolTagIdsByProtocolDbId = {} + self.getProtocolTagIdsCalls = [] + self.setProtocolTagIdsByProtocolDbIdCalls = [] + self.setProtocolTagIdsCalls = [] + self.raiseOnGetProtocolTagIds = None + self.raiseOnSetProtocolTagIdsByProtocolDbId = None + self.projectTags = [] + + def listProjectTags(self, projectId): + return list(self.projectTags) + + def getProtocolTagIds(self, projectId, protocolDbId): + self.getProtocolTagIdsCalls.append({ + "projectId": projectId, + "protocolDbId": protocolDbId, + }) + + if self.raiseOnGetProtocolTagIds is not None: + raise self.raiseOnGetProtocolTagIds + + return list(self.protocolTagIdsByProtocolDbId.get(int(protocolDbId), [])) + + def setProtocolTagIdsByProtocolDbId(self, projectId, protocolDbId, tagIds): + self.setProtocolTagIdsByProtocolDbIdCalls.append({ + "projectId": projectId, + "protocolDbId": protocolDbId, + "tagIds": list(tagIds or []), + }) + + if self.raiseOnSetProtocolTagIdsByProtocolDbId is not None: + raise self.raiseOnSetProtocolTagIdsByProtocolDbId + + cleanTagIds = sorted({ + str(tagId).strip() + for tagId in (tagIds or []) + if str(tagId).strip() + }) + + self.protocolTagIdsByProtocolDbId[int(protocolDbId)] = cleanTagIds + + return { + "protocolId": self.db.getRuntimeProtocolId(projectId, protocolDbId), + "protocolDbId": int(protocolDbId), + "tagIds": cleanTagIds, + } + + def setProtocolTagIds(self, projectId, protocolId, tagIds): + self.setProtocolTagIdsCalls.append({ + "projectId": projectId, + "protocolId": protocolId, + "tagIds": list(tagIds or []), + }) + + for row in self.db.protocolRows: + if row["projectId"] == int(projectId) and row["protocolId"] == str(protocolId): + return self.setProtocolTagIdsByProtocolDbId( + projectId=projectId, + protocolDbId=row["id"], + tagIds=tagIds, + ) + + raise Exception("Protocol not found in project") + + +class FakeMapperWithoutDbIdSetter(FakeMapper): + setProtocolTagIdsByProtocolDbId = None + + def setProtocolTagIds(self, projectId, protocolId, tagIds): + self.setProtocolTagIdsCalls.append({ + "projectId": projectId, + "protocolId": protocolId, + "tagIds": list(tagIds or []), + }) + + for row in self.db.protocolRows: + if row["projectId"] == int(projectId) and row["protocolId"] == str(protocolId): + cleanTagIds = sorted({ + str(tagId).strip() + for tagId in (tagIds or []) + if str(tagId).strip() + }) + + self.protocolTagIdsByProtocolDbId[int(row["id"])] = cleanTagIds + + return { + "protocolId": str(row["protocolId"]), + "protocolDbId": int(row["id"]), + "tagIds": cleanTagIds, + } + + raise Exception("Protocol not found in project") + + +@pytest.fixture +def projectServiceModule(authTestEnv): + return importlib.import_module("app.backend.api.services.project_service") + + +@pytest.fixture +def service(projectServiceModule): + return projectServiceModule.ProjectService() + + +@pytest.fixture +def mapper(): + return FakeMapper() + + +@pytest.fixture +def currentUser(): + return {"id": 1} + + +def test_ListProjectTagsUsesMapperProjectTags(service, mapper, currentUser): + mapper.projectTags = [ + { + "id": "good", + "title": "Good particles", + "description": None, + "color": "#00ff00", + }, + { + "id": "bad", + "title": "Bad particles", + "description": "Rejected items", + "color": "#ff0000", + }, + ] + + result = service.listProjectTags( + mapper=mapper, + projectId=1, + currentUser=currentUser, + ) + + assert result == mapper.projectTags + + +def test_ListProtocolTagsResolvesPostgresqlProtocolDbId( + service, + mapper, + currentUser, +): + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + mapper.protocolTagIdsByProtocolDbId[500] = ["good", "selected"] + + result = service.listProtocolTags( + mapper=mapper, + projectId=1, + protocolId=500, + currentUser=currentUser, + ) + + assert result == { + "protocolId": "500", + "protocolDbId": 500, + "tagIds": ["good", "selected"], + } + + assert mapper.getProtocolTagIdsCalls == [ + { + "projectId": 1, + "protocolDbId": 500, + } + ] + + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_ListProtocolTagsAlsoAcceptsRuntimeProtocolId( + service, + mapper, + currentUser, +): + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + mapper.protocolTagIdsByProtocolDbId[500] = ["movie-alignment"] + + result = service.listProtocolTags( + mapper=mapper, + projectId=1, + protocolId=10, + currentUser=currentUser, + ) + + assert result == { + "protocolId": "10", + "protocolDbId": 500, + "tagIds": ["movie-alignment"], + } + + assert mapper.getProtocolTagIdsCalls == [ + { + "projectId": 1, + "protocolDbId": 500, + } + ] + + assert mapper.db.fetchOneCalls[0]["params"] == (1, 10, "10") + + +def test_ListProtocolTagsRaises404WhenProtocolCannotBeResolved( + service, + mapper, + currentUser, +): + with pytest.raises(HTTPException) as exc: + service.listProtocolTags( + mapper=mapper, + projectId=1, + protocolId=999, + currentUser=currentUser, + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Protocol not found in PostgreSQL: 999" + assert mapper.getProtocolTagIdsCalls == [] + + +def test_ListProtocolTagsWrapsMapperErrors( + service, + mapper, + currentUser, +): + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + mapper.raiseOnGetProtocolTagIds = RuntimeError("database error") + + with pytest.raises(HTTPException) as exc: + service.listProtocolTags( + mapper=mapper, + projectId=1, + protocolId=500, + currentUser=currentUser, + ) + + assert exc.value.status_code == 500 + assert exc.value.detail == "Failed to list protocol tags: database error" + + assert mapper.getProtocolTagIdsCalls == [ + { + "projectId": 1, + "protocolDbId": 500, + } + ] + + +def test_SetProtocolTagsResolvesPostgresqlProtocolDbId( + service, + mapper, + currentUser, +): + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + + result = service.setProtocolTags( + mapper=mapper, + projectId=1, + protocolId=500, + tagIds=[" selected ", "good", "good", ""], + currentUser=currentUser, + ) + + assert result == { + "protocolId": "10", + "protocolDbId": 500, + "tagIds": ["good", "selected"], + } + + assert mapper.setProtocolTagIdsByProtocolDbIdCalls == [ + { + "projectId": 1, + "protocolDbId": 500, + "tagIds": [" selected ", "good", "good", ""], + } + ] + + assert mapper.setProtocolTagIdsCalls == [] + assert mapper.protocolTagIdsByProtocolDbId[500] == ["good", "selected"] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_SetProtocolTagsAlsoAcceptsRuntimeProtocolId( + service, + mapper, + currentUser, +): + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + + result = service.setProtocolTags( + mapper=mapper, + projectId=1, + protocolId=10, + tagIds=["movie-alignment"], + currentUser=currentUser, + ) + + assert result == { + "protocolId": "10", + "protocolDbId": 500, + "tagIds": ["movie-alignment"], + } + + assert mapper.setProtocolTagIdsByProtocolDbIdCalls == [ + { + "projectId": 1, + "protocolDbId": 500, + "tagIds": ["movie-alignment"], + } + ] + + assert mapper.setProtocolTagIdsCalls == [] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 10, "10") + + +def test_SetProtocolTagsRaises404WhenProtocolCannotBeResolved( + service, + mapper, + currentUser, +): + with pytest.raises(HTTPException) as exc: + service.setProtocolTags( + mapper=mapper, + projectId=1, + protocolId=999, + tagIds=["good"], + currentUser=currentUser, + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Protocol not found in PostgreSQL: 999" + assert mapper.setProtocolTagIdsByProtocolDbIdCalls == [] + assert mapper.setProtocolTagIdsCalls == [] + + +def test_SetProtocolTagsWrapsMapperErrors( + service, + mapper, + currentUser, +): + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + mapper.raiseOnSetProtocolTagIdsByProtocolDbId = RuntimeError("write failed") + + with pytest.raises(HTTPException) as exc: + service.setProtocolTags( + mapper=mapper, + projectId=1, + protocolId=500, + tagIds=["good"], + currentUser=currentUser, + ) + + assert exc.value.status_code == 500 + assert exc.value.detail == "Failed to set protocol tags: write failed" + + assert mapper.setProtocolTagIdsByProtocolDbIdCalls == [ + { + "projectId": 1, + "protocolDbId": 500, + "tagIds": ["good"], + } + ] + + +def test_SetProtocolTagsFallsBackToRuntimeSetterWhenDbIdSetterIsMissing( + projectServiceModule, + currentUser, +): + service = projectServiceModule.ProjectService() + mapper = FakeMapperWithoutDbIdSetter() + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + + result = service.setProtocolTags( + mapper=mapper, + projectId=1, + protocolId=10, + tagIds=["good"], + currentUser=currentUser, + ) + + assert result == { + "protocolId": "10", + "protocolDbId": 500, + "tagIds": ["good"], + } + + assert mapper.setProtocolTagIdsCalls == [ + { + "projectId": 1, + "protocolId": 10, + "tagIds": ["good"], + } + ] \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_tiltseries.py b/tests/unit/backend/api/services/test_project_service_tiltseries.py index ef4cd9fa..ebde0ca5 100644 --- a/tests/unit/backend/api/services/test_project_service_tiltseries.py +++ b/tests/unit/backend/api/services/test_project_service_tiltseries.py @@ -281,6 +281,154 @@ def service(projectServiceModule): return instance +def test_GetPostgresqlTiltSeriesReaderIfAvailableUsesResolvedProtocolDbId( + service, + monkeypatch, +): + createdReaders = [] + + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakePostgresqlTiltSeriesReader: + # fakePostgresqlTiltSeriesReader + def __init__(self, db, projectId, protocolId, outputName): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + createdReaders.append(self) + + def hasOutput(self): + return True + + readerModule = importlib.import_module( + "app.backend.viewers.postgresql_tiltseries_reader" + ) + + monkeypatch.setattr( + readerModule, + "PostgresqlTiltSeriesReader", + FakePostgresqlTiltSeriesReader, + ) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 321, + ) + + mapper = FakeMapper() + + reader = service._getPostgresqlTiltSeriesReaderIfAvailable( + mapper=mapper, + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + ) + + assert reader is createdReaders[0] + assert createdReaders[0].db is mapper.db + assert createdReaders[0].projectId == 1 + assert createdReaders[0].protocolId == 321 + assert createdReaders[0].outputName == "outputTiltSeries" + +def test_ListOutputTiltSeriesServiceRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + monkeypatch.setattr( + service, + "_getPostgresqlTiltSeriesReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy TiltSeries fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForTiltSeries", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service.listOutputTiltSeriesService( + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "TiltSeries output is not available in PostgreSQL metadata" in exc.value.detail + assert "reader_not_available" in exc.value.detail + + +def test_GetTiltSeriesFramesServiceRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + monkeypatch.setattr( + service, + "_getPostgresqlTiltSeriesReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy TiltSeries fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForTiltSeries", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service.getTiltSeriesFramesService( + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + tiltSeriesId="TS_001", + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "TiltSeries frames output is not available in PostgreSQL metadata" in exc.value.detail + assert "reader_not_available" in exc.value.detail + + +def test_RenderTiltSeriesImageServiceRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + monkeypatch.setattr( + service, + "_getPostgresqlTiltSeriesReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy TiltSeries image fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForTiltSeries", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service.renderTiltSeriesImageService( + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + tiltSeriesId="TS_001", + index=0, + size=512, + fmt="png", + applyTransform=True, + inline=True, + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "TiltSeries image output is not available in PostgreSQL metadata" in exc.value.detail + assert "reader_not_available" in exc.value.detail + + def test_ListOutputTiltSeriesServiceBuildsSummaries(service): ts1 = FakeTiltSeries( tsId="TS_001", @@ -495,6 +643,143 @@ def create(projectPath, suffix): assert createdOutputSet._dim == [128, 128, 40] +def test_CreateNewSetOfTiltSeriesServiceStoresSetWithResolvedProtocolDbId( + projectServiceModule, + service, + monkeypatch, +): + storedCalls = [] + + class FakeDb: + # fakeDb + def fetchOne(self, *args, **kwargs): + return None + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakeScipionSetPostgresqlMapper: + # fakeScipionSetPostgresqlMapper + def __init__(self, db): + self.db = db + + def storeSet(self, projectId, protocolDbId, outputName, scipionSet): + storedCalls.append( + { + "projectId": projectId, + "protocolDbId": protocolDbId, + "outputName": outputName, + "scipionSet": scipionSet, + } + ) + return { + "stored": True, + "protocolDbId": protocolDbId, + "outputName": outputName, + } + + class FakeCreatedTiltSeries: + # fakeCreatedTiltSeries + def __init__(self): + self._items = [] + self._dim = None + self._anglesCount = None + self._written = False + + def copyInfo(self, tiltSeries): + self._copiedInfoFrom = tiltSeries + + def append(self, item): + self._items.append(item) + + def setEnabled(self, value): + self._enabled = value + + def getSize(self): + return len(self._items) + + def setDim(self, dim): + self._dim = dim + + def setAnglesCount(self, count): + self._anglesCount = count + + def write(self): + self._written = True + + class FakeSetOfTiltSeriesFactory: + # fakeSetOfTiltSeriesFactory + @staticmethod + def create(projectPath, suffix): + return createdOutputSet + + createdOutputSet = FakeCreatedTiltSeriesOutputSet() + createdOutputSet.update = lambda item: None + createdOutputSet.remove = lambda item: None + + tiltSeries = FakeTiltSeries( + tsId="TS_001", + size=0, + dims=[128, 128, 40], + samplingRate=1.5, + tiltAxisAngle=90.0, + items=[], + ) + inputSet = FakeTiltSeriesSet( + items=[tiltSeries], + hasOddEven=False, + dims=[128, 128, 40], + ) + protocol = FakeProtocol("outputTiltSeries", inputSet) + service.currentProject = FakeCurrentProject(protocol) + + scipionSetMapperModule = importlib.import_module( + "app.backend.mapper.scipion_set_mapper" + ) + + monkeypatch.setattr( + scipionSetMapperModule, + "ScipionSetPostgresqlMapper", + FakeScipionSetPostgresqlMapper, + ) + monkeypatch.setattr(projectServiceModule, "SetOfTiltSeries", FakeSetOfTiltSeriesFactory) + monkeypatch.setattr(projectServiceModule, "TiltSeries", FakeCreatedTiltSeries) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 321, + ) + + mapper = FakeMapper() + + result = service.createNewSetOfTiltSeriesService( + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + exclusions={}, + restack=False, + mapper=mapper, + ) + + assert result["status"] == 0 + assert result["outputName"] == "TiltSeries_0" + assert result["postgresqlSync"] == { + "stored": True, + "protocolDbId": 321, + "outputName": "TiltSeries_0", + } + assert result["postgresqlError"] is None + assert storedCalls == [ + { + "projectId": 1, + "protocolDbId": 321, + "outputName": "TiltSeries_0", + "scipionSet": createdOutputSet, + } + ] + def test_ResolveOutputForTiltSeriesReturns404WhenProtocolMissing(service): class BrokenCurrentProject: # brokenCurrentProject @@ -507,4 +792,4 @@ def getProtocol(self, protocolId): service._resolveOutputForTiltSeries(10, "outputTiltSeries") assert exc.value.status_code == 404 - assert exc.value.detail == "Protocol not found" \ No newline at end of file + assert str(exc.value.detail).startswith("Protocol not found in Scipion runtime:") \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_volumes.py b/tests/unit/backend/api/services/test_project_service_volumes.py index d5d08d98..b165d2bd 100644 --- a/tests/unit/backend/api/services/test_project_service_volumes.py +++ b/tests/unit/backend/api/services/test_project_service_volumes.py @@ -174,6 +174,90 @@ def test_ResolveOutputForVolumesReturnsExactOutput(service): assert resolvedProtocol is protocol assert resolvedOutput is volume +def test_GetPostgresqlVolumeReaderIfAvailableUsesResolvedProtocolDbId( + service, + monkeypatch, +): + createdCoordsVolumeReaders = [] + createdVolumeReaders = [] + + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakePostgresqlCoords3dTomogramVolumeReader: + # fakePostgresqlCoords3dTomogramVolumeReader + def __init__(self, db, projectId, protocolId, outputName): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + createdCoordsVolumeReaders.append(self) + + def hasOutput(self): + return False + + class FakePostgresqlVolumeReader: + # fakePostgresqlVolumeReader + def __init__(self, db, projectId, protocolId, outputName): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + createdVolumeReaders.append(self) + + def hasOutput(self): + return True + + coordsVolumeModule = importlib.import_module( + "app.backend.viewers.postgresql_coords3d_tomogram_volume_reader" + ) + volumeModule = importlib.import_module( + "app.backend.viewers.postgresql_volume_reader" + ) + + monkeypatch.setattr( + coordsVolumeModule, + "PostgresqlCoords3dTomogramVolumeReader", + FakePostgresqlCoords3dTomogramVolumeReader, + ) + monkeypatch.setattr( + volumeModule, + "PostgresqlVolumeReader", + FakePostgresqlVolumeReader, + ) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 741, + ) + + mapper = FakeMapper() + + reader = service._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=1, + protocolId=10, + outputName="outputVolumes", + ) + + assert reader is createdVolumeReaders[0] + + assert createdCoordsVolumeReaders[0].db is mapper.db + assert createdCoordsVolumeReaders[0].projectId == 1 + assert createdCoordsVolumeReaders[0].protocolId == 741 + assert createdCoordsVolumeReaders[0].outputName == "outputVolumes" + + assert createdVolumeReaders[0].db is mapper.db + assert createdVolumeReaders[0].projectId == 1 + assert createdVolumeReaders[0].protocolId == 741 + assert createdVolumeReaders[0].outputName == "outputVolumes" + def test_ResolveOutputForVolumesSupportsAliasFallback(service): volume = FakeVolumeOutput("/tmp/volume.mrc") @@ -185,6 +269,107 @@ def test_ResolveOutputForVolumesSupportsAliasFallback(service): assert resolvedProtocol is protocol assert resolvedOutput is volume +@pytest.mark.parametrize( + "serviceCall, expectedDetail", + [ + ( + lambda service, mapper: service.listOutputVolumesService( + projectId=1, + protocolId=10, + outputName="outputVolumes", + mapper=mapper, + ), + "Volume output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.getVolumeInfoService( + projectId=1, + protocolId=10, + outputName="outputVolumes", + volumeId=0, + mapper=mapper, + ), + "Volume output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.getVolumeHistogramService( + projectId=1, + protocolId=10, + outputName="outputVolumes", + volumeId=0, + bins=32, + mapper=mapper, + ), + "Volume histogram output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.renderVolumeSliceService( + projectId=1, + protocolId=10, + outputName="outputVolumes", + volumeId=0, + sliceIndex=0, + axis="z", + colormap=None, + normalize="minmax", + scale=1.0, + inline=True, + mapper=mapper, + ), + "Volume slice output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.getVolumeData3dService( + projectId=1, + protocolId=10, + outputName="outputVolumes", + volumeId=0, + maxDim=32, + method="binning", + mapper=mapper, + ), + "Volume 3D data output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.getVolumeSurfaceMesh( + projectId=1, + protocolId=10, + outputName="outputVolumes", + volumeId=0, + level=0.1, + maxDim=32, + method="binning", + maxTriangles=1000, + currentUser={"id": 1}, + mapper=mapper, + ), + "Volume surface mesh output is not available in PostgreSQL metadata", + ), + ], +) +def test_VolumeServicesRequirePostgresqlWhenMapperIsPresent( + service, + monkeypatch, + serviceCall, + expectedDetail, +): + monkeypatch.setattr( + service, + "_getPostgresqlVolumeReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy volume fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForVolumes", failRuntimeFallback) + + with pytest.raises(HTTPException) as exc: + serviceCall(service, object()) + + assert exc.value.status_code == 404 + assert expectedDetail in exc.value.detail + assert "reader_not_available" in exc.value.detail def test_ResolveOutputForVolumesReturns404WhenProtocolMissing(service): service.currentProject = FakeCurrentProject(protocolError=RuntimeError("missing")) @@ -193,7 +378,7 @@ def test_ResolveOutputForVolumesReturns404WhenProtocolMissing(service): service._resolveOutputForVolumes(10, "outputVolumes") assert exc.value.status_code == 404 - assert exc.value.detail == "Protocol not found" + assert str(exc.value.detail).startswith("Protocol not found in Scipion runtime: 10") def test_ListOutputVolumesServiceDelegatesToOutputsPreview(projectServiceModule, service, monkeypatch): diff --git a/tests/unit/backend/api/services/test_protocol_wizard_service.py b/tests/unit/backend/api/services/test_protocol_wizard_service.py new file mode 100644 index 00000000..3fd298ec --- /dev/null +++ b/tests/unit/backend/api/services/test_protocol_wizard_service.py @@ -0,0 +1,501 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +import importlib + +import pytest +from fastapi import HTTPException + + +class FakeValueHolder: + def __init__(self, value=None): + self.value = value + + def get(self): + return self.value + + def set(self, value): + self.value = value + + +class FakeParam: + def __init__(self, label="Param", validationErrors=None): + self.label = FakeValueHolder(label) + self.validationErrors = validationErrors or [] + self.value = None + + def get(self): + return self.value + + def set(self, value): + self.value = value + + def validate(self, value): + return list(self.validationErrors) + + +class FakeProtocol: + def __init__(self, objId=None, className="ProtClass"): + self._objId = objId + self._className = className + self._params = {} + self.attributeValues = {} + self.label = None + + def addParam(self, name, param): + self._params[name] = param + + def getParam(self, name): + return self._params.get(name) + + def getObjId(self): + return self._objId + + def getClassName(self): + return self._className + + def setObjLabel(self, value): + self.label = value + + def setAttributeValue(self, name, value): + self.attributeValues[name] = value + + +class FakeProtocolFactory: + def __init__(self, protocol): + self.protocol = protocol + + def __call__(self): + return self.protocol + + +class FakeDomain: + def __init__(self, protocols=None, wizards=None): + self.protocols = protocols or {} + self.wizards = wizards or {} + + def getProtocols(self): + return self.protocols + + def getWizards(self): + return self.wizards + + +class FakeCurrentProject: + def __init__(self): + self.protocols = {} + self.protocolFactories = {} + self.fixedProtocolParams = [] + self.createdProtocols = [] + + def getDomain(self): + return FakeDomain(protocols=self.protocolFactories) + + def getProtocol(self, protocolId): + return self.protocols[int(protocolId)] + + def newProtocol(self, protClass): + protocol = protClass() + self.createdProtocols.append(protocol) + return protocol + + def _fixProtParamsConfiguration(self, protocol): + self.fixedProtocolParams.append(protocol) + + +class FakeMapper: + pass + + +class FakeProjectService: + def __init__(self, currentProject): + self.currentProject = currentProject + self.runtimeProtocolIdByDbId = {} + self.runtimeCalls = [] + self.castParamValueCalls = [] + self.applyParamsToProtocolCalls = [] + self.getProjectByIdCalls = [] + self.projectRow = {"id": 1} + + def _getScipionProtocolForRuntime(self, mapper, projectId, protocolId): + self.runtimeCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + }) + + runtimeProtocolId = self.runtimeProtocolIdByDbId.get(int(protocolId), int(protocolId)) + return self.currentProject.protocols[int(runtimeProtocolId)] + + def castParamValue(self, param, value): + self.castParamValueCalls.append({ + "param": param, + "value": value, + }) + + if isinstance(value, str) and value.isdigit(): + return int(value) + + return value + + def applyParamsToProtocol(self, mapper, projectId, protocol, params): + self.applyParamsToProtocolCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocol": protocol, + "params": dict(params or {}), + }) + return [] + + def getProjectById(self, mapper, projectId, currentUser, refresh=False, checkPid=False): + self.getProjectByIdCalls.append({ + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + "refresh": refresh, + "checkPid": checkPid, + }) + return self.projectRow + + +class FakePayload: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + +class FakeWizardClass: + pass + + +@pytest.fixture +def protocolWizardServiceModule(authTestEnv): + return importlib.import_module("app.backend.api.services.protocol_wizard_service") + + +@pytest.fixture +def currentProject(): + return FakeCurrentProject() + + +@pytest.fixture +def projectService(currentProject): + return FakeProjectService(currentProject) + + +@pytest.fixture +def wizardService(protocolWizardServiceModule, currentProject, projectService): + return protocolWizardServiceModule.ProtocolWizardService( + currentProject=currentProject, + projectService=projectService, + ) + + +@pytest.fixture +def mapper(): + return FakeMapper() + + +def test_SanitizeWizardFormValuesDropsEmptyValues(wizardService): + result = wizardService._sanitizeWizardFormValues({ + "noneValue": None, + "blankValue": "", + "spacesValue": " ", + "zeroValue": 0, + "falseValue": False, + "textValue": "value", + }) + + assert result == { + "zeroValue": 0, + "falseValue": False, + "textValue": "value", + } + + +def test_BuildWizardReadyProtocolResolvesPostgresqlProtocolId( + wizardService, + currentProject, + projectService, + mapper, +): + protocol = FakeProtocol(objId=10, className="ProtWizardTarget") + iterationsParam = FakeParam(label="Iterations") + protocol.addParam("iterations", iterationsParam) + + currentProject.protocols[10] = protocol + projectService.runtimeProtocolIdByDbId[500] = 10 + + result = wizardService._buildWizardReadyProtocol( + protocolId=500, + protocolClassName="ProtWizardTarget", + formValues={ + "iterations": "7", + }, + mapper=mapper, + projectId=1, + ) + + assert result is protocol + assert currentProject.fixedProtocolParams == [protocol] + assert iterationsParam.get() == 7 + assert protocol.attributeValues["iterations"] == 7 + + assert projectService.runtimeCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + + assert projectService.applyParamsToProtocolCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocol": protocol, + "params": { + "iterations": "7", + }, + } + ] + + +def test_BuildWizardReadyProtocolCreatesNewProtocolWhenProtocolIdIsMissing( + wizardService, + currentProject, + projectService, + mapper, +): + protocol = FakeProtocol(objId=None, className="ProtNewWizardTarget") + runNameParam = FakeParam(label="Run name") + protocol.addParam("runName", runNameParam) + + currentProject.protocolFactories["ProtNewWizardTarget"] = FakeProtocolFactory(protocol) + + result = wizardService._buildWizardReadyProtocol( + protocolId=None, + protocolClassName="ProtNewWizardTarget", + formValues={ + "runName": "New protocol", + }, + mapper=mapper, + projectId=1, + ) + + assert result is protocol + assert currentProject.createdProtocols == [protocol] + assert currentProject.fixedProtocolParams == [protocol] + assert runNameParam.get() == "New protocol" + assert protocol.attributeValues["runName"] == "New protocol" + assert protocol.label == "New protocol" + assert projectService.runtimeCalls == [] + + +def test_BuildWizardReadyProtocolRaisesWhenNewProtocolClassIsMissing( + wizardService, + mapper, +): + with pytest.raises(HTTPException) as exc: + wizardService._buildWizardReadyProtocol( + protocolId=None, + protocolClassName="MissingProtocol", + formValues={}, + mapper=mapper, + projectId=1, + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Protocol class 'MissingProtocol' not found" + + +def test_ExecuteProtocolWizardResolvesPostgresqlProtocolId( + protocolWizardServiceModule, + wizardService, + currentProject, + projectService, + mapper, + monkeypatch, +): + protocol = FakeProtocol(objId=10, className="ProtWizardTarget") + iterationsParam = FakeParam(label="Iterations") + protocol.addParam("iterations", iterationsParam) + + currentProject.protocols[10] = protocol + projectService.runtimeProtocolIdByDbId[500] = 10 + + descriptor = { + "id": "tests.FakeWizardClass", + "kind": "compute", + "targetParams": ["boxSize"], + } + + monkeypatch.setattr( + wizardService, + "_resolveWizardDescriptorForParam", + lambda protocol, paramName, wizardId: descriptor, + ) + monkeypatch.setattr( + wizardService, + "_getWizardClassById", + lambda wizardId: FakeWizardClass, + ) + + handlerCalls = [] + + def fakeExecuteWizardHandler( + kind, + wizardClass, + protocol, + paramName, + descriptor, + wizardInputs, + currentProject, + projectId, + ): + handlerCalls.append({ + "kind": kind, + "wizardClass": wizardClass, + "protocol": protocol, + "paramName": paramName, + "descriptor": descriptor, + "wizardInputs": wizardInputs, + "currentProject": currentProject, + "projectId": projectId, + }) + return { + "paramUpdates": { + "boxSize": 128, + }, + "message": "Box size computed", + "preview": { + "enabled": True, + }, + } + + monkeypatch.setattr( + protocolWizardServiceModule, + "executeWizardHandler", + fakeExecuteWizardHandler, + ) + + payload = FakePayload( + protocolId=500, + protocolClassName="ProtWizardTarget", + formValues={ + "iterations": "7", + }, + paramName="boxSize", + wizardId="tests.FakeWizardClass", + wizardInputs={ + "diameter": 180, + }, + ) + + result = wizardService.executeProtocolWizard( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=payload, + ) + + assert result == { + "success": True, + "wizardId": "tests.FakeWizardClass", + "kind": "compute", + "paramUpdates": { + "boxSize": 128, + }, + "message": "Box size computed", + "preview": { + "enabled": True, + }, + } + + assert projectService.getProjectByIdCalls == [ + { + "mapper": mapper, + "projectId": 1, + "currentUser": {"id": 1}, + "refresh": False, + "checkPid": False, + } + ] + + assert projectService.runtimeCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + + assert currentProject.fixedProtocolParams == [protocol] + assert iterationsParam.get() == 7 + assert protocol.attributeValues["iterations"] == 7 + + assert handlerCalls == [ + { + "kind": "compute", + "wizardClass": FakeWizardClass, + "protocol": protocol, + "paramName": "boxSize", + "descriptor": descriptor, + "wizardInputs": { + "diameter": 180, + }, + "currentProject": currentProject, + "projectId": 1, + } + ] + + +def test_ExecuteProtocolWizardReturns404WhenProjectDoesNotExist( + wizardService, + projectService, + mapper, +): + projectService.projectRow = None + + payload = FakePayload( + protocolId=500, + protocolClassName="ProtWizardTarget", + formValues={}, + paramName="boxSize", + wizardId="tests.FakeWizardClass", + wizardInputs={}, + ) + + with pytest.raises(HTTPException) as exc: + wizardService.executeProtocolWizard( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=payload, + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Project not found" \ No newline at end of file diff --git a/tests/unit/backend/viewers/__init__.py b/tests/unit/backend/viewers/__init__.py new file mode 100644 index 00000000..4cdb527e --- /dev/null +++ b/tests/unit/backend/viewers/__init__.py @@ -0,0 +1,25 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** diff --git a/tests/unit/backend/viewers/test_postgresql_coords2d_reader.py b/tests/unit/backend/viewers/test_postgresql_coords2d_reader.py new file mode 100644 index 00000000..06d97405 --- /dev/null +++ b/tests/unit/backend/viewers/test_postgresql_coords2d_reader.py @@ -0,0 +1,163 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from app.backend.viewers.postgresql_coords2d_reader import PostgresqlCoords2dReader + + +def make_reader(stored_set): + reader = PostgresqlCoords2dReader( + db=None, + projectId=1, + protocolId=2, + outputName="coordinates", + ) + reader._storedSet = stored_set + return reader + + +def test_PostgresqlCoords2dReaderListsMicrographs(): + reader = make_reader( + { + "setClassName": "SetOfCoordinates", + "itemClassName": "Coordinate", + "properties": {"boxSize": 128}, + "items": [ + { + "scipionItemId": 1, + "values": { + "_micId": 10, + "_x": 11.5, + "_y": 22.5, + }, + }, + { + "scipionItemId": 2, + "values": { + "_micId": 10, + "_x": 12.5, + "_y": 23.5, + }, + }, + { + "scipionItemId": 3, + "values": { + "_micId": 20, + "_x": 5.0, + "_y": 6.0, + }, + }, + ], + } + ) + + payload = reader.listMicrographs() + + assert payload == { + "micrographs": [ + { + "id": "10", + "index": 1, + "fileName": "", + "label": "Micrograph 10", + "particles": 2, + "updated": False, + "width": None, + "height": None, + "locationIndex": None, + "thumbnailUrl": None, + }, + { + "id": "20", + "index": 2, + "fileName": "", + "label": "Micrograph 20", + "particles": 1, + "updated": False, + "width": None, + "height": None, + "locationIndex": None, + "thumbnailUrl": None, + }, + ], + "totalMicrographs": 2, + "totalPicks": 3, + "boxSize": 128, + } + + +def test_PostgresqlCoords2dReaderListsCoordinatesForMicrograph(): + reader = make_reader( + { + "setClassName": "SetOfCoordinates", + "itemClassName": "Coordinate", + "items": [ + { + "scipionItemId": 1, + "values": { + "_micId": 10, + "_x": 11.5, + "_y": 22.5, + "_score": 0.9, + "_classId": 2, + }, + }, + { + "scipionItemId": 2, + "values": { + "_micId": 20, + "_x": 5.0, + "_y": 6.0, + }, + }, + ], + } + ) + + payload = reader.listCoordinatesForMicrograph("10") + + assert payload == { + "coordinates": [ + { + "id": 1, + "micId": "10", + "x": 11.5, + "y": 22.5, + "score": 0.9, + "classLabel": "2", + } + ] + } + + +def test_PostgresqlCoords2dReaderRejectsCoords3dStoredSet(): + reader = make_reader( + { + "setClassName": "SetOfCoordinates3D", + "itemClassName": "Coordinate3D", + "items": [], + } + ) + + assert reader.hasOutput() is False \ No newline at end of file diff --git a/tests/unit/backend/viewers/test_postgresql_dao.py b/tests/unit/backend/viewers/test_postgresql_dao.py new file mode 100644 index 00000000..b1cf6918 --- /dev/null +++ b/tests/unit/backend/viewers/test_postgresql_dao.py @@ -0,0 +1,456 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +import importlib +import numpy as np + +import pytest + + +class FakeDb: + def __init__(self, countsByTableId=None): + self.countsByTableId = countsByTableId or {} + self.fetchOneCalls = [] + + def fetchOne(self, sql, params=None): + self.fetchOneCalls.append( + { + "sql": sql, + "params": params, + } + ) + + tableId = None + if params: + try: + tableId = int(params[0]) + except Exception: + tableId = None + + return {"count": self.countsByTableId.get(tableId, 0)} + + +class FakeSetMapper: + def __init__( + self, + storedSet, + logicalTables=None, + columnsByTableId=None, + itemsByTableId=None, + ): + self.storedSet = storedSet + self.logicalTables = logicalTables or [] + self.columnsByTableId = columnsByTableId or {} + self.itemsByTableId = itemsByTableId or {} + self.getStoredSetCalls = [] + self.listStoredSetTablesCalls = [] + self.getStoredSetTableColumnsCalls = [] + self.getStoredSetTableItemsCalls = [] + + def getStoredSet(self, projectId, protocolDbId, outputName, limit=None, offset=0): + self.getStoredSetCalls.append( + { + "projectId": projectId, + "protocolDbId": protocolDbId, + "outputName": outputName, + "limit": limit, + "offset": offset, + } + ) + + storedSet = dict(self.storedSet) + items = list(storedSet.get("items") or []) + start = max(0, int(offset or 0)) + + if limit is None: + storedSet["items"] = items[start:] + else: + storedSet["items"] = items[start:start + int(limit)] + + return storedSet + + def listStoredSetTables(self, setId): + self.listStoredSetTablesCalls.append(setId) + return list(self.logicalTables) + + def getStoredSetTableColumns(self, tableId): + self.getStoredSetTableColumnsCalls.append(tableId) + return list(self.columnsByTableId.get(int(tableId), [])) + + def getStoredSetTableItems(self, tableId, limit=None, offset=0): + self.getStoredSetTableItemsCalls.append( + { + "tableId": tableId, + "limit": limit, + "offset": offset, + } + ) + + items = list(self.itemsByTableId.get(int(tableId), [])) + start = max(0, int(offset or 0)) + + if limit is None: + return items[start:] + + return items[start:start + int(limit)] + + +class FakeObjectManager: + def __init__(self, hiddenLabels=None): + self.hiddenLabels = set(hiddenLabels or []) + + def isLabelVisible(self, label): + return label not in self.hiddenLabels + + +@pytest.fixture +def postgresqlDaoModule(authTestEnv): + return importlib.import_module("app.backend.viewers.postgresql_dao") + + +def _makeStoredSet(setClassName="SetOfClasses3D", itemClassName="Class3D"): + return { + "id": 100, + "setClassName": setClassName, + "itemClassName": itemClassName, + "columns": [ + { + "labelProperty": "score", + "position": 0, + "className": "Float", + }, + ], + "items": [ + { + "id": 1, + "scipionItemId": 1, + "enabled": True, + "values": { + "score": 0.5, + }, + }, + { + "id": 2, + "scipionItemId": 2, + "enabled": True, + "values": { + "score": 0.7, + }, + }, + ], + "properties": { + "boxSize": 128, + "itemsCount": 2, + }, + "setProperties": [ + { + "key": "samplingRate", + "value": 1.25, + }, + ], + } + + +def _makeLogicalTables(rootItemClassName="Class3D", childItemClassName="Particle"): + return [ + { + "id": 1001, + "name": "objects", + "alias": "Root objects", + "itemClassName": rootItemClassName, + }, + { + "id": 1002, + "name": "Class001_Objects", + "alias": "Class001 objects", + "itemClassName": childItemClassName, + }, + ] + + +def _makeColumnsByTableId(): + return { + 1001: [ + { + "labelProperty": "score", + "position": 0, + "className": "Float", + }, + ], + 1002: [ + { + "labelProperty": "particleId", + "position": 0, + "className": "Integer", + }, + ], + } + + +def _makeItemsByTableId(): + return { + 1001: [ + { + "id": 1, + "scipionItemId": 1, + "enabled": True, + "values": { + "score": 0.5, + }, + }, + ], + 1002: [ + { + "id": 11, + "scipionItemId": 11, + "enabled": True, + "values": { + "particleId": 101, + }, + }, + ], + } + + +def _buildDao(postgresqlDaoModule, monkeypatch, setMapper, db=None): + db = db or FakeDb(countsByTableId={1001: 1, 1002: 1}) + monkeypatch.setattr( + postgresqlDaoModule, + "ScipionSetPostgresqlMapper", + lambda dbConnection: setMapper, + ) + + return postgresqlDaoModule.PostgresqlDAO( + db=db, + projectId=1, + protocolId=10, + outputName="outputClasses", + ) + + +def _buildLogicalDao( + postgresqlDaoModule, + monkeypatch, + setClassName="SetOfClasses3D", + rootItemClassName="Class3D", + childItemClassName="Particle", +): + storedSet = _makeStoredSet( + setClassName=setClassName, + itemClassName=rootItemClassName, + ) + logicalTables = _makeLogicalTables( + rootItemClassName=rootItemClassName, + childItemClassName=childItemClassName, + ) + setMapper = FakeSetMapper( + storedSet=storedSet, + logicalTables=logicalTables, + columnsByTableId=_makeColumnsByTableId(), + itemsByTableId=_makeItemsByTableId(), + ) + dao = _buildDao(postgresqlDaoModule, monkeypatch, setMapper) + + return dao, setMapper + + +def _getActionNames(table): + return [ + action.getName() + for action in table.getActions() + ] + + +def test_GetTablesAddsPropertiesTable(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao(postgresqlDaoModule, monkeypatch) + + tables = dao.getTables() + + assert set(tables.keys()) == {"objects", "Class001_Objects", "Properties"} + assert tables["Properties"].getAlias() == "Properties" + assert dao._useLogicalTables is True + assert dao._objectsType["Class3D"] == "SetOfClasses3D" + assert dao._objectsType["Particle"] == "SetOfParticles" + + +def test_GetPropertiesTableRowsIncludesSelfAndSize(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao(postgresqlDaoModule, monkeypatch) + dao.getTables() + + rowsByKey = { + row["key"]: row["value"] + for row in dao._getPropertiesRows() + } + + assert rowsByKey["self"] == "SetOfClasses3D" + assert rowsByKey["_size"] == "2" + assert rowsByKey["samplingRate"] == "1.25" + assert rowsByKey["boxSize"] == "128" + + +def test_GetPropertiesTableHasNoActions(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao(postgresqlDaoModule, monkeypatch) + table = dao.getTables()["Properties"] + + dao.fillTable(table, FakeObjectManager()) + + assert _getActionNames(table) == [] + + +def test_GenerateTableActionsAddsParticleAction(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao(postgresqlDaoModule, monkeypatch) + table = dao.getTables()["Class001_Objects"] + + dao.fillTable(table, FakeObjectManager()) + + assert _getActionNames(table) == ["Particle"] + + +def test_GenerateTableActionsAddsVolumesForClass3D(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao( + postgresqlDaoModule, + monkeypatch, + setClassName="SetOfClasses3D", + rootItemClassName="Class3D", + childItemClassName="Particle", + ) + table = dao.getTables()["objects"] + + dao.fillTable(table, FakeObjectManager()) + + actionNames = _getActionNames(table) + assert "Class3D" in actionNames + assert "Particle" in actionNames + assert "Volumes" in actionNames + + +def test_GenerateTableActionsAddsAveragesForClass2D(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao( + postgresqlDaoModule, + monkeypatch, + setClassName="SetOfClasses2D", + rootItemClassName="Class2D", + childItemClassName="Particle", + ) + table = dao.getTables()["objects"] + + dao.fillTable(table, FakeObjectManager()) + + actionNames = _getActionNames(table) + assert "Class2D" in actionNames + assert "Particle" in actionNames + assert "Averages" in actionNames + + +def test_GetTableWithAdditionalInfoReturnsCompatibilityTuple(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao(postgresqlDaoModule, monkeypatch) + + table, displayColumns = dao.getTableWithAdditionalInfo() + + assert table is None + assert displayColumns == postgresqlDaoModule.ADITIONAL_INFO_DISPLAY_COLUMN_LIST + assert displayColumns == ["_size", "id"] + + +def test_PostgresqlDaoNormalizesTypedStringsFromDatabase(postgresqlDaoModule): + dao = object.__new__(postgresqlDaoModule.PostgresqlDAO) + + matrixText = ( + "[[ 1. 0. 0. -2275.2 ]\n" + " [ 0. 1. 0. -1616.34]\n" + " [ 0. 0. 1. 0. ]\n" + " [ 0. 0. 0. 1. ]]" + ) + + item = { + "id": 1, + "enabled": "False", + "values": { + "flag": "False", + "score": "3.5", + "count": "7", + "transform_matrix": matrixText, + }, + } + + columns = [ + { + "labelProperty": "flag", + "position": 0, + "className": "Boolean", + }, + { + "labelProperty": "score", + "position": 1, + "className": "Float", + }, + { + "labelProperty": "count", + "position": 2, + "className": "Integer", + }, + { + "labelProperty": "transform_matrix", + "position": 3, + "className": "Matrix", + }, + ] + + row = dao._itemToRow(item, columns) + + assert row["enabled"] is False + assert row["flag"] is False + assert row["score"] == pytest.approx(3.5) + assert row["count"] == 7 + + assert isinstance(row["transform_matrix"], np.ndarray) + assert row["transform_matrix"].shape == (4, 4) + assert row["transform_matrix"][0, 3] == pytest.approx(-2275.2) + assert row["transform_matrix"][1, 3] == pytest.approx(-1616.34) + + +def test_PostgresqlDaoParsesJsonMatrixString(postgresqlDaoModule): + dao = object.__new__(postgresqlDaoModule.PostgresqlDAO) + + matrix = dao._normalizeValue( + "transform_matrix", + "[[1, 0, 0, -1.5], [0, 1, 0, 2.5], [0, 0, 1, 0], [0, 0, 0, 1]]", + {"className": "Matrix"}, + ) + + assert isinstance(matrix, np.ndarray) + assert matrix.shape == (4, 4) + assert matrix[0, 3] == pytest.approx(-1.5) + assert matrix[1, 3] == pytest.approx(2.5) + + +def test_PostgresqlDaoDoesNotUseEvalForMatrixStrings(postgresqlDaoModule): + dao = object.__new__(postgresqlDaoModule.PostgresqlDAO) + + matrix = dao._toNumpyMatrix("__import__('os').system('echo unsafe')") + + assert isinstance(matrix, np.ndarray) + assert matrix.size == 0 \ No newline at end of file