From 1b1ad073614194eb1f8780e9e01cbab06af8b604 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:06:29 +0200 Subject: [PATCH 001/278] Add Scipion object persistence models --- app/backend/models/scipion_object_model.py | 216 +++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 app/backend/models/scipion_object_model.py diff --git a/app/backend/models/scipion_object_model.py b/app/backend/models/scipion_object_model.py new file mode 100644 index 00000000..98001554 --- /dev/null +++ b/app/backend/models/scipion_object_model.py @@ -0,0 +1,216 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from sqlalchemy import Boolean, CheckConstraint, Column, DateTime, ForeignKey, Index, Integer, Text, UniqueConstraint +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import relationship +from sqlalchemy.sql import func + +from app.backend.database import Base + + +class ScipionObjectType(Base): + __tablename__ = "scipion_object_types" + + id = Column(Integer, primary_key=True, index=True) + className = Column(Text, nullable=False, unique=True) + moduleName = Column(Text, nullable=True) + baseClassName = Column(Text, nullable=True) + mapperKind = Column(Text, nullable=False, default="tree") + classSchema = Column("schema", JSONB, nullable=False, server_default="{}") + createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) + + properties = relationship("ScipionObjectTypeProperty", back_populates="type", cascade="all, delete-orphan") + + __table_args__ = ( + CheckConstraint("mapperKind IN ('tree', 'flat_set', 'scalar', 'pointer')", name="ck_scipion_object_types_mapper_kind"), + Index("idx_scipion_object_types_className", "className"), + Index("idx_scipion_object_types_mapperKind", "mapperKind"), + Index("idx_scipion_object_types_schema_gin", "schema", postgresql_using="gin"), + ) + + +class ScipionObjectTypeProperty(Base): + __tablename__ = "scipion_object_type_properties" + + id = Column(Integer, primary_key=True, index=True) + typeId = Column(Integer, ForeignKey("scipion_object_types.id", ondelete="CASCADE"), nullable=False) + propertyPath = Column(Text, nullable=False) + className = Column(Text, nullable=True) + valueKind = Column(Text, nullable=True) + isPointer = Column(Boolean, nullable=False, default=False) + isNested = Column(Boolean, nullable=False, default=False) + propertySchema = Column("schema", JSONB, nullable=False, server_default="{}") + createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) + + type = relationship("ScipionObjectType", back_populates="properties") + + __table_args__ = ( + UniqueConstraint("typeId", "propertyPath", name="ux_scipion_object_type_properties_type_path"), + Index("idx_scipion_object_type_properties_path", "propertyPath"), + Index("idx_scipion_object_type_properties_schema_gin", "schema", postgresql_using="gin"), + ) + + +class ScipionObject(Base): + __tablename__ = "scipion_objects" + + id = Column(Integer, primary_key=True, index=True) + projectId = Column(Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + protocolDbId = Column(Integer, ForeignKey("protocols.id", ondelete="CASCADE"), nullable=True) + scipionObjId = Column(Integer, nullable=False) + parentObjectId = Column(Integer, ForeignKey("scipion_objects.id", ondelete="CASCADE"), nullable=True) + name = Column(Text, nullable=True) + path = Column(Text, nullable=False) + className = Column(Text, nullable=False) + value = Column(Text, nullable=True) + label = Column(Text, nullable=True) + comment = Column(Text, nullable=True) + creation = Column(DateTime(timezone=True), nullable=True) + objectMetadata = Column("metadata", JSONB, nullable=False, server_default="{}") + createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) + + parent = relationship("ScipionObject", remote_side=[id], back_populates="children") + children = relationship("ScipionObject", back_populates="parent", cascade="all, delete-orphan") + + __table_args__ = ( + UniqueConstraint("projectId", "protocolDbId", "scipionObjId", name="ux_scipion_objects_project_protocol_obj"), + UniqueConstraint("projectId", "protocolDbId", "path", name="ux_scipion_objects_project_protocol_path"), + Index("idx_scipion_objects_project_class", "projectId", "className"), + Index("idx_scipion_objects_project_protocol", "projectId", "protocolDbId"), + Index("idx_scipion_objects_parent", "parentObjectId"), + Index("idx_scipion_objects_metadata_gin", "metadata", postgresql_using="gin"), + ) + + +class ScipionObjectRelation(Base): + __tablename__ = "scipion_object_relations" + + id = Column(Integer, primary_key=True, index=True) + projectId = Column(Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + creatorObjectId = Column(Integer, ForeignKey("scipion_objects.id", ondelete="CASCADE"), nullable=False) + parentObjectId = Column(Integer, ForeignKey("scipion_objects.id", ondelete="CASCADE"), nullable=False) + childObjectId = Column(Integer, ForeignKey("scipion_objects.id", ondelete="CASCADE"), nullable=False) + name = Column(Text, nullable=False) + parentExtended = Column(Text, nullable=True) + childExtended = Column(Text, nullable=True) + relationMetadata = Column("metadata", JSONB, nullable=False, server_default="{}") + creation = Column(DateTime(timezone=True), nullable=True) + createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + + __table_args__ = ( + Index("idx_scipion_object_relations_project_name", "projectId", "name"), + Index("idx_scipion_object_relations_parent", "parentObjectId"), + Index("idx_scipion_object_relations_child", "childObjectId"), + Index("idx_scipion_object_relations_metadata_gin", "metadata", postgresql_using="gin"), + ) + + +class ScipionSet(Base): + __tablename__ = "scipion_sets" + + id = Column(Integer, primary_key=True, index=True) + projectId = Column(Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + protocolDbId = Column(Integer, ForeignKey("protocols.id", ondelete="CASCADE"), nullable=True) + objectId = Column(Integer, ForeignKey("scipion_objects.id", ondelete="SET NULL"), nullable=True) + outputName = Column(Text, nullable=False) + setClassName = Column(Text, nullable=False) + itemClassName = Column(Text, nullable=False) + properties = Column(JSONB, nullable=False, server_default="{}") + createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) + + columns = relationship("ScipionSetColumn", back_populates="set", cascade="all, delete-orphan") + setProperties = relationship("ScipionSetProperty", back_populates="set", cascade="all, delete-orphan") + items = relationship("ScipionSetItem", back_populates="set", cascade="all, delete-orphan") + + __table_args__ = ( + UniqueConstraint("projectId", "protocolDbId", "outputName", name="ux_scipion_sets_project_protocol_output"), + Index("idx_scipion_sets_project_protocol", "projectId", "protocolDbId"), + Index("idx_scipion_sets_properties_gin", "properties", postgresql_using="gin"), + ) + + +class ScipionSetColumn(Base): + __tablename__ = "scipion_set_columns" + + id = Column(Integer, primary_key=True, index=True) + setId = Column(Integer, ForeignKey("scipion_sets.id", ondelete="CASCADE"), nullable=False) + labelProperty = Column(Text, nullable=False) + columnName = Column(Text, nullable=False) + className = Column(Text, nullable=True) + valueType = Column(Text, nullable=True) + position = Column(Integer, nullable=False) + indexed = Column(Boolean, nullable=False, default=False) + + set = relationship("ScipionSet", back_populates="columns") + + __table_args__ = ( + UniqueConstraint("setId", "labelProperty", name="ux_scipion_set_columns_set_label"), + UniqueConstraint("setId", "columnName", name="ux_scipion_set_columns_set_column"), + Index("idx_scipion_set_columns_label", "labelProperty"), + ) + + +class ScipionSetProperty(Base): + __tablename__ = "scipion_set_properties" + + id = Column(Integer, primary_key=True, index=True) + setId = Column(Integer, ForeignKey("scipion_sets.id", ondelete="CASCADE"), nullable=False) + key = Column(Text, nullable=False) + value = Column(Text, nullable=True) + + set = relationship("ScipionSet", back_populates="setProperties") + + __table_args__ = ( + UniqueConstraint("setId", "key", name="ux_scipion_set_properties_set_key"), + Index("idx_scipion_set_properties_key", "key"), + ) + + +class ScipionSetItem(Base): + __tablename__ = "scipion_set_items" + + id = Column(Integer, primary_key=True, index=True) + setId = Column(Integer, ForeignKey("scipion_sets.id", ondelete="CASCADE"), nullable=False) + scipionItemId = Column(Integer, nullable=False) + enabled = Column(Boolean, nullable=False, default=True) + label = Column(Text, nullable=True) + comment = Column(Text, nullable=True) + creation = Column(DateTime(timezone=True), nullable=True) + values = Column(JSONB, nullable=False, server_default="{}") + createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) + + set = relationship("ScipionSet", back_populates="items") + + __table_args__ = ( + UniqueConstraint("setId", "scipionItemId", name="ux_scipion_set_items_set_item"), + Index("idx_scipion_set_items_set", "setId"), + Index("idx_scipion_set_items_values_gin", "values", postgresql_using="gin"), + ) From 46dd719f97056b4ecdd596326056bce35c211794 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:06:46 +0200 Subject: [PATCH 002/278] Register Scipion object models --- app/backend/models/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/backend/models/__init__.py b/app/backend/models/__init__.py index 4abd50b4..8ae235f9 100644 --- a/app/backend/models/__init__.py +++ b/app/backend/models/__init__.py @@ -28,3 +28,13 @@ from app.backend.models.protocol_model import Protocol # noqa: F401 from app.backend.models.settings_models import * # noqa: F401 from app.backend.models.protocol_step_model import ProtocolStep # noqa: F401 +from app.backend.models.scipion_object_model import ( # noqa: F401 + ScipionObject, + ScipionObjectRelation, + ScipionObjectType, + ScipionObjectTypeProperty, + ScipionSet, + ScipionSetColumn, + ScipionSetItem, + ScipionSetProperty, +) From 44a97886c19fb2f79979b067a06cec34a03a3640 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:07:08 +0200 Subject: [PATCH 003/278] Include Scipion object models in Alembic metadata --- alembic/env.py | 1 + 1 file changed, 1 insertion(+) diff --git a/alembic/env.py b/alembic/env.py index b8910b44..c7fecb6e 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -36,6 +36,7 @@ tag_model, protocol_tag_assignment_model, protocol_step_model, + scipion_object_model, ) target_metadata = Base.metadata From 60e7fa92a13842dd6bca0aacf15f92ce1739656a Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:08:39 +0200 Subject: [PATCH 004/278] Use explicit JSONB defaults for Scipion models --- app/backend/models/scipion_object_model.py | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/app/backend/models/scipion_object_model.py b/app/backend/models/scipion_object_model.py index 98001554..82b021ff 100644 --- a/app/backend/models/scipion_object_model.py +++ b/app/backend/models/scipion_object_model.py @@ -23,7 +23,7 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ****************************************************************************** -from sqlalchemy import Boolean, CheckConstraint, Column, DateTime, ForeignKey, Index, Integer, Text, UniqueConstraint +from sqlalchemy import Boolean, CheckConstraint, Column, DateTime, ForeignKey, Index, Integer, Text, UniqueConstraint, text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import relationship from sqlalchemy.sql import func @@ -38,8 +38,8 @@ class ScipionObjectType(Base): className = Column(Text, nullable=False, unique=True) moduleName = Column(Text, nullable=True) baseClassName = Column(Text, nullable=True) - mapperKind = Column(Text, nullable=False, default="tree") - classSchema = Column("schema", JSONB, nullable=False, server_default="{}") + mapperKind = Column(Text, nullable=False, default="tree", server_default="tree") + classSchema = Column("schema", JSONB, nullable=False, server_default=text("'{}'::jsonb")) createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) @@ -61,9 +61,9 @@ class ScipionObjectTypeProperty(Base): propertyPath = Column(Text, nullable=False) className = Column(Text, nullable=True) valueKind = Column(Text, nullable=True) - isPointer = Column(Boolean, nullable=False, default=False) - isNested = Column(Boolean, nullable=False, default=False) - propertySchema = Column("schema", JSONB, nullable=False, server_default="{}") + isPointer = Column(Boolean, nullable=False, default=False, server_default="false") + isNested = Column(Boolean, nullable=False, default=False, server_default="false") + propertySchema = Column("schema", JSONB, nullable=False, server_default=text("'{}'::jsonb")) createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) @@ -91,7 +91,7 @@ class ScipionObject(Base): label = Column(Text, nullable=True) comment = Column(Text, nullable=True) creation = Column(DateTime(timezone=True), nullable=True) - objectMetadata = Column("metadata", JSONB, nullable=False, server_default="{}") + objectMetadata = Column("metadata", JSONB, nullable=False, server_default=text("'{}'::jsonb")) createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) @@ -119,7 +119,7 @@ class ScipionObjectRelation(Base): name = Column(Text, nullable=False) parentExtended = Column(Text, nullable=True) childExtended = Column(Text, nullable=True) - relationMetadata = Column("metadata", JSONB, nullable=False, server_default="{}") + relationMetadata = Column("metadata", JSONB, nullable=False, server_default=text("'{}'::jsonb")) creation = Column(DateTime(timezone=True), nullable=True) createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) @@ -141,7 +141,7 @@ class ScipionSet(Base): outputName = Column(Text, nullable=False) setClassName = Column(Text, nullable=False) itemClassName = Column(Text, nullable=False) - properties = Column(JSONB, nullable=False, server_default="{}") + properties = Column(JSONB, nullable=False, server_default=text("'{}'::jsonb")) createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) @@ -166,7 +166,7 @@ class ScipionSetColumn(Base): className = Column(Text, nullable=True) valueType = Column(Text, nullable=True) position = Column(Integer, nullable=False) - indexed = Column(Boolean, nullable=False, default=False) + indexed = Column(Boolean, nullable=False, default=False, server_default="false") set = relationship("ScipionSet", back_populates="columns") @@ -199,11 +199,11 @@ class ScipionSetItem(Base): id = Column(Integer, primary_key=True, index=True) setId = Column(Integer, ForeignKey("scipion_sets.id", ondelete="CASCADE"), nullable=False) scipionItemId = Column(Integer, nullable=False) - enabled = Column(Boolean, nullable=False, default=True) + enabled = Column(Boolean, nullable=False, default=True, server_default="true") label = Column(Text, nullable=True) comment = Column(Text, nullable=True) creation = Column(DateTime(timezone=True), nullable=True) - values = Column(JSONB, nullable=False, server_default="{}") + values = Column(JSONB, nullable=False, server_default=text("'{}'::jsonb")) createdAt = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) updatedAt = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) From cbc8567b0d0963574d54ec9b948e53978093853b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:17:00 +0200 Subject: [PATCH 005/278] Quote mapperKind in Scipion object type check constraint --- app/backend/models/scipion_object_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/backend/models/scipion_object_model.py b/app/backend/models/scipion_object_model.py index 82b021ff..5f9c9dc1 100644 --- a/app/backend/models/scipion_object_model.py +++ b/app/backend/models/scipion_object_model.py @@ -46,7 +46,7 @@ class ScipionObjectType(Base): properties = relationship("ScipionObjectTypeProperty", back_populates="type", cascade="all, delete-orphan") __table_args__ = ( - CheckConstraint("mapperKind IN ('tree', 'flat_set', 'scalar', 'pointer')", name="ck_scipion_object_types_mapper_kind"), + CheckConstraint('"mapperKind" IN (\'tree\', \'flat_set\', \'scalar\', \'pointer\')', name="ck_scipion_object_types_mapper_kind"), Index("idx_scipion_object_types_className", "className"), Index("idx_scipion_object_types_mapperKind", "mapperKind"), Index("idx_scipion_object_types_schema_gin", "schema", postgresql_using="gin"), From 2b290c325b2221c63ee6319ed6a89b7200f1ea00 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 19 Jun 2026 11:19:17 +0200 Subject: [PATCH 006/278] Add initial PostgreSQL schema for Scipion objects --- ...2_add_scipion_object_persistence_models.py | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 alembic/versions/1f8e5b400102_add_scipion_object_persistence_models.py diff --git a/alembic/versions/1f8e5b400102_add_scipion_object_persistence_models.py b/alembic/versions/1f8e5b400102_add_scipion_object_persistence_models.py new file mode 100644 index 00000000..e8f2a914 --- /dev/null +++ b/alembic/versions/1f8e5b400102_add_scipion_object_persistence_models.py @@ -0,0 +1,249 @@ +"""add scipion object persistence models + +Revision ID: 1f8e5b400102 +Revises: 6806b9a72a6e +Create Date: 2026-06-19 11:10:02.296385 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = "1f8e5b400102" +down_revision: Union[str, None] = "6806b9a72a6e" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "scipion_object_types", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("className", sa.Text(), nullable=False), + sa.Column("moduleName", sa.Text(), nullable=True), + sa.Column("baseClassName", sa.Text(), nullable=True), + sa.Column("mapperKind", sa.Text(), server_default="tree", nullable=False), + sa.Column( + "schema", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.CheckConstraint("\"mapperKind\" IN ('tree', 'flat_set', 'scalar', 'pointer')", name="ck_scipion_object_types_mapper_kind"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("className"), + ) + op.create_index("idx_scipion_object_types_className", "scipion_object_types", ["className"]) + op.create_index("idx_scipion_object_types_mapperKind", "scipion_object_types", ["mapperKind"]) + op.create_index( + "idx_scipion_object_types_schema_gin", + "scipion_object_types", + ["schema"], + postgresql_using="gin", + ) + op.create_index(op.f("ix_scipion_object_types_id"), "scipion_object_types", ["id"]) + + op.create_table( + "scipion_object_type_properties", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("typeId", sa.Integer(), nullable=False), + sa.Column("propertyPath", sa.Text(), nullable=False), + sa.Column("className", sa.Text(), nullable=True), + sa.Column("valueKind", sa.Text(), nullable=True), + sa.Column("isPointer", sa.Boolean(), server_default="false", nullable=False), + sa.Column("isNested", sa.Boolean(), server_default="false", nullable=False), + sa.Column( + "schema", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["typeId"], ["scipion_object_types.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("typeId", "propertyPath", name="ux_scipion_object_type_properties_type_path"), + ) + op.create_index("idx_scipion_object_type_properties_path", "scipion_object_type_properties", ["propertyPath"]) + op.create_index( + "idx_scipion_object_type_properties_schema_gin", + "scipion_object_type_properties", + ["schema"], + postgresql_using="gin", + ) + op.create_index(op.f("ix_scipion_object_type_properties_id"), "scipion_object_type_properties", ["id"]) + + op.create_table( + "scipion_objects", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("projectId", sa.Integer(), nullable=False), + sa.Column("protocolDbId", sa.Integer(), nullable=True), + sa.Column("scipionObjId", sa.Integer(), nullable=False), + sa.Column("parentObjectId", sa.Integer(), nullable=True), + sa.Column("name", sa.Text(), nullable=True), + sa.Column("path", sa.Text(), nullable=False), + sa.Column("className", sa.Text(), nullable=False), + sa.Column("value", sa.Text(), nullable=True), + sa.Column("label", sa.Text(), nullable=True), + sa.Column("comment", sa.Text(), nullable=True), + sa.Column("creation", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "metadata", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["parentObjectId"], ["scipion_objects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["projectId"], ["projects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["protocolDbId"], ["protocols.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("projectId", "protocolDbId", "path", name="ux_scipion_objects_project_protocol_path"), + sa.UniqueConstraint( + "projectId", + "protocolDbId", + "scipionObjId", + name="ux_scipion_objects_project_protocol_obj", + ), + ) + op.create_index("idx_scipion_objects_metadata_gin", "scipion_objects", ["metadata"], postgresql_using="gin") + op.create_index("idx_scipion_objects_parent", "scipion_objects", ["parentObjectId"]) + op.create_index("idx_scipion_objects_project_class", "scipion_objects", ["projectId", "className"]) + op.create_index("idx_scipion_objects_project_protocol", "scipion_objects", ["projectId", "protocolDbId"]) + op.create_index(op.f("ix_scipion_objects_id"), "scipion_objects", ["id"]) + + op.create_table( + "scipion_object_relations", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("projectId", sa.Integer(), nullable=False), + sa.Column("creatorObjectId", sa.Integer(), nullable=False), + sa.Column("parentObjectId", sa.Integer(), nullable=False), + sa.Column("childObjectId", sa.Integer(), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("parentExtended", sa.Text(), nullable=True), + sa.Column("childExtended", sa.Text(), nullable=True), + sa.Column( + "metadata", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("creation", sa.DateTime(timezone=True), nullable=True), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["childObjectId"], ["scipion_objects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["creatorObjectId"], ["scipion_objects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["parentObjectId"], ["scipion_objects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["projectId"], ["projects.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("idx_scipion_object_relations_child", "scipion_object_relations", ["childObjectId"]) + op.create_index( + "idx_scipion_object_relations_metadata_gin", + "scipion_object_relations", + ["metadata"], + postgresql_using="gin", + ) + op.create_index("idx_scipion_object_relations_parent", "scipion_object_relations", ["parentObjectId"]) + op.create_index("idx_scipion_object_relations_project_name", "scipion_object_relations", ["projectId", "name"]) + op.create_index(op.f("ix_scipion_object_relations_id"), "scipion_object_relations", ["id"]) + + op.create_table( + "scipion_sets", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("projectId", sa.Integer(), nullable=False), + sa.Column("protocolDbId", sa.Integer(), nullable=True), + sa.Column("objectId", sa.Integer(), nullable=True), + sa.Column("outputName", sa.Text(), nullable=False), + sa.Column("setClassName", sa.Text(), nullable=False), + sa.Column("itemClassName", sa.Text(), nullable=False), + sa.Column( + "properties", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["objectId"], ["scipion_objects.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["projectId"], ["projects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["protocolDbId"], ["protocols.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("projectId", "protocolDbId", "outputName", name="ux_scipion_sets_project_protocol_output"), + ) + op.create_index("idx_scipion_sets_project_protocol", "scipion_sets", ["projectId", "protocolDbId"]) + op.create_index("idx_scipion_sets_properties_gin", "scipion_sets", ["properties"], postgresql_using="gin") + op.create_index(op.f("ix_scipion_sets_id"), "scipion_sets", ["id"]) + + op.create_table( + "scipion_set_columns", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("setId", sa.Integer(), nullable=False), + sa.Column("labelProperty", sa.Text(), nullable=False), + sa.Column("columnName", sa.Text(), nullable=False), + sa.Column("className", sa.Text(), nullable=True), + sa.Column("valueType", sa.Text(), nullable=True), + sa.Column("position", sa.Integer(), nullable=False), + sa.Column("indexed", sa.Boolean(), server_default="false", nullable=False), + sa.ForeignKeyConstraint(["setId"], ["scipion_sets.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("setId", "columnName", name="ux_scipion_set_columns_set_column"), + sa.UniqueConstraint("setId", "labelProperty", name="ux_scipion_set_columns_set_label"), + ) + op.create_index("idx_scipion_set_columns_label", "scipion_set_columns", ["labelProperty"]) + op.create_index(op.f("ix_scipion_set_columns_id"), "scipion_set_columns", ["id"]) + + op.create_table( + "scipion_set_items", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("setId", sa.Integer(), nullable=False), + sa.Column("scipionItemId", sa.Integer(), nullable=False), + sa.Column("enabled", sa.Boolean(), server_default="true", nullable=False), + sa.Column("label", sa.Text(), nullable=True), + sa.Column("comment", sa.Text(), nullable=True), + sa.Column("creation", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "values", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["setId"], ["scipion_sets.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("setId", "scipionItemId", name="ux_scipion_set_items_set_item"), + ) + op.create_index("idx_scipion_set_items_set", "scipion_set_items", ["setId"]) + op.create_index("idx_scipion_set_items_values_gin", "scipion_set_items", ["values"], postgresql_using="gin") + op.create_index(op.f("ix_scipion_set_items_id"), "scipion_set_items", ["id"]) + + op.create_table( + "scipion_set_properties", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("setId", sa.Integer(), nullable=False), + sa.Column("key", sa.Text(), nullable=False), + sa.Column("value", sa.Text(), nullable=True), + sa.ForeignKeyConstraint(["setId"], ["scipion_sets.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("setId", "key", name="ux_scipion_set_properties_set_key"), + ) + op.create_index("idx_scipion_set_properties_key", "scipion_set_properties", ["key"]) + op.create_index(op.f("ix_scipion_set_properties_id"), "scipion_set_properties", ["id"]) + + +def downgrade() -> None: + op.drop_table("scipion_set_properties") + op.drop_table("scipion_set_items") + op.drop_table("scipion_set_columns") + op.drop_table("scipion_sets") + op.drop_table("scipion_object_relations") + op.drop_table("scipion_objects") + op.drop_table("scipion_object_type_properties") + op.drop_table("scipion_object_types") \ No newline at end of file From f319c9a1e0ab3f1ae32690ef547cfe92824cc312 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:24:54 +0200 Subject: [PATCH 007/278] Add Scipion object PostgreSQL type mapper --- app/backend/mapper/scipion_object_mapper.py | 293 ++++++++++++++++++++ 1 file changed, 293 insertions(+) create mode 100644 app/backend/mapper/scipion_object_mapper.py diff --git a/app/backend/mapper/scipion_object_mapper.py b/app/backend/mapper/scipion_object_mapper.py new file mode 100644 index 00000000..d609ec6c --- /dev/null +++ b/app/backend/mapper/scipion_object_mapper.py @@ -0,0 +1,293 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +import json +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple + +import psycopg2.extras + + +class ScipionObjectPostgresqlMapper: + """Register Scipion object classes and their persistent attributes in PostgreSQL.""" + + def __init__(self, db): + self.db = db + + def registerObjectTypeFromObject( + self, + scipionObj: Any, + mapperKind: Optional[str] = None, + includeProperties: bool = True, + includeNestedProperties: bool = True, + classSchema: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + typeId = self.registerObjectType(scipionObj, mapperKind=mapperKind, classSchema=classSchema) + propertiesCount = 0 + + if includeProperties: + propertiesCount = self.registerObjectTypeProperties( + typeId, + scipionObj, + includeNestedProperties=includeNestedProperties, + ) + + return { + "typeId": typeId, + "className": self._getClassName(scipionObj), + "propertiesCount": propertiesCount, + } + + def registerObjectType( + self, + scipionObj: Any, + mapperKind: Optional[str] = None, + classSchema: Optional[Dict[str, Any]] = None, + ) -> int: + className = self._getClassName(scipionObj) + if not className: + raise ValueError("Cannot register a Scipion object type without className") + + moduleName = self._getModuleName(scipionObj) + baseClassName = self._getBaseClassName(scipionObj) + resolvedMapperKind = mapperKind or self._guessMapperKind(scipionObj) + schema = classSchema or {} + + cur = self.db.execute( + """ + INSERT INTO scipion_object_types ( + "className", + "moduleName", + "baseClassName", + "mapperKind", + "schema" + ) + VALUES (%s, %s, %s, %s, %s::jsonb) + ON CONFLICT ("className") + DO UPDATE SET + "moduleName" = EXCLUDED."moduleName", + "baseClassName" = EXCLUDED."baseClassName", + "mapperKind" = EXCLUDED."mapperKind", + "schema" = scipion_object_types."schema" || EXCLUDED."schema", + "updatedAt" = NOW() + RETURNING id + """, + ( + className, + moduleName, + baseClassName, + resolvedMapperKind, + self._jsonParam(schema), + ), + ) + row = cur.fetchone() + return int(row["id"]) + + def registerObjectTypeProperties( + self, + typeId: int, + scipionObj: Any, + includeNestedProperties: bool = True, + ) -> int: + properties = list(self._iterProperties(scipionObj, includeNestedProperties=includeNestedProperties)) + if not properties: + return 0 + + with self.db.transaction(): + for prop in properties: + self.db.execute( + """ + INSERT INTO scipion_object_type_properties ( + "typeId", + "propertyPath", + "className", + "valueKind", + "isPointer", + "isNested", + "schema" + ) + VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb) + ON CONFLICT ("typeId", "propertyPath") + DO UPDATE SET + "className" = EXCLUDED."className", + "valueKind" = EXCLUDED."valueKind", + "isPointer" = EXCLUDED."isPointer", + "isNested" = EXCLUDED."isNested", + "schema" = scipion_object_type_properties."schema" || EXCLUDED."schema", + "updatedAt" = NOW() + """, + ( + typeId, + prop["propertyPath"], + prop["className"], + prop["valueKind"], + prop["isPointer"], + prop["isNested"], + self._jsonParam(prop.get("schema") or {}), + ), + commit=False, + ) + + return len(properties) + + def getObjectType(self, className: str) -> Optional[Dict[str, Any]]: + return self.db.fetchOne( + """ + SELECT id, "className", "moduleName", "baseClassName", "mapperKind", "schema", "createdAt", "updatedAt" + FROM scipion_object_types + WHERE "className" = %s + """, + (className,), + ) + + def listObjectTypeProperties(self, className: str) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT p.id, + p."typeId", + p."propertyPath", + p."className", + p."valueKind", + p."isPointer", + p."isNested", + p."schema", + p."createdAt", + p."updatedAt" + FROM scipion_object_type_properties p + JOIN scipion_object_types t + ON t.id = p."typeId" + WHERE t."className" = %s + ORDER BY p."propertyPath" ASC + """, + (className,), + ) + + def _iterProperties( + self, + scipionObj: Any, + prefix: str = "", + includeNestedProperties: bool = True, + visited: Optional[Set[int]] = None, + ) -> Iterable[Dict[str, Any]]: + visited = visited or set() + objIdentity = id(scipionObj) + if objIdentity in visited: + return + visited.add(objIdentity) + + for attrName, attrValue in self._getAttributesToStore(scipionObj): + propertyPath = f"{prefix}.{attrName}" if prefix else str(attrName) + childAttributes = self._getAttributesToStore(attrValue) + isPointer = self._isPointer(attrValue) + isNested = bool(childAttributes) + + yield { + "propertyPath": propertyPath, + "className": self._getClassName(attrValue), + "valueKind": self._getValueKind(attrValue, isPointer=isPointer, isNested=isNested), + "isPointer": isPointer, + "isNested": isNested, + "schema": { + "moduleName": self._getModuleName(attrValue), + "baseClassName": self._getBaseClassName(attrValue), + }, + } + + if includeNestedProperties and isNested: + yield from self._iterProperties( + attrValue, + prefix=propertyPath, + includeNestedProperties=includeNestedProperties, + visited=visited, + ) + + def _getAttributesToStore(self, scipionObj: Any) -> List[Tuple[str, Any]]: + getter = getattr(scipionObj, "getAttributesToStore", None) + if not callable(getter): + return [] + + try: + return [(str(name), value) for name, value in getter()] + except Exception: + return [] + + def _getClassName(self, scipionObj: Any) -> Optional[str]: + getter = getattr(scipionObj, "getClassName", None) + if callable(getter): + try: + className = getter() + if className: + return str(className) + except Exception: + pass + + if scipionObj is None: + return None + return scipionObj.__class__.__name__ + + def _getModuleName(self, scipionObj: Any) -> Optional[str]: + if scipionObj is None: + return None + moduleName = getattr(scipionObj.__class__, "__module__", None) + return str(moduleName) if moduleName else None + + def _getBaseClassName(self, scipionObj: Any) -> Optional[str]: + if scipionObj is None: + return None + bases = getattr(scipionObj.__class__, "__bases__", None) or [] + if not bases: + return None + return bases[0].__name__ + + def _guessMapperKind(self, scipionObj: Any) -> str: + className = self._getClassName(scipionObj) or "" + if self._isPointer(scipionObj): + return "pointer" + if className.startswith("SetOf"): + return "flat_set" + if self._getAttributesToStore(scipionObj): + return "tree" + return "scalar" + + def _getValueKind(self, scipionObj: Any, isPointer: bool, isNested: bool) -> str: + if isPointer: + return "pointer" + if isNested: + return "object" + className = self._getClassName(scipionObj) + if className: + return className + return "scalar" + + def _isPointer(self, scipionObj: Any) -> bool: + checker = getattr(scipionObj, "isPointer", None) + if not callable(checker): + return False + try: + return bool(checker()) + except Exception: + return False + + def _jsonParam(self, value: Dict[str, Any]) -> Any: + return psycopg2.extras.Json(value or {}, dumps=json.dumps) From 7ac9fbc39abb70e070fe0207dc2802d753ee500e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:25:18 +0200 Subject: [PATCH 008/278] Expose Scipion object mapper --- app/backend/mapper/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/backend/mapper/__init__.py b/app/backend/mapper/__init__.py index 4cdb527e..beca5fd9 100644 --- a/app/backend/mapper/__init__.py +++ b/app/backend/mapper/__init__.py @@ -23,3 +23,4 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ****************************************************************************** +from app.backend.mapper.scipion_object_mapper import ScipionObjectPostgresqlMapper # noqa: F401 From 62482a7b219e4fa8b93551b6b46accc8377b9025 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:42:13 +0200 Subject: [PATCH 009/278] Add Scipion object tree storage --- app/backend/mapper/scipion_object_mapper.py | 294 +++++++++++++++++++- 1 file changed, 293 insertions(+), 1 deletion(-) diff --git a/app/backend/mapper/scipion_object_mapper.py b/app/backend/mapper/scipion_object_mapper.py index d609ec6c..9a8791b6 100644 --- a/app/backend/mapper/scipion_object_mapper.py +++ b/app/backend/mapper/scipion_object_mapper.py @@ -23,6 +23,7 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ****************************************************************************** +import hashlib import json from typing import Any, Dict, Iterable, List, Optional, Set, Tuple @@ -30,7 +31,7 @@ class ScipionObjectPostgresqlMapper: - """Register Scipion object classes and their persistent attributes in PostgreSQL.""" + """Register and store Scipion data objects in PostgreSQL.""" def __init__(self, db): self.db = db @@ -151,6 +152,105 @@ def registerObjectTypeProperties( return len(properties) + def storeObjectTree( + self, + projectId: int, + protocolDbId: int, + outputName: str, + scipionObj: Any, + registerType: bool = True, + includeNestedProperties: bool = True, + ) -> Dict[str, Any]: + if not projectId: + raise ValueError("projectId is required") + if not protocolDbId: + raise ValueError("protocolDbId is required") + if not outputName: + raise ValueError("outputName is required") + + if registerType: + self.registerObjectTypeFromObject( + scipionObj, + includeNestedProperties=includeNestedProperties, + ) + + storedPaths: List[str] = [] + with self.db.transaction(): + rootObjectId = self._storeObjectNode( + projectId=projectId, + protocolDbId=protocolDbId, + scipionObj=scipionObj, + name=outputName, + path=outputName, + parentObjectId=None, + storedPaths=storedPaths, + includeNestedProperties=includeNestedProperties, + visited=set(), + ) + + return { + "rootObjectId": rootObjectId, + "projectId": projectId, + "protocolDbId": protocolDbId, + "outputName": outputName, + "storedObjectsCount": len(storedPaths), + "storedPaths": storedPaths, + } + + def getStoredObjectTree(self, projectId: int, protocolDbId: int, outputName: str) -> List[Dict[str, Any]]: + rootPath = str(outputName) + return self.db.fetchAll( + """ + SELECT id, + "projectId", + "protocolDbId", + "scipionObjId", + "parentObjectId", + name, + path, + "className", + value, + label, + comment, + creation, + metadata, + "createdAt", + "updatedAt" + FROM scipion_objects + WHERE "projectId" = %s + AND "protocolDbId" = %s + AND (path = %s OR path LIKE %s) + ORDER BY path ASC + """, + (projectId, protocolDbId, rootPath, f"{rootPath}.%"), + ) + + def listProtocolStoredObjects(self, projectId: int, protocolDbId: int) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT id, + "projectId", + "protocolDbId", + "scipionObjId", + "parentObjectId", + name, + path, + "className", + value, + label, + comment, + creation, + metadata, + "createdAt", + "updatedAt" + FROM scipion_objects + WHERE "projectId" = %s + AND "protocolDbId" = %s + ORDER BY path ASC + """, + (projectId, protocolDbId), + ) + def getObjectType(self, className: str) -> Optional[Dict[str, Any]]: return self.db.fetchOne( """ @@ -183,6 +283,102 @@ def listObjectTypeProperties(self, className: str) -> List[Dict[str, Any]]: (className,), ) + def _storeObjectNode( + self, + projectId: int, + protocolDbId: int, + scipionObj: Any, + name: str, + path: str, + parentObjectId: Optional[int], + storedPaths: List[str], + includeNestedProperties: bool, + visited: Set[int], + ) -> int: + objIdentity = id(scipionObj) + if objIdentity in visited: + return parentObjectId or 0 + visited.add(objIdentity) + + attributes = self._getAttributesToStore(scipionObj) + isPointer = self._isPointer(scipionObj) + isNested = bool(attributes) + metadata = { + "moduleName": self._getModuleName(scipionObj), + "baseClassName": self._getBaseClassName(scipionObj), + "isPointer": isPointer, + "isNested": isNested, + "hasSourceObjId": self._getSourceObjId(scipionObj) is not None, + } + + cur = self.db.execute( + """ + INSERT INTO scipion_objects ( + "projectId", + "protocolDbId", + "scipionObjId", + "parentObjectId", + name, + path, + "className", + value, + label, + comment, + creation, + metadata + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb) + ON CONFLICT ON CONSTRAINT ux_scipion_objects_project_protocol_path + DO UPDATE SET + "scipionObjId" = EXCLUDED."scipionObjId", + "parentObjectId" = EXCLUDED."parentObjectId", + name = EXCLUDED.name, + "className" = EXCLUDED."className", + value = EXCLUDED.value, + label = EXCLUDED.label, + comment = EXCLUDED.comment, + creation = EXCLUDED.creation, + metadata = scipion_objects.metadata || EXCLUDED.metadata, + "updatedAt" = NOW() + RETURNING id + """, + ( + projectId, + protocolDbId, + self._getScipionObjId(scipionObj, path), + parentObjectId, + name, + path, + self._getClassName(scipionObj), + self._getObjectValueText(scipionObj), + self._getObjectLabel(scipionObj), + self._getObjectComment(scipionObj), + self._getObjectCreation(scipionObj), + self._jsonParam(metadata), + ), + commit=False, + ) + row = cur.fetchone() + objectId = int(row["id"]) + storedPaths.append(path) + + if includeNestedProperties: + for attrName, attrValue in attributes: + childPath = f"{path}.{attrName}" + self._storeObjectNode( + projectId=projectId, + protocolDbId=protocolDbId, + scipionObj=attrValue, + name=attrName, + path=childPath, + parentObjectId=objectId, + storedPaths=storedPaths, + includeNestedProperties=includeNestedProperties, + visited=visited, + ) + + return objectId + def _iterProperties( self, scipionObj: Any, @@ -280,6 +476,102 @@ def _getValueKind(self, scipionObj: Any, isPointer: bool, isNested: bool) -> str return className return "scalar" + def _getSourceObjId(self, scipionObj: Any) -> Optional[int]: + getters = [getattr(scipionObj, "getObjId", None), getattr(scipionObj, "getId", None)] + for getter in getters: + if not callable(getter): + continue + try: + value = getter() + except Exception: + continue + if value is None: + continue + try: + return int(value) + except Exception: + continue + return None + + def _getScipionObjId(self, scipionObj: Any, path: str) -> int: + sourceObjId = self._getSourceObjId(scipionObj) + if sourceObjId is not None: + return sourceObjId + + digest = hashlib.sha1(path.encode("utf-8")).hexdigest()[:8] + return -int(digest, 16) + + def _getObjectValueText(self, scipionObj: Any) -> Optional[str]: + value = None + + if self._isPointer(scipionObj): + pointedObj = self._getPointerValue(scipionObj) + pointedId = self._getSourceObjId(pointedObj) + if pointedId is not None: + return str(pointedId) + if pointedObj is not None: + return str(pointedObj) + + for methodName in ("getObjValue", "get"): + getter = getattr(scipionObj, methodName, None) + if not callable(getter): + continue + try: + value = getter() + break + except Exception: + continue + + if value is None: + return None + if isinstance(value, (dict, list, tuple)): + return json.dumps(value, ensure_ascii=False) + return str(value) + + def _getPointerValue(self, scipionObj: Any) -> Any: + hasValue = getattr(scipionObj, "hasValue", None) + if callable(hasValue): + try: + if not hasValue(): + return None + except Exception: + return None + + getter = getattr(scipionObj, "get", None) + if not callable(getter): + return None + try: + return getter() + except Exception: + return None + + def _getObjectLabel(self, scipionObj: Any) -> Optional[str]: + return self._getOptionalObjectText(scipionObj, "getObjLabel", "_objLabel") + + def _getObjectComment(self, scipionObj: Any) -> Optional[str]: + return self._getOptionalObjectText(scipionObj, "getObjComment", "_objComment") + + def _getObjectCreation(self, scipionObj: Any) -> Any: + getter = getattr(scipionObj, "getObjCreation", None) + if callable(getter): + try: + return getter() + except Exception: + pass + return getattr(scipionObj, "_objCreation", None) + + def _getOptionalObjectText(self, scipionObj: Any, getterName: str, attributeName: str) -> Optional[str]: + getter = getattr(scipionObj, getterName, None) + if callable(getter): + try: + value = getter() + return str(value) if value else None + except Exception: + pass + + value = getattr(scipionObj, attributeName, None) + return str(value) if value else None + def _isPointer(self, scipionObj: Any) -> bool: checker = getattr(scipionObj, "isPointer", None) if not callable(checker): From 428ed4691e59cbc824f5d325ab3253778d502140 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:46:14 +0200 Subject: [PATCH 010/278] Keep synthetic Scipion object ids in PostgreSQL integer range --- app/backend/mapper/scipion_object_mapper.py | 140 +++++--------------- 1 file changed, 30 insertions(+), 110 deletions(-) diff --git a/app/backend/mapper/scipion_object_mapper.py b/app/backend/mapper/scipion_object_mapper.py index 9a8791b6..56d276a6 100644 --- a/app/backend/mapper/scipion_object_mapper.py +++ b/app/backend/mapper/scipion_object_mapper.py @@ -46,14 +46,12 @@ def registerObjectTypeFromObject( ) -> Dict[str, Any]: typeId = self.registerObjectType(scipionObj, mapperKind=mapperKind, classSchema=classSchema) propertiesCount = 0 - if includeProperties: propertiesCount = self.registerObjectTypeProperties( typeId, scipionObj, includeNestedProperties=includeNestedProperties, ) - return { "typeId": typeId, "className": self._getClassName(scipionObj), @@ -70,19 +68,10 @@ def registerObjectType( if not className: raise ValueError("Cannot register a Scipion object type without className") - moduleName = self._getModuleName(scipionObj) - baseClassName = self._getBaseClassName(scipionObj) - resolvedMapperKind = mapperKind or self._guessMapperKind(scipionObj) - schema = classSchema or {} - cur = self.db.execute( """ INSERT INTO scipion_object_types ( - "className", - "moduleName", - "baseClassName", - "mapperKind", - "schema" + "className", "moduleName", "baseClassName", "mapperKind", "schema" ) VALUES (%s, %s, %s, %s, %s::jsonb) ON CONFLICT ("className") @@ -96,14 +85,13 @@ def registerObjectType( """, ( className, - moduleName, - baseClassName, - resolvedMapperKind, - self._jsonParam(schema), + self._getModuleName(scipionObj), + self._getBaseClassName(scipionObj), + mapperKind or self._guessMapperKind(scipionObj), + self._jsonParam(classSchema or {}), ), ) - row = cur.fetchone() - return int(row["id"]) + return int(cur.fetchone()["id"]) def registerObjectTypeProperties( self, @@ -120,13 +108,8 @@ def registerObjectTypeProperties( self.db.execute( """ INSERT INTO scipion_object_type_properties ( - "typeId", - "propertyPath", - "className", - "valueKind", - "isPointer", - "isNested", - "schema" + "typeId", "propertyPath", "className", "valueKind", + "isPointer", "isNested", "schema" ) VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb) ON CONFLICT ("typeId", "propertyPath") @@ -149,7 +132,6 @@ def registerObjectTypeProperties( ), commit=False, ) - return len(properties) def storeObjectTree( @@ -169,10 +151,7 @@ def storeObjectTree( raise ValueError("outputName is required") if registerType: - self.registerObjectTypeFromObject( - scipionObj, - includeNestedProperties=includeNestedProperties, - ) + self.registerObjectTypeFromObject(scipionObj, includeNestedProperties=includeNestedProperties) storedPaths: List[str] = [] with self.db.transaction(): @@ -201,21 +180,9 @@ def getStoredObjectTree(self, projectId: int, protocolDbId: int, outputName: str rootPath = str(outputName) return self.db.fetchAll( """ - SELECT id, - "projectId", - "protocolDbId", - "scipionObjId", - "parentObjectId", - name, - path, - "className", - value, - label, - comment, - creation, - metadata, - "createdAt", - "updatedAt" + SELECT id, "projectId", "protocolDbId", "scipionObjId", "parentObjectId", + name, path, "className", value, label, comment, creation, + metadata, "createdAt", "updatedAt" FROM scipion_objects WHERE "projectId" = %s AND "protocolDbId" = %s @@ -228,21 +195,9 @@ def getStoredObjectTree(self, projectId: int, protocolDbId: int, outputName: str def listProtocolStoredObjects(self, projectId: int, protocolDbId: int) -> List[Dict[str, Any]]: return self.db.fetchAll( """ - SELECT id, - "projectId", - "protocolDbId", - "scipionObjId", - "parentObjectId", - name, - path, - "className", - value, - label, - comment, - creation, - metadata, - "createdAt", - "updatedAt" + SELECT id, "projectId", "protocolDbId", "scipionObjId", "parentObjectId", + name, path, "className", value, label, comment, creation, + metadata, "createdAt", "updatedAt" FROM scipion_objects WHERE "projectId" = %s AND "protocolDbId" = %s @@ -264,16 +219,8 @@ def getObjectType(self, className: str) -> Optional[Dict[str, Any]]: def listObjectTypeProperties(self, className: str) -> List[Dict[str, Any]]: return self.db.fetchAll( """ - SELECT p.id, - p."typeId", - p."propertyPath", - p."className", - p."valueKind", - p."isPointer", - p."isNested", - p."schema", - p."createdAt", - p."updatedAt" + SELECT p.id, p."typeId", p."propertyPath", p."className", p."valueKind", + p."isPointer", p."isNested", p."schema", p."createdAt", p."updatedAt" FROM scipion_object_type_properties p JOIN scipion_object_types t ON t.id = p."typeId" @@ -301,31 +248,19 @@ def _storeObjectNode( visited.add(objIdentity) attributes = self._getAttributesToStore(scipionObj) - isPointer = self._isPointer(scipionObj) - isNested = bool(attributes) metadata = { "moduleName": self._getModuleName(scipionObj), "baseClassName": self._getBaseClassName(scipionObj), - "isPointer": isPointer, - "isNested": isNested, + "isPointer": self._isPointer(scipionObj), + "isNested": bool(attributes), "hasSourceObjId": self._getSourceObjId(scipionObj) is not None, } cur = self.db.execute( """ INSERT INTO scipion_objects ( - "projectId", - "protocolDbId", - "scipionObjId", - "parentObjectId", - name, - path, - "className", - value, - label, - comment, - creation, - metadata + "projectId", "protocolDbId", "scipionObjId", "parentObjectId", + name, path, "className", value, label, comment, creation, metadata ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb) ON CONFLICT ON CONSTRAINT ux_scipion_objects_project_protocol_path @@ -358,19 +293,17 @@ def _storeObjectNode( ), commit=False, ) - row = cur.fetchone() - objectId = int(row["id"]) + objectId = int(cur.fetchone()["id"]) storedPaths.append(path) if includeNestedProperties: for attrName, attrValue in attributes: - childPath = f"{path}.{attrName}" self._storeObjectNode( projectId=projectId, protocolDbId=protocolDbId, scipionObj=attrValue, name=attrName, - path=childPath, + path=f"{path}.{attrName}", parentObjectId=objectId, storedPaths=storedPaths, includeNestedProperties=includeNestedProperties, @@ -397,7 +330,6 @@ def _iterProperties( childAttributes = self._getAttributesToStore(attrValue) isPointer = self._isPointer(attrValue) isNested = bool(childAttributes) - yield { "propertyPath": propertyPath, "className": self._getClassName(attrValue), @@ -409,7 +341,6 @@ def _iterProperties( "baseClassName": self._getBaseClassName(attrValue), }, } - if includeNestedProperties and isNested: yield from self._iterProperties( attrValue, @@ -422,7 +353,6 @@ def _getAttributesToStore(self, scipionObj: Any) -> List[Tuple[str, Any]]: getter = getattr(scipionObj, "getAttributesToStore", None) if not callable(getter): return [] - try: return [(str(name), value) for name, value in getter()] except Exception: @@ -437,7 +367,6 @@ def _getClassName(self, scipionObj: Any) -> Optional[str]: return str(className) except Exception: pass - if scipionObj is None: return None return scipionObj.__class__.__name__ @@ -452,9 +381,7 @@ def _getBaseClassName(self, scipionObj: Any) -> Optional[str]: if scipionObj is None: return None bases = getattr(scipionObj.__class__, "__bases__", None) or [] - if not bases: - return None - return bases[0].__name__ + return bases[0].__name__ if bases else None def _guessMapperKind(self, scipionObj: Any) -> str: className = self._getClassName(scipionObj) or "" @@ -471,14 +398,11 @@ def _getValueKind(self, scipionObj: Any, isPointer: bool, isNested: bool) -> str return "pointer" if isNested: return "object" - className = self._getClassName(scipionObj) - if className: - return className - return "scalar" + return self._getClassName(scipionObj) or "scalar" def _getSourceObjId(self, scipionObj: Any) -> Optional[int]: - getters = [getattr(scipionObj, "getObjId", None), getattr(scipionObj, "getId", None)] - for getter in getters: + for getterName in ("getObjId", "getId"): + getter = getattr(scipionObj, getterName, None) if not callable(getter): continue try: @@ -497,13 +421,10 @@ def _getScipionObjId(self, scipionObj: Any, path: str) -> int: sourceObjId = self._getSourceObjId(scipionObj) if sourceObjId is not None: return sourceObjId - - digest = hashlib.sha1(path.encode("utf-8")).hexdigest()[:8] - return -int(digest, 16) + digest = hashlib.sha1(path.encode("utf-8")).hexdigest() + return -(int(digest, 16) % 2147483647) def _getObjectValueText(self, scipionObj: Any) -> Optional[str]: - value = None - if self._isPointer(scipionObj): pointedObj = self._getPointerValue(scipionObj) pointedId = self._getSourceObjId(pointedObj) @@ -512,6 +433,7 @@ def _getObjectValueText(self, scipionObj: Any) -> Optional[str]: if pointedObj is not None: return str(pointedObj) + value = None for methodName in ("getObjValue", "get"): getter = getattr(scipionObj, methodName, None) if not callable(getter): @@ -536,7 +458,6 @@ def _getPointerValue(self, scipionObj: Any) -> Any: return None except Exception: return None - getter = getattr(scipionObj, "get", None) if not callable(getter): return None @@ -568,7 +489,6 @@ def _getOptionalObjectText(self, scipionObj: Any, getterName: str, attributeName return str(value) if value else None except Exception: pass - value = getattr(scipionObj, attributeName, None) return str(value) if value else None From e770d7ef8c21c35cdba84e370bd306ec1e928e4c Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:55:42 +0200 Subject: [PATCH 011/278] Allow nullable Scipion object ids --- app/backend/models/scipion_object_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/backend/models/scipion_object_model.py b/app/backend/models/scipion_object_model.py index 5f9c9dc1..ed10ab6b 100644 --- a/app/backend/models/scipion_object_model.py +++ b/app/backend/models/scipion_object_model.py @@ -82,7 +82,7 @@ class ScipionObject(Base): id = Column(Integer, primary_key=True, index=True) projectId = Column(Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) protocolDbId = Column(Integer, ForeignKey("protocols.id", ondelete="CASCADE"), nullable=True) - scipionObjId = Column(Integer, nullable=False) + scipionObjId = Column(Integer, nullable=True) parentObjectId = Column(Integer, ForeignKey("scipion_objects.id", ondelete="CASCADE"), nullable=True) name = Column(Text, nullable=True) path = Column(Text, nullable=False) From 189a7d705d515e47d1d661fa8bd821eb15dfba67 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:57:45 +0200 Subject: [PATCH 012/278] Allow nullable Scipion object ids --- ...4a901_allow_nullable_scipion_object_ids.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 alembic/versions/c3d2b8f4a901_allow_nullable_scipion_object_ids.py diff --git a/alembic/versions/c3d2b8f4a901_allow_nullable_scipion_object_ids.py b/alembic/versions/c3d2b8f4a901_allow_nullable_scipion_object_ids.py new file mode 100644 index 00000000..caf1f078 --- /dev/null +++ b/alembic/versions/c3d2b8f4a901_allow_nullable_scipion_object_ids.py @@ -0,0 +1,38 @@ +"""allow nullable scipion object ids + +Revision ID: c3d2b8f4a901 +Revises: 1f8e5b400102 +Create Date: 2026-06-19 10:15:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "c3d2b8f4a901" +down_revision: Union[str, None] = "1f8e5b400102" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column( + "scipion_objects", + "scipionObjId", + existing_type=sa.Integer(), + nullable=True, + ) + op.execute('UPDATE scipion_objects SET "scipionObjId" = NULL WHERE "scipionObjId" < 0') + + +def downgrade() -> None: + op.execute('UPDATE scipion_objects SET "scipionObjId" = -id WHERE "scipionObjId" IS NULL') + op.alter_column( + "scipion_objects", + "scipionObjId", + existing_type=sa.Integer(), + nullable=False, + ) From 568986069ffa636d06758933d6f20ec971240e6e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:59:38 +0200 Subject: [PATCH 013/278] Stop generating synthetic Scipion object ids --- app/backend/mapper/scipion_object_mapper.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/app/backend/mapper/scipion_object_mapper.py b/app/backend/mapper/scipion_object_mapper.py index 56d276a6..46db5f25 100644 --- a/app/backend/mapper/scipion_object_mapper.py +++ b/app/backend/mapper/scipion_object_mapper.py @@ -16,14 +16,13 @@ # * # * You should have received a copy of the GNU General Public License # * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * Foundation, Inc., 59 Temple Place, Suite 330, MA # * 02111-1307 USA # * # * All comments concerning this program package may be sent to the # * e-mail address 'scipion@cnb.csic.es' # * # ****************************************************************************** -import hashlib import json from typing import Any, Dict, Iterable, List, Optional, Set, Tuple @@ -417,12 +416,8 @@ def _getSourceObjId(self, scipionObj: Any) -> Optional[int]: continue return None - def _getScipionObjId(self, scipionObj: Any, path: str) -> int: - sourceObjId = self._getSourceObjId(scipionObj) - if sourceObjId is not None: - return sourceObjId - digest = hashlib.sha1(path.encode("utf-8")).hexdigest() - return -(int(digest, 16) % 2147483647) + def _getScipionObjId(self, scipionObj: Any, path: str) -> Optional[int]: + return self._getSourceObjId(scipionObj) def _getObjectValueText(self, scipionObj: Any) -> Optional[str]: if self._isPointer(scipionObj): From b69b02b6742a373459df2b95c214e87c9bb2d783 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 12:01:21 +0200 Subject: [PATCH 014/278] Restore mapper license header --- app/backend/mapper/scipion_object_mapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/backend/mapper/scipion_object_mapper.py b/app/backend/mapper/scipion_object_mapper.py index 46db5f25..984b76f1 100644 --- a/app/backend/mapper/scipion_object_mapper.py +++ b/app/backend/mapper/scipion_object_mapper.py @@ -16,7 +16,7 @@ # * # * You should have received a copy of the GNU General Public License # * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, MA +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA # * 02111-1307 USA # * # * All comments concerning this program package may be sent to the From 1e103f3333450b8b473e9f0c16331d69c1a48518 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 12:09:20 +0200 Subject: [PATCH 015/278] Add PostgreSQL mapper for Scipion sets --- app/backend/mapper/scipion_set_mapper.py | 543 +++++++++++++++++++++++ 1 file changed, 543 insertions(+) create mode 100644 app/backend/mapper/scipion_set_mapper.py diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py new file mode 100644 index 00000000..c971d538 --- /dev/null +++ b/app/backend/mapper/scipion_set_mapper.py @@ -0,0 +1,543 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +import json +from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple + +import psycopg2.extras + +from app.backend.mapper.scipion_object_mapper import ScipionObjectPostgresqlMapper + + +SELF_LABEL = "self" + + +class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): + """Store Scipion SetOf... objects in PostgreSQL using a flat JSONB layout.""" + + def storeSet( + self, + projectId: int, + protocolDbId: int, + outputName: str, + scipionSet: Any, + registerType: bool = True, + batchSize: int = 1000, + ) -> Dict[str, Any]: + if not projectId: + raise ValueError("projectId is required") + if not protocolDbId: + raise ValueError("protocolDbId is required") + if not outputName: + raise ValueError("outputName is required") + if batchSize <= 0: + raise ValueError("batchSize must be greater than zero") + + if registerType: + self.registerObjectTypeFromObject( + scipionSet, + mapperKind="flat_set", + includeProperties=False, + classSchema={"storage": "flat_set"}, + ) + + itemIterator = iter(self._iterSetItems(scipionSet)) + firstItem = self._nextOrNone(itemIterator) + itemSchema = self._getItemSchema(firstItem) if firstItem is not None else {} + itemClassName = self._getItemClassName(firstItem, itemSchema) + columns = self._getSetColumns(itemSchema) + initialProperties = self._getSetProperties(scipionSet) + + storedPaths: List[str] = [] + with self.db.transaction(): + rootObjectId = self._storeObjectNode( + projectId=projectId, + protocolDbId=protocolDbId, + scipionObj=scipionSet, + name=outputName, + path=outputName, + parentObjectId=None, + storedPaths=storedPaths, + includeNestedProperties=False, + visited=set(), + ) + + setId = self._upsertSet( + projectId=projectId, + protocolDbId=protocolDbId, + objectId=rootObjectId, + outputName=outputName, + setClassName=self._getClassName(scipionSet) or scipionSet.__class__.__name__, + itemClassName=itemClassName, + properties=initialProperties, + ) + self._clearSetRows(setId) + self._insertSetColumns(setId, columns) + + itemsCount = 0 + if firstItem is not None: + itemsCount = self._insertSetItems( + setId=setId, + firstItem=firstItem, + remainingItems=itemIterator, + batchSize=batchSize, + ) + + finalProperties = dict(initialProperties) + finalProperties["columnsCount"] = len(columns) + finalProperties["itemsCount"] = itemsCount + self._updateSetProperties(setId, finalProperties) + self._insertSetProperties(setId, finalProperties) + + return { + "setId": setId, + "rootObjectId": rootObjectId, + "projectId": projectId, + "protocolDbId": protocolDbId, + "outputName": outputName, + "setClassName": self._getClassName(scipionSet), + "itemClassName": itemClassName, + "columnsCount": len(columns), + "itemsCount": itemsCount, + } + + def getStoredSet( + self, + projectId: int, + protocolDbId: int, + outputName: str, + limit: Optional[int] = None, + offset: int = 0, + ) -> Optional[Dict[str, Any]]: + storedSet = self.db.fetchOne( + """ + SELECT id, "projectId", "protocolDbId", "objectId", "outputName", + "setClassName", "itemClassName", properties, "createdAt", "updatedAt" + FROM scipion_sets + WHERE "projectId" = %s + AND "protocolDbId" = %s + AND "outputName" = %s + """, + (projectId, protocolDbId, outputName), + ) + if storedSet is None: + return None + + storedSet["columns"] = self.getStoredSetColumns(storedSet["id"]) + storedSet["setProperties"] = self.getStoredSetProperties(storedSet["id"]) + storedSet["items"] = self.getStoredSetItems(storedSet["id"], limit=limit, offset=offset) + return storedSet + + def listProtocolStoredSets(self, projectId: int, protocolDbId: int) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT id, "projectId", "protocolDbId", "objectId", "outputName", + "setClassName", "itemClassName", properties, "createdAt", "updatedAt" + FROM scipion_sets + WHERE "projectId" = %s + AND "protocolDbId" = %s + ORDER BY "outputName" ASC + """, + (projectId, protocolDbId), + ) + + def getStoredSetColumns(self, setId: int) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT id, "setId", "labelProperty", "columnName", "className", + "valueType", position, indexed + FROM scipion_set_columns + WHERE "setId" = %s + ORDER BY position ASC + """, + (setId,), + ) + + def getStoredSetProperties(self, setId: int) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT id, "setId", key, value + FROM scipion_set_properties + WHERE "setId" = %s + ORDER BY key ASC + """, + (setId,), + ) + + def getStoredSetItems(self, setId: int, limit: Optional[int] = None, offset: int = 0) -> List[Dict[str, Any]]: + if limit is None: + return self.db.fetchAll( + """ + SELECT id, "setId", "scipionItemId", enabled, label, comment, + creation, values, "createdAt", "updatedAt" + FROM scipion_set_items + WHERE "setId" = %s + ORDER BY "scipionItemId" ASC + """, + (setId,), + ) + + return self.db.fetchAll( + """ + SELECT id, "setId", "scipionItemId", enabled, label, comment, + creation, values, "createdAt", "updatedAt" + FROM scipion_set_items + WHERE "setId" = %s + ORDER BY "scipionItemId" ASC + LIMIT %s OFFSET %s + """, + (setId, limit, offset), + ) + + def _upsertSet( + self, + projectId: int, + protocolDbId: int, + objectId: int, + outputName: str, + setClassName: str, + itemClassName: str, + properties: Dict[str, Any], + ) -> int: + cur = self.db.execute( + """ + INSERT INTO scipion_sets ( + "projectId", "protocolDbId", "objectId", "outputName", + "setClassName", "itemClassName", properties + ) + VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb) + ON CONFLICT ON CONSTRAINT ux_scipion_sets_project_protocol_output + DO UPDATE SET + "objectId" = EXCLUDED."objectId", + "setClassName" = EXCLUDED."setClassName", + "itemClassName" = EXCLUDED."itemClassName", + properties = EXCLUDED.properties, + "updatedAt" = NOW() + RETURNING id + """, + ( + projectId, + protocolDbId, + objectId, + outputName, + setClassName, + itemClassName, + self._jsonParam(properties), + ), + commit=False, + ) + return int(cur.fetchone()["id"]) + + def _clearSetRows(self, setId: int) -> None: + for tableName in ("scipion_set_items", "scipion_set_properties", "scipion_set_columns"): + self.db.execute( + f'DELETE FROM {tableName} WHERE "setId" = %s', + (setId,), + commit=False, + ) + + def _insertSetColumns(self, setId: int, columns: List[Dict[str, Any]]) -> None: + for column in columns: + self.db.execute( + """ + INSERT INTO scipion_set_columns ( + "setId", "labelProperty", "columnName", "className", "valueType", position, indexed + ) + VALUES (%s, %s, %s, %s, %s, %s, %s) + """, + ( + setId, + column["labelProperty"], + column["columnName"], + column["className"], + column["valueType"], + column["position"], + column["indexed"], + ), + commit=False, + ) + + def _insertSetProperties(self, setId: int, properties: Dict[str, Any]) -> None: + for key, value in sorted(properties.items()): + self.db.execute( + """ + INSERT INTO scipion_set_properties ("setId", key, value) + VALUES (%s, %s, %s) + """, + (setId, str(key), self._stringifyPropertyValue(value)), + commit=False, + ) + + def _insertSetItems( + self, + setId: int, + firstItem: Any, + remainingItems: Iterator[Any], + batchSize: int, + ) -> int: + rows: List[Tuple[Any, ...]] = [] + itemsCount = 0 + + for item in self._chainFirst(firstItem, remainingItems): + itemId = self._getSourceObjId(item) + if itemId is None: + raise ValueError("Cannot store a Scipion set item without getObjId()/getId()") + + rows.append( + ( + setId, + itemId, + self._getItemEnabled(item), + self._getObjectLabel(item), + self._getObjectComment(item), + self._getObjectCreation(item), + self._jsonParam(self._getItemValues(item)), + ) + ) + itemsCount += 1 + + if len(rows) >= batchSize: + self._flushSetItems(rows) + rows = [] + + if rows: + self._flushSetItems(rows) + + return itemsCount + + def _flushSetItems(self, rows: List[Tuple[Any, ...]]) -> None: + psycopg2.extras.execute_values( + self.db.cursor, + """ + INSERT INTO scipion_set_items ( + "setId", "scipionItemId", enabled, label, comment, creation, values + ) + VALUES %s + ON CONFLICT ON CONSTRAINT ux_scipion_set_items_set_item + DO UPDATE SET + enabled = EXCLUDED.enabled, + label = EXCLUDED.label, + comment = EXCLUDED.comment, + creation = EXCLUDED.creation, + values = EXCLUDED.values, + "updatedAt" = NOW() + """, + rows, + template="(%s, %s, %s, %s, %s, %s, %s::jsonb)", + page_size=len(rows), + ) + + def _updateSetProperties(self, setId: int, properties: Dict[str, Any]) -> None: + self.db.execute( + """ + UPDATE scipion_sets + SET properties = %s::jsonb, + "updatedAt" = NOW() + WHERE id = %s + """, + (self._jsonParam(properties), setId), + commit=False, + ) + + def _iterSetItems(self, scipionSet: Any) -> Iterable[Any]: + iterItems = getattr(scipionSet, "iterItems", None) + if callable(iterItems): + return iterItems() + + try: + return iter(scipionSet) + except TypeError: + raise ValueError("scipionSet must provide iterItems() or be iterable") + + def _getItemSchema(self, item: Any) -> Dict[str, Any]: + return self._getObjDict(item, includeClass=True) + + def _getItemValues(self, item: Any) -> Dict[str, Any]: + values = self._getObjDict(item, includeClass=False) + if not values: + return {} + + return { + str(label): self._toJsonValue(value) + for label, value in values.items() + if str(label) != SELF_LABEL + } + + def _getObjDict(self, scipionObj: Any, includeClass: bool) -> Dict[str, Any]: + getter = getattr(scipionObj, "getObjDict", None) + if not callable(getter): + return {} + + try: + return dict(getter(includeClass=includeClass) or {}) + except TypeError: + try: + return dict(getter(includeClass) or {}) + except TypeError: + if includeClass: + return {} + return dict(getter() or {}) + except Exception: + return {} + + def _getSetColumns(self, itemSchema: Dict[str, Any]) -> List[Dict[str, Any]]: + columns = [] + position = 0 + + for label, rawValue in itemSchema.items(): + labelProperty = str(label) + if labelProperty == SELF_LABEL: + continue + + className = self._getSchemaClassName(rawValue) + columns.append( + { + "labelProperty": labelProperty, + "columnName": "c%02d" % position, + "className": className, + "valueType": self._getColumnValueType(className), + "position": position, + "indexed": False, + } + ) + position += 1 + + return columns + + def _getItemClassName(self, item: Any, itemSchema: Dict[str, Any]) -> str: + selfSchema = itemSchema.get(SELF_LABEL) + schemaClassName = self._getSchemaClassName(selfSchema) + if schemaClassName: + return schemaClassName + return self._getClassName(item) or item.__class__.__name__ if item is not None else "Unknown" + + def _getSchemaClassName(self, schemaValue: Any) -> Optional[str]: + if isinstance(schemaValue, (tuple, list)) and schemaValue: + return str(schemaValue[0]) if schemaValue[0] else None + if isinstance(schemaValue, dict): + className = schemaValue.get("className") or schemaValue.get("class_name") + return str(className) if className else None + return None + + def _getColumnValueType(self, className: Optional[str]) -> Optional[str]: + if className in ("Integer", "Long", "Boolean"): + return "integer" + if className in ("Float", "Decimal"): + return "float" + if className in ("String", "CsvList", "Pointer", "PointerList"): + return "text" + return className + + def _getSetProperties(self, scipionSet: Any) -> Dict[str, Any]: + properties: Dict[str, Any] = { + "className": self._getClassName(scipionSet), + "moduleName": self._getModuleName(scipionSet), + "baseClassName": self._getBaseClassName(scipionSet), + "scipionObjId": self._getSourceObjId(scipionSet), + } + + fileName = self._callOptionalGetter(scipionSet, "getFileName") + if fileName is not None: + properties["fileName"] = self._toJsonValue(fileName) + + streamState = self._callOptionalGetter(scipionSet, "getStreamState") + if streamState is not None: + properties["streamState"] = self._toJsonValue(streamState) + + for attrName, attrValue in self._getAttributesToStore(scipionSet): + if self._getAttributesToStore(attrValue): + continue + properties[str(attrName)] = self._toJsonValue(self._getObjectValueText(attrValue)) + + return {key: value for key, value in properties.items() if value is not None} + + def _getItemEnabled(self, item: Any) -> bool: + isEnabled = getattr(item, "isEnabled", None) + if not callable(isEnabled): + return True + try: + return bool(isEnabled()) + except Exception: + return True + + def _callOptionalGetter(self, scipionObj: Any, getterName: str) -> Any: + getter = getattr(scipionObj, getterName, None) + if not callable(getter): + return None + try: + return getter() + except Exception: + return None + + def _toJsonValue(self, value: Any) -> Any: + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if isinstance(value, (list, tuple)): + return [self._toJsonValue(item) for item in value] + if isinstance(value, dict): + return {str(key): self._toJsonValue(item) for key, item in value.items()} + + isoformat = getattr(value, "isoformat", None) + if callable(isoformat): + try: + return isoformat() + except Exception: + pass + + getter = getattr(value, "getObjValue", None) + if callable(getter): + try: + return self._toJsonValue(getter()) + except Exception: + pass + + getter = getattr(value, "get", None) + if callable(getter): + try: + return self._toJsonValue(getter()) + except Exception: + pass + + return str(value) + + def _stringifyPropertyValue(self, value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, (dict, list, tuple)): + return json.dumps(self._toJsonValue(value), ensure_ascii=False) + return str(value) + + def _nextOrNone(self, iterator: Iterator[Any]) -> Any: + try: + return next(iterator) + except StopIteration: + return None + + def _chainFirst(self, firstItem: Any, remainingItems: Iterator[Any]) -> Iterator[Any]: + yield firstItem + for item in remainingItems: + yield item From ea1a7756311c4194bf646ad9360c9dbb16b654ea Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 12:09:36 +0200 Subject: [PATCH 016/278] Export Scipion set PostgreSQL mapper --- app/backend/mapper/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/backend/mapper/__init__.py b/app/backend/mapper/__init__.py index beca5fd9..9851601f 100644 --- a/app/backend/mapper/__init__.py +++ b/app/backend/mapper/__init__.py @@ -24,3 +24,4 @@ # * # ****************************************************************************** from app.backend.mapper.scipion_object_mapper import ScipionObjectPostgresqlMapper # noqa: F401 +from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper # noqa: F401 From da919242ba2ebceb3f67abc8e7bedf9afbbd4956 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 12:11:55 +0200 Subject: [PATCH 017/278] Quote set item values column in SQL --- app/backend/mapper/scipion_set_mapper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index c971d538..667d54a2 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -191,7 +191,7 @@ def getStoredSetItems(self, setId: int, limit: Optional[int] = None, offset: int return self.db.fetchAll( """ SELECT id, "setId", "scipionItemId", enabled, label, comment, - creation, values, "createdAt", "updatedAt" + creation, "values", "createdAt", "updatedAt" FROM scipion_set_items WHERE "setId" = %s ORDER BY "scipionItemId" ASC @@ -202,7 +202,7 @@ def getStoredSetItems(self, setId: int, limit: Optional[int] = None, offset: int return self.db.fetchAll( """ SELECT id, "setId", "scipionItemId", enabled, label, comment, - creation, values, "createdAt", "updatedAt" + creation, "values", "createdAt", "updatedAt" FROM scipion_set_items WHERE "setId" = %s ORDER BY "scipionItemId" ASC @@ -332,7 +332,7 @@ def _flushSetItems(self, rows: List[Tuple[Any, ...]]) -> None: self.db.cursor, """ INSERT INTO scipion_set_items ( - "setId", "scipionItemId", enabled, label, comment, creation, values + "setId", "scipionItemId", enabled, label, comment, creation, "values" ) VALUES %s ON CONFLICT ON CONSTRAINT ux_scipion_set_items_set_item @@ -341,7 +341,7 @@ def _flushSetItems(self, rows: List[Tuple[Any, ...]]) -> None: label = EXCLUDED.label, comment = EXCLUDED.comment, creation = EXCLUDED.creation, - values = EXCLUDED.values, + "values" = EXCLUDED."values", "updatedAt" = NOW() """, rows, From 9f861be0e3328e896aa4f817a71d318ff3747c3b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 12:34:48 +0200 Subject: [PATCH 018/278] Resolve Scipion protocol ids before storing sets --- app/backend/mapper/scipion_set_mapper.py | 32 ++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index 667d54a2..f2d14559 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -55,6 +55,8 @@ def storeSet( if batchSize <= 0: raise ValueError("batchSize must be greater than zero") + protocolDbId = self._resolveProtocolDbId(projectId, protocolDbId) + if registerType: self.registerObjectTypeFromObject( scipionSet, @@ -211,6 +213,36 @@ def getStoredSetItems(self, setId: int, limit: Optional[int] = None, offset: int (setId, limit, offset), ) + def _resolveProtocolDbId(self, projectId: int, protocolDbId: int) -> int: + byDatabaseId = self.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE id = %s + AND "projectId" = %s + """, + (protocolDbId, projectId), + ) + if byDatabaseId is not None: + return int(byDatabaseId["id"]) + + byScipionId = self.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE "projectId" = %s + AND "protocolId" = %s + """, + (projectId, str(protocolDbId)), + ) + if byScipionId is not None: + return int(byScipionId["id"]) + + raise ValueError( + "Protocol %s was not found in PostgreSQL protocols table for project %s" + % (protocolDbId, projectId) + ) + def _upsertSet( self, projectId: int, From 9b98875ce0d00c5497c62d8d041709b0082db653 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 12:39:24 +0200 Subject: [PATCH 019/278] Resolve protocol ids when reading stored sets --- app/backend/mapper/scipion_set_mapper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index f2d14559..1f332cb6 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -133,6 +133,8 @@ def getStoredSet( limit: Optional[int] = None, offset: int = 0, ) -> Optional[Dict[str, Any]]: + protocolDbId = self._resolveProtocolDbId(projectId, protocolDbId) + storedSet = self.db.fetchOne( """ SELECT id, "projectId", "protocolDbId", "objectId", "outputName", @@ -153,6 +155,8 @@ def getStoredSet( return storedSet def listProtocolStoredSets(self, projectId: int, protocolDbId: int) -> List[Dict[str, Any]]: + protocolDbId = self._resolveProtocolDbId(projectId, protocolDbId) + return self.db.fetchAll( """ SELECT id, "projectId", "protocolDbId", "objectId", "outputName", From b200ae69a85836703b46920a3f63d57b47e03fe8 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 19 Jun 2026 13:02:11 +0200 Subject: [PATCH 020/278] add generic scipion output persistence helper --- app/backend/api/services/project_service.py | 176 ++++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index a9f634aa..56c28c75 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2632,6 +2632,182 @@ def listProjectThumbnailItems( inlineImages=inlineImages, ) + def registerOutput( + self, + projectId: int, + protocol: Any, + raiseOnError: bool = False, + ) -> List[Dict[str, Any]]: + """ + Persist all Scipion protocol outputs in PostgreSQL. + + This method does not special-case concrete output classes. It decides + how to persist each output by object capabilities: + + - Set-like Scipion outputs -> ScipionSetPostgresqlMapper + - Regular Scipion objects -> ScipionObjectPostgresqlMapper + + Protocols are not stored here; only protocol outputs. + """ + from app.backend.database import getMapper + from app.backend.mapper import ScipionObjectPostgresqlMapper, ScipionSetPostgresqlMapper + + results: List[Dict[str, Any]] = [] + + mapper = getMapper() + try: + protocolDbId = self._resolveProtocolDbIdForOutputPersistence( + mapper.db, + projectId, + protocol, + ) + + objectMapper = ScipionObjectPostgresqlMapper(mapper.db) + setMapper = ScipionSetPostgresqlMapper(mapper.db) + + for outputName, outputObj in protocol.iterOutputAttributes(): + if outputObj is None: + continue + + try: + if self._isScipionSetOutput(outputObj): + result = setMapper.storeSet( + projectId=projectId, + protocolDbId=protocolDbId, + outputName=outputName, + scipionSet=outputObj, + ) + result["mapperKind"] = "flat_set" + + elif self._isScipionObjectOutput(outputObj): + result = objectMapper.storeObjectTree( + projectId=projectId, + protocolDbId=protocolDbId, + outputName=outputName, + scipionObj=outputObj, + includeNestedProperties=True, + ) + result["mapperKind"] = "tree" + + else: + continue + + result["outputName"] = outputName + result["outputClassName"] = self._getOutputClassName(outputObj) + results.append(result) + + except Exception as exc: + if raiseOnError: + raise + + logger.warning( + "Could not persist Scipion output. projectId=%s protocolId=%s outputName=%s outputClass=%s error=%s", + projectId, + self._getScipionObjId(protocol), + outputName, + self._getOutputClassName(outputObj), + exc, + exc_info=True, + ) + + finally: + mapper.db.close() + + return results + + def _resolveProtocolDbIdForOutputPersistence(self, db, projectId: int, protocol: Any) -> int: + scipionProtocolId = self._getScipionObjId(protocol) + if scipionProtocolId is None: + raise ValueError("Cannot persist Scipion outputs without protocol getObjId()/getId()") + + row = db.fetchOne( + """ + SELECT id + FROM protocols + WHERE "projectId" = %s + AND "protocolId" = %s + """, + (projectId, str(scipionProtocolId)), + ) + if row is not None: + return int(row["id"]) + + row = db.fetchOne( + """ + SELECT id + FROM protocols + WHERE id = %s + AND "projectId" = %s + """, + (scipionProtocolId, projectId), + ) + if row is not None: + return int(row["id"]) + + raise ValueError( + "Protocol %s was not found in PostgreSQL protocols table for project %s" + % (scipionProtocolId, projectId) + ) + + def _isScipionSetOutput(self, outputObj: Any) -> bool: + className = self._getOutputClassName(outputObj) + + if className.startswith("SetOf"): + return True + + iterItems = getattr(outputObj, "iterItems", None) + if callable(iterItems): + return True + + return False + + def _isScipionObjectOutput(self, outputObj: Any) -> bool: + if outputObj is None: + return False + + getClassName = getattr(outputObj, "getClassName", None) + if callable(getClassName): + return True + + getAttributesToStore = getattr(outputObj, "getAttributesToStore", None) + if callable(getAttributesToStore): + return True + + return False + + def _getOutputClassName(self, outputObj: Any) -> str: + getClassName = getattr(outputObj, "getClassName", None) + if callable(getClassName): + try: + className = getClassName() + if className: + return str(className) + except Exception: + pass + + return outputObj.__class__.__name__ if outputObj is not None else "" + + def _getScipionObjId(self, obj: Any) -> Optional[int]: + for getterName in ("getObjId", "getId"): + getter = getattr(obj, getterName, None) + if not callable(getter): + continue + + try: + value = getter() + except Exception: + continue + + if value is None: + continue + + try: + return int(value) + except Exception: + continue + + return None + def _buildProtocolContext(self, projectId, protocol) -> dict: """ Build the common context dictionary for a protocol, From a2b99627a8c89cde779ec1bad2b25549e69a968e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 19 Jun 2026 13:28:44 +0200 Subject: [PATCH 021/278] register scipion outputs during protocol sync --- app/backend/api/services/project_service.py | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 56c28c75..446160ef 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -580,6 +580,25 @@ def _validateImportableScipionProject(self, sourcePath: Path) -> Dict[str, Any]: "status": statusValue or "active", } + def _shouldRegisterProtocolOutputs(self, protocol: Any) -> bool: + try: + outputs = list(protocol.iterOutputAttributes()) + except Exception: + return False + + if not outputs: + return False + + status = protocol.getStatus() + + if status == STATUS_FINISHED: + return True + + if status in (STATUS_LAUNCHED, STATUS_RUNNING, STATUS_SCHEDULED): + return True + + return False + def syncProjectProtocolsAndDependencies( self, mapper: PostgresqlFlatMapper, @@ -618,6 +637,9 @@ def syncProjectProtocolsAndDependencies( protocolContext = self._buildProtocolContext(projectId, protocol) protocolDbId = mapper.saveProtocol(protocolContext) + if self._shouldRegisterProtocolOutputs(protocol): + self.registerOutput(projectId, protocol) + currentProtocolIds.add(nodeIdText) protocolDbIdByScipionId[nodeIdText] = int(protocolDbId) From e012db675b8308ec583def13917942f51c4af543 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 13:33:19 +0200 Subject: [PATCH 022/278] Make Scipion set persistence incremental --- app/backend/mapper/scipion_set_mapper.py | 31 ++++++++++++------------ 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index 1f332cb6..73b1f41d 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -95,12 +95,11 @@ def storeSet( itemClassName=itemClassName, properties=initialProperties, ) - self._clearSetRows(setId) - self._insertSetColumns(setId, columns) + self._upsertSetColumns(setId, columns) itemsCount = 0 if firstItem is not None: - itemsCount = self._insertSetItems( + itemsCount = self._upsertSetItems( setId=setId, firstItem=firstItem, remainingItems=itemIterator, @@ -111,7 +110,7 @@ def storeSet( finalProperties["columnsCount"] = len(columns) finalProperties["itemsCount"] = itemsCount self._updateSetProperties(setId, finalProperties) - self._insertSetProperties(setId, finalProperties) + self._upsertSetProperties(setId, finalProperties) return { "setId": setId, @@ -286,15 +285,7 @@ def _upsertSet( ) return int(cur.fetchone()["id"]) - def _clearSetRows(self, setId: int) -> None: - for tableName in ("scipion_set_items", "scipion_set_properties", "scipion_set_columns"): - self.db.execute( - f'DELETE FROM {tableName} WHERE "setId" = %s', - (setId,), - commit=False, - ) - - def _insertSetColumns(self, setId: int, columns: List[Dict[str, Any]]) -> None: + def _upsertSetColumns(self, setId: int, columns: List[Dict[str, Any]]) -> None: for column in columns: self.db.execute( """ @@ -302,6 +293,13 @@ def _insertSetColumns(self, setId: int, columns: List[Dict[str, Any]]) -> None: "setId", "labelProperty", "columnName", "className", "valueType", position, indexed ) VALUES (%s, %s, %s, %s, %s, %s, %s) + ON CONFLICT ON CONSTRAINT ux_scipion_set_columns_set_label + DO UPDATE SET + "columnName" = EXCLUDED."columnName", + "className" = EXCLUDED."className", + "valueType" = EXCLUDED."valueType", + position = EXCLUDED.position, + indexed = EXCLUDED.indexed """, ( setId, @@ -315,18 +313,21 @@ def _insertSetColumns(self, setId: int, columns: List[Dict[str, Any]]) -> None: commit=False, ) - def _insertSetProperties(self, setId: int, properties: Dict[str, Any]) -> None: + def _upsertSetProperties(self, setId: int, properties: Dict[str, Any]) -> None: for key, value in sorted(properties.items()): self.db.execute( """ INSERT INTO scipion_set_properties ("setId", key, value) VALUES (%s, %s, %s) + ON CONFLICT ON CONSTRAINT ux_scipion_set_properties_set_key + DO UPDATE SET + value = EXCLUDED.value """, (setId, str(key), self._stringifyPropertyValue(value)), commit=False, ) - def _insertSetItems( + def _upsertSetItems( self, setId: int, firstItem: Any, From f7ac45c0b04023ecdf53516c5ba1f859a6904123 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 13:45:32 +0200 Subject: [PATCH 023/278] Track Scipion set sync metadata --- app/backend/mapper/scipion_set_mapper.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index 73b1f41d..fe892a17 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -24,6 +24,7 @@ # * # ****************************************************************************** import json +from datetime import datetime, timezone from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple import psycopg2.extras @@ -71,6 +72,7 @@ def storeSet( itemClassName = self._getItemClassName(firstItem, itemSchema) columns = self._getSetColumns(itemSchema) initialProperties = self._getSetProperties(scipionSet) + syncTimestamp = datetime.now(timezone.utc).isoformat() storedPaths: List[str] = [] with self.db.transaction(): @@ -98,8 +100,9 @@ def storeSet( self._upsertSetColumns(setId, columns) itemsCount = 0 + maxItemId = None if firstItem is not None: - itemsCount = self._upsertSetItems( + itemsCount, maxItemId = self._upsertSetItems( setId=setId, firstItem=firstItem, remainingItems=itemIterator, @@ -109,6 +112,9 @@ def storeSet( finalProperties = dict(initialProperties) finalProperties["columnsCount"] = len(columns) finalProperties["itemsCount"] = itemsCount + finalProperties["maxItemId"] = maxItemId + finalProperties["lastSyncAt"] = syncTimestamp + finalProperties["incremental"] = True self._updateSetProperties(setId, finalProperties) self._upsertSetProperties(setId, finalProperties) @@ -122,6 +128,8 @@ def storeSet( "itemClassName": itemClassName, "columnsCount": len(columns), "itemsCount": itemsCount, + "maxItemId": maxItemId, + "lastSyncAt": syncTimestamp, } def getStoredSet( @@ -333,15 +341,17 @@ def _upsertSetItems( firstItem: Any, remainingItems: Iterator[Any], batchSize: int, - ) -> int: + ) -> Tuple[int, Optional[int]]: rows: List[Tuple[Any, ...]] = [] itemsCount = 0 + maxItemId: Optional[int] = None for item in self._chainFirst(firstItem, remainingItems): itemId = self._getSourceObjId(item) if itemId is None: raise ValueError("Cannot store a Scipion set item without getObjId()/getId()") + maxItemId = itemId if maxItemId is None else max(maxItemId, itemId) rows.append( ( setId, @@ -362,7 +372,7 @@ def _upsertSetItems( if rows: self._flushSetItems(rows) - return itemsCount + return itemsCount, maxItemId def _flushSetItems(self, rows: List[Tuple[Any, ...]]) -> None: psycopg2.extras.execute_values( From f4f932d01b363d9553af95bd58ad01a65f6212e3 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 13:54:21 +0200 Subject: [PATCH 024/278] Skip unchanged Scipion set syncs --- app/backend/mapper/scipion_set_mapper.py | 164 ++++++++++++++++++++++- 1 file changed, 163 insertions(+), 1 deletion(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index fe892a17..5822f49e 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -24,6 +24,7 @@ # * # ****************************************************************************** import json +import os from datetime import datetime, timezone from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple @@ -57,6 +58,42 @@ def storeSet( raise ValueError("batchSize must be greater than zero") protocolDbId = self._resolveProtocolDbId(projectId, protocolDbId) + syncTimestamp = datetime.now(timezone.utc).isoformat() + itemsCountHint = self._getSetItemsCountHint(scipionSet) + maxItemIdHint = self._getSetMaxItemIdHint(scipionSet) + sourceMTime = self._getSetSourceMTime(scipionSet) + existingSet = self._getExistingSet(projectId, protocolDbId, outputName) + + if existingSet is not None: + existingProperties = self._normalizeProperties(existingSet.get("properties")) + if self._shouldSkipSetSync(existingProperties, itemsCountHint, maxItemIdHint, sourceMTime): + skippedProperties = dict(existingProperties) + skippedProperties["lastCheckedAt"] = syncTimestamp + skippedProperties["lastSkipReason"] = "unchanged_signature" + skippedProperties["skippedLastSync"] = True + skippedProperties["incremental"] = True + if sourceMTime is not None: + skippedProperties["sourceMTime"] = sourceMTime + + with self.db.transaction(): + self._updateSetProperties(int(existingSet["id"]), skippedProperties) + self._upsertSetProperties(int(existingSet["id"]), skippedProperties) + + return { + "setId": int(existingSet["id"]), + "rootObjectId": existingSet.get("objectId"), + "projectId": projectId, + "protocolDbId": protocolDbId, + "outputName": outputName, + "setClassName": existingSet.get("setClassName"), + "itemClassName": existingSet.get("itemClassName"), + "columnsCount": self._toOptionalInt(existingProperties.get("columnsCount")), + "itemsCount": self._toOptionalInt(existingProperties.get("itemsCount")), + "maxItemId": self._toOptionalInt(existingProperties.get("maxItemId")), + "lastSyncAt": existingProperties.get("lastSyncAt"), + "lastCheckedAt": syncTimestamp, + "skipped": True, + } if registerType: self.registerObjectTypeFromObject( @@ -72,7 +109,6 @@ def storeSet( itemClassName = self._getItemClassName(firstItem, itemSchema) columns = self._getSetColumns(itemSchema) initialProperties = self._getSetProperties(scipionSet) - syncTimestamp = datetime.now(timezone.utc).isoformat() storedPaths: List[str] = [] with self.db.transaction(): @@ -114,7 +150,12 @@ def storeSet( finalProperties["itemsCount"] = itemsCount finalProperties["maxItemId"] = maxItemId finalProperties["lastSyncAt"] = syncTimestamp + finalProperties["lastCheckedAt"] = syncTimestamp + finalProperties["lastSkipReason"] = None + finalProperties["skippedLastSync"] = False finalProperties["incremental"] = True + if sourceMTime is not None: + finalProperties["sourceMTime"] = sourceMTime self._updateSetProperties(setId, finalProperties) self._upsertSetProperties(setId, finalProperties) @@ -130,6 +171,8 @@ def storeSet( "itemsCount": itemsCount, "maxItemId": maxItemId, "lastSyncAt": syncTimestamp, + "lastCheckedAt": syncTimestamp, + "skipped": False, } def getStoredSet( @@ -254,6 +297,18 @@ def _resolveProtocolDbId(self, projectId: int, protocolDbId: int) -> int: % (protocolDbId, projectId) ) + def _getExistingSet(self, projectId: int, protocolDbId: int, outputName: str) -> Optional[Dict[str, Any]]: + return self.db.fetchOne( + """ + SELECT id, "objectId", "setClassName", "itemClassName", properties + FROM scipion_sets + WHERE "projectId" = %s + AND "protocolDbId" = %s + AND "outputName" = %s + """, + (projectId, protocolDbId, outputName), + ) + def _upsertSet( self, projectId: int, @@ -408,6 +463,113 @@ def _updateSetProperties(self, setId: int, properties: Dict[str, Any]) -> None: commit=False, ) + def _shouldSkipSetSync( + self, + existingProperties: Dict[str, Any], + itemsCountHint: Optional[int], + maxItemIdHint: Optional[int], + sourceMTime: Optional[float], + ) -> bool: + if not existingProperties or not existingProperties.get("incremental"): + return False + + if itemsCountHint is None: + return False + + storedItemsCount = self._toOptionalInt(existingProperties.get("itemsCount")) + if storedItemsCount != itemsCountHint: + return False + + storedMaxItemId = self._toOptionalInt(existingProperties.get("maxItemId")) + if maxItemIdHint is not None and storedMaxItemId is not None and storedMaxItemId != maxItemIdHint: + return False + + storedSourceMTime = self._toOptionalFloat(existingProperties.get("sourceMTime")) + if sourceMTime is not None and storedSourceMTime is not None and abs(storedSourceMTime - sourceMTime) > 0.000001: + return False + + return True + + def _getSetItemsCountHint(self, scipionSet: Any) -> Optional[int]: + for methodName in ("getSize", "count"): + getter = getattr(scipionSet, methodName, None) + if not callable(getter): + continue + try: + value = getter() + except Exception: + continue + countValue = self._toOptionalInt(value) + if countValue is not None: + return countValue + + try: + return int(len(scipionSet)) + except Exception: + return None + + def _getSetMaxItemIdHint(self, scipionSet: Any) -> Optional[int]: + for methodName in ("getMaxId", "maxId", "getLastId"): + getter = getattr(scipionSet, methodName, None) + if not callable(getter): + continue + try: + value = getter() + except Exception: + continue + maxValue = self._toOptionalInt(value) + if maxValue is not None: + return maxValue + + getter = getattr(scipionSet, "getLastItem", None) + if callable(getter): + try: + return self._getSourceObjId(getter()) + except Exception: + return None + + return None + + def _getSetSourceMTime(self, scipionSet: Any) -> Optional[float]: + fileName = self._callOptionalGetter(scipionSet, "getFileName") + if not fileName: + return None + + try: + filePath = str(fileName) + if not os.path.exists(filePath): + return None + return float(os.path.getmtime(filePath)) + except Exception: + return None + + def _normalizeProperties(self, properties: Any) -> Dict[str, Any]: + if isinstance(properties, dict): + return dict(properties) + if isinstance(properties, str): + try: + parsed = json.loads(properties) + return dict(parsed) if isinstance(parsed, dict) else {} + except Exception: + return {} + return {} + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except Exception: + return None + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + try: + return float(value) + except Exception: + return None + def _iterSetItems(self, scipionSet: Any) -> Iterable[Any]: iterItems = getattr(scipionSet, "iterItems", None) if callable(iterItems): From e49320c28c70536aa9324c510911d3c95d37c8f0 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 13:57:06 +0200 Subject: [PATCH 025/278] Require stored sync hints before skipping set sync --- app/backend/mapper/scipion_set_mapper.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index 5822f49e..a2e506f4 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -2,7 +2,7 @@ # * # * Authors: Yunior C. Fonseca Reyna # * -# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * Unidad de Bioinformatica of Centro Nacional de Bioinformatica , CSIC # * # * This program is free software; you can redistribute it and/or modify # * it under the terms of the GNU General Public License as published by @@ -481,12 +481,18 @@ def _shouldSkipSetSync( return False storedMaxItemId = self._toOptionalInt(existingProperties.get("maxItemId")) - if maxItemIdHint is not None and storedMaxItemId is not None and storedMaxItemId != maxItemIdHint: - return False + if maxItemIdHint is not None: + if storedMaxItemId is None: + return False + if storedMaxItemId != maxItemIdHint: + return False storedSourceMTime = self._toOptionalFloat(existingProperties.get("sourceMTime")) - if sourceMTime is not None and storedSourceMTime is not None and abs(storedSourceMTime - sourceMTime) > 0.000001: - return False + if sourceMTime is not None: + if storedSourceMTime is None: + return False + if abs(storedSourceMTime - sourceMTime) > 0.000001: + return False return True From 28d03a61a06de55f95c3184bed8fafcba4a855f2 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" <34970661+fonsecareyna82@users.noreply.github.com> Date: Fri, 19 Jun 2026 14:01:12 +0200 Subject: [PATCH 026/278] Fix Scipion set mapper header --- app/backend/mapper/scipion_set_mapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index a2e506f4..bddc3316 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -2,7 +2,7 @@ # * # * Authors: Yunior C. Fonseca Reyna # * -# * Unidad de Bioinformatica of Centro Nacional de Bioinformatica , CSIC +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC # * # * This program is free software; you can redistribute it and/or modify # * it under the terms of the GNU General Public License as published by From 971c4661a50fadbbed374a773255721f610d69da Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 19 Jun 2026 15:27:58 +0200 Subject: [PATCH 027/278] Resync protocol outputs when run status changes --- app/backend/api/routers/project_router.py | 2 +- app/backend/api/services/project_service.py | 47 ++++++++++++++++++++- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 31cccff0..66e519df 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -320,7 +320,7 @@ def loadProtocols( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) + project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=True) if not project: raise HTTPException(status_code=404, detail="Project not found") diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 446160ef..ca275d59 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -1555,6 +1555,18 @@ def loadProject(self, dbProj: dict, mapper: PostgresqlFlatMapper = None, refresh scipionProtocolCount = 0 scipionEdgeCount = 0 + liveProtocolStatusById: Dict[str, str] = {} + activeOutputProtocolIds: Set[str] = set() + + def normalizeStatus(value: Any) -> str: + return str(value or "").strip().lower() + + activeOutputStatuses = { + normalizeStatus(STATUS_LAUNCHED), + normalizeStatus(STATUS_RUNNING), + normalizeStatus(STATUS_SCHEDULED), + } + try: runs = self.currentProject.getRunsGraph(refresh=refresh, checkPids=checkPid) nodesDict = getattr(runs, "_nodesDict", {}) or {} @@ -1564,7 +1576,24 @@ def loadProject(self, dbProj: dict, mapper: PostgresqlFlatMapper = None, refresh continue scipionProtocolCount += 1 - runMap[str(nodeId)] = getattr(nodeObj, "run", None) + protocol = getattr(nodeObj, "run", None) + runMap[str(nodeId)] = protocol + + if protocol is not None: + try: + liveStatus = protocol.getStatus() + liveProtocolStatusById[str(nodeId)] = normalizeStatus(liveStatus) + except Exception: + liveStatus = None + + try: + if ( + normalizeStatus(liveStatus) in activeOutputStatuses + and self._shouldRegisterProtocolOutputs(protocol) + ): + activeOutputProtocolIds.add(str(nodeId)) + except Exception: + pass for parent in getattr(nodeObj, "_parents", []) or []: parentNodeId = str(parent.getName()) @@ -1615,9 +1644,23 @@ def loadProject(self, dbProj: dict, mapper: PostgresqlFlatMapper = None, refresh dbProtocolCount = len(protocolRows) dbEdgeCount = sum(len(v.get("parents") or []) for v in dependencyMap.values()) + dbStatusByProtocolId = { + str(row.get("protocolId")): normalizeStatus(row.get("status")) + for row in protocolRows + if row.get("protocolId") is not None + } + + statusChangedProtocolIds = [ + protocolId + for protocolId, liveStatus in liveProtocolStatusById.items() + if dbStatusByProtocolId.get(protocolId) != liveStatus + ] + shouldResyncGraph = ( scipionProtocolCount != dbProtocolCount or - scipionEdgeCount != dbEdgeCount + scipionEdgeCount != dbEdgeCount or + bool(statusChangedProtocolIds) or + bool(activeOutputProtocolIds) ) if shouldResyncGraph: From 4b0167640c08ad12fb8902f9f8fca6d2bb6a9a07 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 19 Jun 2026 16:15:38 +0200 Subject: [PATCH 028/278] include persisted outputs in protocol graph --- app/backend/api/services/project_service.py | 153 ++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index ca275d59..05d9c487 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -1303,6 +1303,116 @@ def _buildProjectThumbnailVersion( return f"{projectId}:{updatedText}:{protocolsCount}:{runsMtime}" + def _loadPersistedOutputsByProtocolId( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + ) -> Dict[str, Dict[str, Dict[str, Any]]]: + def toOptionalInt(value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except Exception: + return None + + result: Dict[str, Dict[str, Dict[str, Any]]] = {} + + setRows = mapper.db.fetchAll( + """ + SELECT + p."protocolId", + s.id, + s."objectId", + s."outputName", + s."setClassName", + s."itemClassName", + s.properties, + s."createdAt", + s."updatedAt" + FROM scipion_sets s + JOIN protocols p + ON p.id = s."protocolDbId" + WHERE s."projectId" = %s + ORDER BY p."protocolId", s."outputName" + """, + (projectId,), + ) + + for row in setRows: + protocolId = str(row.get("protocolId")) + outputName = str(row.get("outputName") or "") + if not protocolId or not outputName: + continue + + properties = row.get("properties") or {} + + result.setdefault(protocolId, {})[outputName] = { + "mapperKind": "flat_set", + "setId": row.get("id"), + "rootObjectId": row.get("objectId"), + "className": row.get("setClassName"), + "itemClassName": row.get("itemClassName"), + "itemsCount": toOptionalInt(properties.get("itemsCount")) if isinstance(properties, dict) else None, + "maxItemId": toOptionalInt(properties.get("maxItemId")) if isinstance(properties, dict) else None, + "lastSyncAt": properties.get("lastSyncAt") if isinstance(properties, dict) else None, + "lastCheckedAt": properties.get("lastCheckedAt") if isinstance(properties, dict) else None, + "skippedLastSync": properties.get("skippedLastSync") if isinstance(properties, dict) else None, + "createdAt": row.get("createdAt"), + "updatedAt": row.get("updatedAt"), + } + + treeRows = mapper.db.fetchAll( + """ + SELECT + p."protocolId", + o.id, + o."scipionObjId", + o.name, + o.path, + o."className", + o.value, + o.label, + o.comment, + o.metadata, + o."createdAt", + o."updatedAt" + FROM scipion_objects o + JOIN protocols p + ON p.id = o."protocolDbId" + WHERE o."projectId" = %s + AND o."parentObjectId" IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM scipion_sets s + WHERE s."objectId" = o.id + ) + ORDER BY p."protocolId", o.path + """, + (projectId,), + ) + + for row in treeRows: + protocolId = str(row.get("protocolId")) + outputName = str(row.get("path") or row.get("name") or "") + if not protocolId or not outputName: + continue + + result.setdefault(protocolId, {})[outputName] = { + "mapperKind": "tree", + "rootObjectId": row.get("id"), + "scipionObjId": row.get("scipionObjId"), + "className": row.get("className"), + "value": row.get("value"), + "label": row.get("label"), + "comment": row.get("comment"), + "metadata": row.get("metadata") or {}, + "createdAt": row.get("createdAt"), + "updatedAt": row.get("updatedAt"), + } + + return result + def buildProtocolsGraph( self, projectId: int, @@ -1310,11 +1420,13 @@ def buildProtocolsGraph( tags: Dict[str, List[str]], dependencyMap: Optional[Dict[str, Dict[str, List[str]]]] = None, runMap: Optional[Dict[str, Any]] = None, + persistedOutputsByProtocolId: Optional[Dict[str, Dict[str, Dict[str, Any]]]] = None, ) -> dict: """Assemble protocol graph using PostgreSQL as source of truth for nodes + edges.""" graphData: Dict[str, Any] = {} adjacency = dependencyMap or {} liveRuns = runMap or {} + persistedOutputsByProtocolId = persistedOutputsByProtocolId or {} def sortKey(row: Dict[str, Any]): raw = str(row.get("protocolId") or "") @@ -1371,6 +1483,7 @@ def sortKey(row: Dict[str, Any]): continue nodeId = str(rawNodeId) + persistedOutputsByName = persistedOutputsByProtocolId.get(nodeId, {}) nodeDeps = adjacency.get(nodeId, {"parents": [], "children": []}) childrenIds = list(nodeDeps.get("children") or []) parentIds = list(nodeDeps.get("parents") or []) @@ -1491,7 +1604,12 @@ def sortKey(row: Dict[str, Any]): inputs = [] try: + seenOutputNames = set() + for key, attr in protocol.iterOutputAttributes(): + outputName = str(key) + seenOutputNames.add(outputName) + outputItem = {} outputItem["name"] = key outputItem["paramClass"] = "PointerParam" @@ -1509,7 +1627,27 @@ def sortKey(row: Dict[str, Any]): outputItem["value"] = "" outputItem["parentId"] = None + persistedOutput = persistedOutputsByName.get(outputName) + outputItem["persisted"] = bool(persistedOutput) + outputItem["persistence"] = persistedOutput + outputs.append(outputItem) + + for outputName, persistedOutput in persistedOutputsByName.items(): + if outputName in seenOutputNames: + continue + + outputs.append({ + "name": outputName, + "paramClass": "PointerParam", + "pointerClass": persistedOutput.get("className") or "", + "info": "", + "value": "%s.%s" % (nodeId, outputName), + "parentId": nodeId, + "persisted": True, + "persistence": persistedOutput, + }) + except Exception: outputs = [] else: @@ -1690,12 +1828,27 @@ def normalizeStatus(value: Any) -> str: dbProj['id'], ) + persistedOutputsByProtocolId = {} + if mapper is not None: + try: + persistedOutputsByProtocolId = self._loadPersistedOutputsByProtocolId( + mapper, + dbProj['id'], + ) + except Exception: + logger.exception( + "Failed to load persisted Scipion outputs for graph. projectId=%s", + dbProj['id'], + ) + persistedOutputsByProtocolId = {} + graphData = self.buildProtocolsGraph( dbProj['id'], protocolRows, tags, dependencyMap=dependencyMap, runMap=runMap, + persistedOutputsByProtocolId=persistedOutputsByProtocolId, ) stats = projPath.stat() From 7120f2c0e94140be1dc429ede1037646f3b665b4 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 19 Jun 2026 18:00:12 +0200 Subject: [PATCH 029/278] serve persisted scipion sets through metadata endpoints --- app/backend/api/services/project_service.py | 59 +++ app/backend/viewers/__init__.py | 25 ++ app/backend/viewers/postgresql_dao.py | 409 ++++++++++++++++++++ 3 files changed, 493 insertions(+) create mode 100644 app/backend/viewers/__init__.py create mode 100644 app/backend/viewers/postgresql_dao.py diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 05d9c487..7d2bc561 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -8541,6 +8541,18 @@ def listOutputMetadataTablesService(self, projectId: int, """ List logical metadata tables associated with an output. """ + pgDao = self._getPersistedMetadataDAOIfAvailable( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + if pgDao is not None: + return pgDao.listTables( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + with _metadataLock: _, _, metaPath = self._resolveOutputForMetadata(protocolId, outputName) objMgr = self._getMetadataObjectManager(metaPath) @@ -8567,6 +8579,24 @@ def listOutputMetadataTablesService(self, projectId: int, return items + def _getPostgresqlMetadataDAO(self): + from app.backend.database import getMapper + from app.backend.viewers.postgresql_dao import PostgresqlMetadataDAO + + mapper = getMapper() + return PostgresqlMetadataDAO(mapper.db) + + def _getPersistedMetadataDAOIfAvailable( + self, + projectId: int, + protocolId: int, + outputName: str, + ): + dao = self._getPostgresqlMetadataDAO() + if dao.hasOutput(projectId, protocolId, outputName): + return dao + return None + def getMetadataTableSchemaService(self, projectId: int, protocolId: int, outputName: str, @@ -8574,6 +8604,18 @@ def getMetadataTableSchemaService(self, projectId: int, """ Return logical schema for one metadata table: columns, renderers, flags. """ + pgDao = self._getPersistedMetadataDAOIfAvailable( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + if pgDao is not None and pgDao.canServeTable(tableName): + return pgDao.getSchema( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + tableName=tableName, + ) with _metadataLock: objMgr, table = self._openMetadataTable(protocolId, outputName, tableName) @@ -9324,6 +9366,23 @@ def getMetadataTableWindowService( - full table if selectionOnly == False - number of selected rows if selectionOnly == True """ + pgDao = self._getPersistedMetadataDAOIfAvailable( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + if pgDao is not None and pgDao.canServeTable(tableName): + return pgDao.getWindow( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + tableName=tableName, + offset=offset, + limit=limit, + selectionOnly=selectionOnly, + sortBy=sortBy, + asc=asc, + ) with _metadataLock: objMgr, table = self._openMetadataTable(protocolId, outputName, tableName) columns = list(table.getColumns()) diff --git a/app/backend/viewers/__init__.py b/app/backend/viewers/__init__.py new file mode 100644 index 00000000..4cdb527e --- /dev/null +++ b/app/backend/viewers/__init__.py @@ -0,0 +1,25 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** diff --git a/app/backend/viewers/postgresql_dao.py b/app/backend/viewers/postgresql_dao.py new file mode 100644 index 00000000..9594f61b --- /dev/null +++ b/app/backend/viewers/postgresql_dao.py @@ -0,0 +1,409 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + + +# ******************************************************************************* +# * PostgreSQL metadata DAO for persisted Scipion sets. +# * +# * This DAO adapts Scipion sets persisted in PostgreSQL to the metadata-table +# * response shape already consumed by the web metadata viewer. +# ******************************************************************************* + +from typing import Any, Dict, List, Optional + +from fastapi import HTTPException, status + +from app.backend.mapper import ScipionSetPostgresqlMapper + + +class PostgresqlMetadataDAO: + """ + Read persisted Scipion SetOf... outputs from PostgreSQL and expose them with + the same API shape used by the metadata viewer endpoints. + + This class intentionally hides PostgreSQL persistence details from + ProjectService. + """ + + OBJECTS_TABLE = "objects" + SOURCE = "postgresql" + + def __init__(self, db): + self.db = db + self.setMapper = ScipionSetPostgresqlMapper(db) + + # ------------------------------------------------------------------------- + # Public API used by ProjectService + # ------------------------------------------------------------------------- + + def hasOutput(self, projectId: int, protocolId: int, outputName: str) -> bool: + try: + storedSet = self._getStoredSet( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + limit=0, + offset=0, + ) + return storedSet is not None + except Exception: + return False + + def canServeTable(self, tableName: str) -> bool: + return str(tableName or "").strip() == self.OBJECTS_TABLE + + def listTables( + self, + projectId: int, + protocolId: int, + outputName: str, + ) -> List[Dict[str, Any]]: + storedSet = self._requireStoredSet( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + limit=0, + offset=0, + ) + + rowCount = self._getStoredSetItemsCount(storedSet) + + return [ + { + "name": self.OBJECTS_TABLE, + "alias": storedSet.get("setClassName") or outputName, + "rowCount": int(rowCount), + "hasColumnId": True, + "persisted": True, + "source": self.SOURCE, + "setId": storedSet.get("id"), + "setClassName": storedSet.get("setClassName"), + "itemClassName": storedSet.get("itemClassName"), + } + ] + + def getSchema( + self, + projectId: int, + protocolId: int, + outputName: str, + tableName: str, + ) -> Dict[str, Any]: + self._requireObjectsTable(tableName) + + storedSet = self._requireStoredSet( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + limit=0, + offset=0, + ) + + columns = self._normalizeColumns(storedSet.get("columns") or []) + + return { + "name": self.OBJECTS_TABLE, + "alias": storedSet.get("setClassName") or outputName, + "hasColumnId": True, + "actions": [], + "columns": [ + self._columnToSchemaItem(column, index) + for index, column in enumerate(columns) + ], + "persisted": True, + "source": self.SOURCE, + "setId": storedSet.get("id"), + "setClassName": storedSet.get("setClassName"), + "itemClassName": storedSet.get("itemClassName"), + } + + def getWindow( + self, + projectId: int, + protocolId: int, + outputName: str, + tableName: str, + offset: int, + limit: int, + selectionOnly: bool = False, + sortBy: str = "id", + asc: bool = True, + ) -> Dict[str, Any]: + self._requireObjectsTable(tableName) + + offset = max(0, int(offset or 0)) + limit = max(1, int(limit or 1)) + + if selectionOnly: + return { + "offset": offset, + "limit": limit, + "totalRows": 0, + "rows": [], + "persisted": True, + "source": self.SOURCE, + } + + storedSet = self._requireStoredSet( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + limit=limit, + offset=offset, + ) + + columns = self._normalizeColumns(storedSet.get("columns") or []) + items = storedSet.get("items") or [] + totalRows = self._getStoredSetItemsCount(storedSet) + + resultRows = [] + for localIndex, item in enumerate(items): + globalIndex = offset + localIndex + valuesPayload = self._itemToValuesPayload(item, columns) + + logicalId = item.get("scipionItemId") + if logicalId is None: + logicalId = item.get("id") + + resultRows.append( + { + "id": globalIndex, + "index": globalIndex, + "rowId": logicalId, + "values": valuesPayload, + "enabled": bool(item.get("enabled", True)), + } + ) + + return { + "offset": offset, + "limit": limit, + "totalRows": int(totalRows), + "rows": resultRows, + "persisted": True, + "source": self.SOURCE, + "setId": storedSet.get("id"), + } + + # ------------------------------------------------------------------------- + # Stored set resolution + # ------------------------------------------------------------------------- + + def _getStoredSet( + self, + projectId: int, + protocolId: int, + outputName: str, + limit: Optional[int], + offset: int, + ) -> Optional[Dict[str, Any]]: + return self.setMapper.getStoredSet( + projectId=projectId, + protocolDbId=protocolId, + outputName=outputName, + limit=limit, + offset=offset, + ) + + def _requireStoredSet( + self, + projectId: int, + protocolId: int, + outputName: str, + limit: Optional[int], + offset: int, + ) -> Dict[str, Any]: + try: + storedSet = self._getStoredSet( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + limit=limit, + offset=offset, + ) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Persisted Scipion set not found: %s" % exc, + ) + + if storedSet is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Persisted Scipion set not found", + ) + + return storedSet + + def _requireObjectsTable(self, tableName: str) -> None: + if not self.canServeTable(tableName): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Persisted metadata table '%s' not found" % tableName, + ) + + # ------------------------------------------------------------------------- + # Shape adapters + # ------------------------------------------------------------------------- + + def _normalizeColumns(self, columns: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + def sortKey(column: Dict[str, Any]): + try: + return int(column.get("position") or 0) + except Exception: + return 0 + + return sorted(columns or [], key=sortKey) + + def _columnToSchemaItem( + self, + column: Dict[str, Any], + index: int, + ) -> Dict[str, Any]: + rawName = ( + column.get("labelProperty") + or column.get("columnName") + or "column_%s" % index + ) + name = str(rawName) + + return { + "name": name, + "alias": self._getColumnAlias(name), + "index": index, + "sortable": bool(column.get("indexed", False)) or name in ("id", "_objId"), + "visible": True, + "rendererType": self._getRendererType(column), + "decimals": self._getDecimals(column), + "hasTransformation": False, + "persisted": True, + "source": self.SOURCE, + } + + def _itemToValuesPayload( + self, + item: Dict[str, Any], + columns: List[Dict[str, Any]], + ) -> List[Any]: + values = item.get("values") or {} + if not isinstance(values, dict): + values = {} + + payload = [] + for column in columns: + label = column.get("labelProperty") + value = values.get(label) + + if value is None and label is not None: + value = values.get(str(label).strip()) + + payload.append(self._normalizeCellValue(value)) + + return payload + + def _normalizeCellValue(self, value: Any) -> Any: + if value is None: + return None + + if isinstance(value, (str, int, float, bool)): + return value + + if isinstance(value, list): + return [self._normalizeCellValue(item) for item in value] + + if isinstance(value, tuple): + return [self._normalizeCellValue(item) for item in value] + + if isinstance(value, dict): + return { + str(key): self._normalizeCellValue(item) + for key, item in value.items() + } + + return str(value) + + # ------------------------------------------------------------------------- + # Metadata helpers + # ------------------------------------------------------------------------- + + def _getStoredSetItemsCount(self, storedSet: Dict[str, Any]) -> int: + properties = storedSet.get("properties") or {} + + if isinstance(properties, dict): + count = self._toInt(properties.get("itemsCount")) + if count is not None: + return count + + for prop in storedSet.get("setProperties") or []: + if str(prop.get("key")) == "itemsCount": + count = self._toInt(prop.get("value")) + if count is not None: + return count + + return 0 + + def _getColumnAlias(self, name: str) -> str: + clean = str(name or "").strip() + if clean.startswith("_"): + clean = clean[1:] + return clean or str(name) + + def _getRendererType(self, column: Dict[str, Any]) -> str: + valueType = str(column.get("valueType") or "").lower() + className = str(column.get("className") or "").lower() + + text = "%s %s" % (valueType, className) + + if "bool" in text: + return "bool" + + if "float" in text or "decimal" in text: + return "float" + + if "integer" in text or "long" in text or className == "int": + return "int" + + if "matrix" in text: + return "matrix" + + if "image" in text or "filename" in text or "file" in text: + return "image" + + return "str" + + def _getDecimals(self, column: Dict[str, Any]) -> Optional[int]: + rendererType = self._getRendererType(column) + if rendererType == "float": + return 2 + return None + + def _toInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except Exception: + return None \ No newline at end of file From a430ab2ed212a050bd99a85ba38f0063bad7b6ed Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 19 Jun 2026 19:44:17 +0200 Subject: [PATCH 030/278] use object manager for persisted scipion metadata --- app/backend/api/services/project_service.py | 143 ++-- app/backend/viewers/postgresql_dao.py | 800 +++++++++++++------- 2 files changed, 606 insertions(+), 337 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 7d2bc561..918865c5 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -8461,18 +8461,81 @@ def _getMetadataObjectManager(self, metaPath: str) -> ObjectManager: objMgr.getTables() return objMgr - def _openMetadataTable(self, protocolId: int, outputName: str, tableName: str): + def _getPostgresqlDAOIfAvailable( + self, + projectId: int, + protocolId: int, + outputName: str, + ): + from app.backend.database import getMapper + from app.backend.viewers.postgresql_dao import PostgresqlDAO + + mapper = getMapper() + dao = PostgresqlDAO( + db=mapper.db, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if dao.hasOutput(): + return dao + + return None + + def _getMetadataObjectManagerForOutput( + self, + projectId: int, + protocolId: int, + outputName: str, + ) -> ObjectManager: + pgDao = self._getPostgresqlDAOIfAvailable( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgDao is not None: + objMgr = self._createObjectManager() + objMgr._fileName = "postgresql://project/%s/protocol/%s/output/%s" % ( + projectId, + protocolId, + outputName, + ) + objMgr._dao = pgDao + objMgr._tables = {} + objMgr.getTables() + return objMgr + + _, _, metaPath = self._resolveOutputForMetadata(protocolId, outputName) + return self._getMetadataObjectManager(metaPath) + + def _openMetadataTable( + self, + projectId: int, + protocolId: int, + outputName: str, + tableName: str, + ): """ Resolve (ObjectManager, Table) for a given output + tableName. + + PostgreSQL persisted outputs and SQLite/STAR outputs are both exposed + through ObjectManager. """ - _, _, metaPath = self._resolveOutputForMetadata(protocolId, outputName) - objMgr = self._getMetadataObjectManager(metaPath) + objMgr = self._getMetadataObjectManagerForOutput( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + table = objMgr.getTable(tableName) if table is None: raise HTTPException( status_code=404, - detail=f"Metadata table '{tableName}' not found" + detail=f"Metadata table '{tableName}' not found", ) + return objMgr, table def _rendererTypeFromInstance(self, renderer) -> str: @@ -8541,29 +8604,22 @@ def listOutputMetadataTablesService(self, projectId: int, """ List logical metadata tables associated with an output. """ - pgDao = self._getPersistedMetadataDAOIfAvailable( - projectId=projectId, - protocolId=protocolId, - outputName=outputName, - ) - if pgDao is not None: - return pgDao.listTables( + with _metadataLock: + objMgr = self._getMetadataObjectManagerForOutput( projectId=projectId, protocolId=protocolId, outputName=outputName, ) - with _metadataLock: - _, _, metaPath = self._resolveOutputForMetadata(protocolId, outputName) - objMgr = self._getMetadataObjectManager(metaPath) - tables = objMgr.getTables() or {} items = [] + for name, table in tables.items(): try: rowCount = objMgr.getTableRowCount(name) or 0 except Exception: rowCount = 0 + hasColumnId = True try: hasColumnId = table.hasColumnId() @@ -8579,24 +8635,6 @@ def listOutputMetadataTablesService(self, projectId: int, return items - def _getPostgresqlMetadataDAO(self): - from app.backend.database import getMapper - from app.backend.viewers.postgresql_dao import PostgresqlMetadataDAO - - mapper = getMapper() - return PostgresqlMetadataDAO(mapper.db) - - def _getPersistedMetadataDAOIfAvailable( - self, - projectId: int, - protocolId: int, - outputName: str, - ): - dao = self._getPostgresqlMetadataDAO() - if dao.hasOutput(projectId, protocolId, outputName): - return dao - return None - def getMetadataTableSchemaService(self, projectId: int, protocolId: int, outputName: str, @@ -8604,20 +8642,8 @@ def getMetadataTableSchemaService(self, projectId: int, """ Return logical schema for one metadata table: columns, renderers, flags. """ - pgDao = self._getPersistedMetadataDAOIfAvailable( - projectId=projectId, - protocolId=protocolId, - outputName=outputName, - ) - if pgDao is not None and pgDao.canServeTable(tableName): - return pgDao.getSchema( - projectId=projectId, - protocolId=protocolId, - outputName=outputName, - tableName=tableName, - ) with _metadataLock: - objMgr, table = self._openMetadataTable(protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable(projectId, protocolId, outputName, tableName) visibleLabels = [] orderLabels = [] @@ -8808,7 +8834,7 @@ def getMetadataTablePageService( Sorting is currently delegated to the underlying DAO default order. """ with _metadataLock: - objMgr, table = self._openMetadataTable(protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable(projectId, protocolId, outputName, tableName) columns = list(table.getColumns()) if selectionOnly: @@ -8897,7 +8923,7 @@ def exportMetadataTableService( import csv with _metadataLock: - objMgr, table = self._openMetadataTable(protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable(projectId, protocolId, outputName, tableName) columns = list(table.getColumns()) colNames = [c.getName() for c in columns] @@ -9097,7 +9123,7 @@ def renderMetadataImageCellService( except Exception: metaDir = None - objMgr, table = self._openMetadataTable(protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable(projectId, protocolId, outputName, tableName) columns = list(table.getColumns()) # Resolve column index @@ -9366,25 +9392,8 @@ def getMetadataTableWindowService( - full table if selectionOnly == False - number of selected rows if selectionOnly == True """ - pgDao = self._getPersistedMetadataDAOIfAvailable( - projectId=projectId, - protocolId=protocolId, - outputName=outputName, - ) - if pgDao is not None and pgDao.canServeTable(tableName): - return pgDao.getWindow( - projectId=projectId, - protocolId=protocolId, - outputName=outputName, - tableName=tableName, - offset=offset, - limit=limit, - selectionOnly=selectionOnly, - sortBy=sortBy, - asc=asc, - ) with _metadataLock: - objMgr, table = self._openMetadataTable(protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable(projectId, protocolId, outputName, tableName) columns = list(table.getColumns()) table.setSortingColumn(sortBy) table.setSortingAsc(asc) diff --git a/app/backend/viewers/postgresql_dao.py b/app/backend/viewers/postgresql_dao.py index 9594f61b..7b32aa9e 100644 --- a/app/backend/viewers/postgresql_dao.py +++ b/app/backend/viewers/postgresql_dao.py @@ -24,332 +24,568 @@ # * # ****************************************************************************** - -# ******************************************************************************* -# * PostgreSQL metadata DAO for persisted Scipion sets. +# ****************************************************************************** +# * PostgreSQL DAO for persisted Scipion sets. +# * +# * This DAO implements the metadata-viewer DAO contract so PostgreSQL can be +# * consumed through ObjectManager just like SQLite/STAR metadata sources. # * -# * This DAO adapts Scipion sets persisted in PostgreSQL to the metadata-table -# * response shape already consumed by the web metadata viewer. -# ******************************************************************************* +# ****************************************************************************** + +import logging +from typing import Any, Dict, Iterable, List, Optional + +import numpy -from typing import Any, Dict, List, Optional +from metadataviewer.dao.model import IDAO +from metadataviewer.model import ( + Table, + Column, + BoolRenderer, + FloatRenderer, + ImageRenderer, + StrRenderer, +) -from fastapi import HTTPException, status +from pwem.convert.transformations import euler_from_matrix from app.backend.mapper import ScipionSetPostgresqlMapper -class PostgresqlMetadataDAO: - """ - Read persisted Scipion SetOf... outputs from PostgreSQL and expose them with - the same API shape used by the metadata viewer endpoints. +logger = logging.getLogger(__name__) + +ALLOWED_COLUMNS_TYPES = [ + "String", + "Float", + "Integer", + "Boolean", + "Matrix", + "CsvList", +] + +EXCLUDED_COLUMNS = ["label", "comment", "creation", "_streamState"] +PERMANENT_COLUMNS = ["id", "enabled"] + +OBJECT_TABLE = "objects" +ENABLED_COLUMN = "enabled" +EXTENDED_COLUMN_NAME = "stack" + + +def _guessType(value): + if value is None: + return str + + if isinstance(value, bool): + return bool + + if isinstance(value, int): + return int + + if isinstance(value, float): + return float + + try: + int(value) + return int + except Exception: + pass + + try: + float(value) + return float + except Exception: + pass + + return str + + +class ScipionColumn(Column): + def __init__(self, name, renderer=None, callback=None): + super().__init__(name, renderer=renderer) + self.callback = callback + + def setCallback(self, callback): + self.callback = callback + + def calculate(self, row, values): + if self.callback is not None: + self.callback(row, values) + - This class intentionally hides PostgreSQL persistence details from - ProjectService. +class PostgresqlDAO(IDAO): """ + DAO compatible with metadata-viewer ObjectManager. - OBJECTS_TABLE = "objects" - SOURCE = "postgresql" + It reads persisted Scipion SetOf... outputs from PostgreSQL using + ScipionSetPostgresqlMapper, but exposes them as metadata-viewer Tables, + Columns and Pages. + """ - def __init__(self, db): + def __init__( + self, + db, + projectId: int, + protocolId: int, + outputName: str, + ): self.db = db + self.projectId = int(projectId) + self.protocolId = int(protocolId) + self.outputName = str(outputName) + self.setMapper = ScipionSetPostgresqlMapper(db) + self._tables: Dict[str, Table] = {} + self._tableCount: Dict[str, int] = {} + self._labels: Dict[str, List[str]] = {} + self._labelsTypes: Dict[str, List[Any]] = {} + self._columns: List[Dict[str, Any]] = [] + self._storedSet: Optional[Dict[str, Any]] = None + # ------------------------------------------------------------------------- - # Public API used by ProjectService + # metadata-viewer DAO API # ------------------------------------------------------------------------- - def hasOutput(self, projectId: int, protocolId: int, outputName: str) -> bool: + @classmethod + def getCompatibleFileTypes(cls): + # Kept for compatibility with ObjectManager.selectDAO(), even though + # ScipionWeb injects this DAO directly. + return ["pgset"] + + def hasOutput(self) -> bool: try: - storedSet = self._getStoredSet( - projectId=projectId, - protocolId=protocolId, - outputName=outputName, - limit=0, - offset=0, - ) - return storedSet is not None + return self._getStoredSet(limit=0, offset=0) is not None except Exception: return False - def canServeTable(self, tableName: str) -> bool: - return str(tableName or "").strip() == self.OBJECTS_TABLE + def getTables(self): + if self._tables: + return self._tables - def listTables( - self, - projectId: int, - protocolId: int, - outputName: str, - ) -> List[Dict[str, Any]]: - storedSet = self._requireStoredSet( - projectId=projectId, - protocolId=protocolId, - outputName=outputName, - limit=0, - offset=0, - ) + storedSet = self._requireStoredSet(limit=0, offset=0) - rowCount = self._getStoredSetItemsCount(storedSet) - - return [ - { - "name": self.OBJECTS_TABLE, - "alias": storedSet.get("setClassName") or outputName, - "rowCount": int(rowCount), - "hasColumnId": True, - "persisted": True, - "source": self.SOURCE, - "setId": storedSet.get("id"), - "setClassName": storedSet.get("setClassName"), - "itemClassName": storedSet.get("itemClassName"), - } + table = Table(OBJECT_TABLE) + table.setAlias(storedSet.get("setClassName") or self.outputName) + + self._tables[OBJECT_TABLE] = table + self._columns = self._normalizeColumns(storedSet.get("columns") or []) + + return self._tables + + def fillTable(self, table, objectManager): + tableName = table.getName() + if tableName != OBJECT_TABLE: + return + + firstRow = self.getTableRow(tableName, 0) + + if "id" not in firstRow: + table.setHasColumnId(False) + + labels = [ + key + for key in firstRow.keys() + if key not in EXCLUDED_COLUMNS ] - def getSchema( + self._labels[tableName] = labels + self._labelsTypes[tableName] = [ + _guessType(firstRow.get(key)) + for key in labels + ] + + imgRenderer = None + computedColsCount = 0 + + for index, colName in enumerate(labels): + value = firstRow.get(colName) + isFileNameCol = imgRenderer is None and colName.endswith("_filename") + + if colName == ENABLED_COLUMN: + renderer = BoolRenderer() + elif isFileNameCol: + renderer = StrRenderer() + else: + renderer = table.guessRenderer("" if value is None else str(value)) + + newCol = ScipionColumn(colName, renderer) + newCol.setIsSorteable(True) + + if tableName == OBJECT_TABLE: + newCol.setIsVisible(objectManager.isLabelVisible(colName)) + else: + newCol.setIsVisible(True) + + table.addColumn(newCol) + + if isFileNameCol: + previousCol = labels[index - 1] if index > 0 else "" + if previousCol.endswith("_index"): + extraRenderer = renderer + imageValue = "" if value is None else str(value) + + try: + if imageValue and ImageRenderer().getImageReader(imageValue) is not None: + extraRenderer = ImageRenderer() + imgRenderer = extraRenderer + except Exception: + pass + + extraCol = ScipionColumn(EXTENDED_COLUMN_NAME, extraRenderer) + extraCol.setCallback(self.composeImageFilename) + extraCol.setIsVisible(newCol.isVisible()) + extraCol.setIsSorteable(False) + + table.addColumn(extraCol) + newCol.setIsVisible(False) + computedColsCount += 1 + + elif colName.endswith("_matrix"): + + def addAlignmentColumn(name, offset, position): + extraCol = ScipionColumn(name, renderer=FloatRenderer()) + extraCol.setIsSorteable(False) + extraCol.setIsVisible(newCol.isVisible()) + extraCol.setCallback( + lambda row, values, off=offset, pos=position: + self.extractAngularValue(values, off, pos) + ) + table.addColumn(extraCol) + + def addShiftColumn(name, offset, position): + extraCol = ScipionColumn(name, renderer=FloatRenderer()) + extraCol.setIsVisible(newCol.isVisible()) + extraCol.setIsSorteable(False) + extraCol.setCallback( + lambda row, values, off=offset, pos=position: + self.extractShift(values, off, pos) + ) + table.addColumn(extraCol) + + colNamePrefix = colName.split("_matrix")[0] + + addAlignmentColumn(colNamePrefix + "_rot", -1, 0) + computedColsCount += 1 + + if imgRenderer: + imgRenderer.setRotationColumnIndex(index + computedColsCount) + + addAlignmentColumn(colNamePrefix + "_tilt", -2, 1) + computedColsCount += 1 + + addAlignmentColumn(colNamePrefix + "_psi", -3, 2) + computedColsCount += 1 + + addShiftColumn(colNamePrefix + "_shiftX", -4, 0) + computedColsCount += 1 + + addShiftColumn(colNamePrefix + "_shiftY", -5, 1) + computedColsCount += 1 + + if table.getColumns(): + table.setSortingColumn(table.getColumns()[0].getName()) + + def fillPage(self, page, actualColumn: str, orderAsc=True): + table = page.getTable() + tableName = table.getName() + + pageNumber = page.getPageNumber() + pageSize = page.getPageSize() + firstRow = pageNumber * pageSize - pageSize + + columnLabel = actualColumn or "id" + mode = "ASC" if orderAsc else "DESC" + + for rowCount, row in enumerate( + self.iterTable( + tableName, + start=firstRow, + limit=pageSize, + orderBy=columnLabel, + mode=mode, + ) + ): + values = [] + + for column in page.getTable().getColumns(): + if column.isSorteable(): + values.append(row.get(column.getName())) + else: + column.calculate(row, values) + + idValue = row.get("id", firstRow + rowCount + 1) + page.addRow((int(idValue), values)) + + def getTableRowCount(self, tableName): + if tableName not in self._tableCount: + self._tableCount[tableName] = self._getStoredSetItemsCount() + return self._tableCount[tableName] + + def getSelectedRangeRowsIds( self, - projectId: int, - protocolId: int, - outputName: str, - tableName: str, - ) -> Dict[str, Any]: - self._requireObjectsTable(tableName) - - storedSet = self._requireStoredSet( - projectId=projectId, - protocolId=protocolId, - outputName=outputName, - limit=0, - offset=0, + tableName, + startRow, + numberOfRows, + column, + reverse=True, + ): + rows = list( + self.iterTable( + tableName, + start=max(0, int(startRow) - 1), + limit=int(numberOfRows), + orderBy=column or "id", + mode="ASC" if reverse else "DESC", + ) ) + return [int(row.get("id")) for row in rows if row.get("id") is not None] - columns = self._normalizeColumns(storedSet.get("columns") or []) - - return { - "name": self.OBJECTS_TABLE, - "alias": storedSet.get("setClassName") or outputName, - "hasColumnId": True, - "actions": [], - "columns": [ - self._columnToSchemaItem(column, index) - for index, column in enumerate(columns) - ], - "persisted": True, - "source": self.SOURCE, - "setId": storedSet.get("id"), - "setClassName": storedSet.get("setClassName"), - "itemClassName": storedSet.get("itemClassName"), - } - - def getWindow( + def getColumnsValues( self, - projectId: int, - protocolId: int, - outputName: str, - tableName: str, - offset: int, - limit: int, - selectionOnly: bool = False, - sortBy: str = "id", - asc: bool = True, - ) -> Dict[str, Any]: - self._requireObjectsTable(tableName) + tableName, + columns, + xAxis, + selection, + limit, + useSelection, + reverse=True, + ): + selectedColumns = list(columns or []) + if xAxis and xAxis not in selectedColumns: + selectedColumns.append(xAxis) + if "id" not in selectedColumns: + selectedColumns.append("id") + + rows = list( + self.iterTable( + tableName, + start=0, + limit=limit, + orderBy=xAxis or "id", + mode="ASC" if reverse else "DESC", + ) + ) - offset = max(0, int(offset or 0)) - limit = max(1, int(limit or 1)) + if useSelection and selection is not None: + try: + selectedIds = set(selection.getSelection().keys()) + rows = [row for row in rows if row.get("id") in selectedIds] + except Exception: + pass - if selectionOnly: - return { - "offset": offset, - "limit": limit, - "totalRows": 0, - "rows": [], - "persisted": True, - "source": self.SOURCE, - } + values = {col: [] for col in selectedColumns} + for row in rows: + for col in selectedColumns: + values[col].append(row.get(col)) + + return values + + def getTableWithAdditionalInfo(self): + return None - storedSet = self._requireStoredSet( - projectId=projectId, - protocolId=protocolId, - outputName=outputName, + def close(self): + # Do not close the shared PostgreSQL connection here. + pass + + # ------------------------------------------------------------------------- + # Row/table helpers + # ------------------------------------------------------------------------- + + def iterTable(self, tableName, **kwargs): + if tableName != OBJECT_TABLE: + return + + start = max(0, int(kwargs.get("start", 0) or 0)) + limit = kwargs.get("limit", None) + limit = int(limit) if limit is not None else None + + orderBy = kwargs.get("orderBy") or "id" + mode = str(kwargs.get("mode") or "ASC").upper() + orderAsc = mode != "DESC" + + rows = self._getRows( + start=start, limit=limit, - offset=offset, + orderBy=orderBy, + orderAsc=orderAsc, ) - columns = self._normalizeColumns(storedSet.get("columns") or []) - items = storedSet.get("items") or [] - totalRows = self._getStoredSetItemsCount(storedSet) - - resultRows = [] - for localIndex, item in enumerate(items): - globalIndex = offset + localIndex - valuesPayload = self._itemToValuesPayload(item, columns) - - logicalId = item.get("scipionItemId") - if logicalId is None: - logicalId = item.get("id") - - resultRows.append( - { - "id": globalIndex, - "index": globalIndex, - "rowId": logicalId, - "values": valuesPayload, - "enabled": bool(item.get("enabled", True)), - } - ) + for row in rows: + yield row - return { - "offset": offset, - "limit": limit, - "totalRows": int(totalRows), - "rows": resultRows, - "persisted": True, - "source": self.SOURCE, - "setId": storedSet.get("id"), - } + def getTableRow(self, tableName, rowIndex): + rows = self._getRows( + start=max(0, int(rowIndex or 0)), + limit=1, + orderBy="id", + orderAsc=True, + ) + + if rows: + return rows[0] + + return self._emptyRow() + + def composeImageFilename(self, row, values): + if len(values) < 2: + values.append("") + return + + indexValue = values[-2] + filenameValue = values[-1] + + if filenameValue is None: + values.append("") + return + + indexStr = "" + try: + if int(indexValue) != 0: + indexStr = "%s@" % indexValue + except Exception: + if indexValue not in (None, "", 0, "0"): + indexStr = "%s@" % indexValue + + values.append("%s%s" % (indexStr, filenameValue)) + + def extractAngularValue(self, values, offset, position): + matrix = values[offset] + matrix = self._toNumpyMatrix(matrix) + + matrixI = numpy.linalg.inv(matrix) + eulerData = euler_from_matrix(matrix=matrixI, axes="szyz") + values.append(numpy.rad2deg(eulerData[position])) + + def extractShift(self, values, offset, position): + matrix = values[offset] + matrix = self._toNumpyMatrix(matrix) + + shape = matrix.shape[0] + values.append(matrix[position, shape - 1]) # ------------------------------------------------------------------------- - # Stored set resolution + # PostgreSQL-backed loading # ------------------------------------------------------------------------- def _getStoredSet( self, - projectId: int, - protocolId: int, - outputName: str, limit: Optional[int], offset: int, ) -> Optional[Dict[str, Any]]: return self.setMapper.getStoredSet( - projectId=projectId, - protocolDbId=protocolId, - outputName=outputName, + projectId=self.projectId, + protocolDbId=self.protocolId, + outputName=self.outputName, limit=limit, offset=offset, ) def _requireStoredSet( self, - projectId: int, - protocolId: int, - outputName: str, limit: Optional[int], offset: int, ) -> Dict[str, Any]: - try: - storedSet = self._getStoredSet( - projectId=projectId, - protocolId=protocolId, - outputName=outputName, - limit=limit, - offset=offset, - ) - except Exception as exc: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Persisted Scipion set not found: %s" % exc, - ) - + storedSet = self._getStoredSet(limit=limit, offset=offset) if storedSet is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Persisted Scipion set not found", + raise ValueError( + "Persisted Scipion set was not found: projectId=%s protocolId=%s outputName=%s" + % (self.projectId, self.protocolId, self.outputName) ) - return storedSet - def _requireObjectsTable(self, tableName: str) -> None: - if not self.canServeTable(tableName): - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Persisted metadata table '%s' not found" % tableName, - ) + def _getStoredSetHeader(self) -> Dict[str, Any]: + if self._storedSet is None: + self._storedSet = self._requireStoredSet(limit=0, offset=0) + self._columns = self._normalizeColumns(self._storedSet.get("columns") or []) + return self._storedSet - # ------------------------------------------------------------------------- - # Shape adapters - # ------------------------------------------------------------------------- + def _getRows( + self, + start: int, + limit: Optional[int], + orderBy: str, + orderAsc: bool, + ) -> List[Dict[str, Any]]: + orderBy = str(orderBy or "id") - def _normalizeColumns(self, columns: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - def sortKey(column: Dict[str, Any]): - try: - return int(column.get("position") or 0) - except Exception: - return 0 + canUsePagedRead = orderAsc and orderBy in ("id", "_objId", "SCIPION_OBJECT_ID") - return sorted(columns or [], key=sortKey) + if canUsePagedRead: + storedSet = self._requireStoredSet(limit=limit, offset=start) + items = storedSet.get("items") or [] + self._columns = self._normalizeColumns(storedSet.get("columns") or self._columns) + return [self._itemToRow(item) for item in items] - def _columnToSchemaItem( - self, - column: Dict[str, Any], - index: int, - ) -> Dict[str, Any]: - rawName = ( - column.get("labelProperty") - or column.get("columnName") - or "column_%s" % index + storedSet = self._requireStoredSet(limit=None, offset=0) + items = storedSet.get("items") or [] + self._columns = self._normalizeColumns(storedSet.get("columns") or self._columns) + + rows = [self._itemToRow(item) for item in items] + rows.sort( + key=lambda row: self._sortValue(row.get(orderBy)), + reverse=not orderAsc, ) - name = str(rawName) - - return { - "name": name, - "alias": self._getColumnAlias(name), - "index": index, - "sortable": bool(column.get("indexed", False)) or name in ("id", "_objId"), - "visible": True, - "rendererType": self._getRendererType(column), - "decimals": self._getDecimals(column), - "hasTransformation": False, - "persisted": True, - "source": self.SOURCE, - } - def _itemToValuesPayload( - self, - item: Dict[str, Any], - columns: List[Dict[str, Any]], - ) -> List[Any]: + if limit is None: + return rows[start:] + + return rows[start:start + limit] + + def _itemToRow(self, item: Dict[str, Any]) -> Dict[str, Any]: values = item.get("values") or {} if not isinstance(values, dict): values = {} - payload = [] - for column in columns: - label = column.get("labelProperty") - value = values.get(label) - - if value is None and label is not None: - value = values.get(str(label).strip()) + itemId = item.get("scipionItemId") + if itemId is None: + itemId = item.get("id") - payload.append(self._normalizeCellValue(value)) + row: Dict[str, Any] = { + "id": itemId, + "enabled": bool(item.get("enabled", True)), + } - return payload + for column in self._columns: + label = column.get("labelProperty") + if not label: + continue - def _normalizeCellValue(self, value: Any) -> Any: - if value is None: - return None + value = values.get(label) + if value is None: + value = values.get(str(label).strip()) - if isinstance(value, (str, int, float, bool)): - return value + row[str(label)] = self._normalizeValue(str(label), value) - if isinstance(value, list): - return [self._normalizeCellValue(item) for item in value] + return row - if isinstance(value, tuple): - return [self._normalizeCellValue(item) for item in value] + def _emptyRow(self) -> Dict[str, Any]: + row = { + "id": 0, + "enabled": True, + } - if isinstance(value, dict): - return { - str(key): self._normalizeCellValue(item) - for key, item in value.items() - } + for column in self._columns: + label = column.get("labelProperty") + if label: + row[str(label)] = None - return str(value) + return row # ------------------------------------------------------------------------- # Metadata helpers # ------------------------------------------------------------------------- - def _getStoredSetItemsCount(self, storedSet: Dict[str, Any]) -> int: + def _normalizeColumns(self, columns: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + def sortKey(column: Dict[str, Any]): + try: + return int(column.get("position") or 0) + except Exception: + return 0 + + return sorted(columns or [], key=sortKey) + + def _getStoredSetItemsCount(self) -> int: + storedSet = self._getStoredSetHeader() properties = storedSet.get("properties") or {} if isinstance(properties, dict): @@ -365,45 +601,69 @@ def _getStoredSetItemsCount(self, storedSet: Dict[str, Any]) -> int: return 0 - def _getColumnAlias(self, name: str) -> str: - clean = str(name or "").strip() - if clean.startswith("_"): - clean = clean[1:] - return clean or str(name) + def _normalizeValue(self, label: str, value: Any) -> Any: + if value is None: + return None - def _getRendererType(self, column: Dict[str, Any]) -> str: - valueType = str(column.get("valueType") or "").lower() - className = str(column.get("className") or "").lower() + if label.endswith("_matrix"): + return self._toNumpyMatrix(value) - text = "%s %s" % (valueType, className) + if isinstance(value, (str, int, float, bool)): + return value + + if isinstance(value, list): + return [ + self._normalizeValue(label, item) + for item in value + ] + + if isinstance(value, tuple): + return [ + self._normalizeValue(label, item) + for item in value + ] + + if isinstance(value, dict): + return { + str(key): self._normalizeValue(label, item) + for key, item in value.items() + } - if "bool" in text: - return "bool" + return str(value) - if "float" in text or "decimal" in text: - return "float" + def _toNumpyMatrix(self, value: Any): + if isinstance(value, numpy.ndarray): + return value - if "integer" in text or "long" in text or className == "int": - return "int" + if isinstance(value, str): + try: + value = eval(value) + except Exception: + value = [] - if "matrix" in text: - return "matrix" + return numpy.array(value) - if "image" in text or "filename" in text or "file" in text: - return "image" + def _sortValue(self, value: Any): + if value is None: + return "" - return "str" + if isinstance(value, numpy.ndarray): + return str(value.tolist()) - def _getDecimals(self, column: Dict[str, Any]) -> Optional[int]: - rendererType = self._getRendererType(column) - if rendererType == "float": - return 2 - return None + if isinstance(value, (list, tuple, dict)): + return str(value) + + return value def _toInt(self, value: Any) -> Optional[int]: if value is None or value == "": return None + try: return int(value) except Exception: - return None \ No newline at end of file + return None + + +# Backward-compatible alias while project_service.py is migrated. +PostgresqlMetadataDAO = PostgresqlDAO \ No newline at end of file From bb5b43f1640be4184925fa69bdfd514810f5b544 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 19 Jun 2026 19:48:33 +0200 Subject: [PATCH 031/278] route metadata image rendering through object manager --- app/backend/api/services/project_service.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 918865c5..ef62525e 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -9114,18 +9114,19 @@ def renderMetadataImageCellService( from pathlib import Path as LocalPath # Resolve metadata root path for relative image paths - try: - _protocol, _output, metaPath = self._resolveOutputForMetadata(protocolId, outputName) - metaDir = LocalPath(metaPath).parent - except HTTPException: - # If the output / metadata file is really missing, that is a real error - raise - except Exception: - metaDir = None - objMgr, table = self._openMetadataTable(projectId, protocolId, outputName, tableName) columns = list(table.getColumns()) + metaDir = None + objMgrFileName = str(getattr(objMgr, "_fileName", "") or "") + + if not objMgrFileName.startswith("postgresql://"): + try: + _protocol, _output, metaPath = self._resolveOutputForMetadata(protocolId, outputName) + metaDir = LocalPath(metaPath).parent + except Exception: + metaDir = None + # Resolve column index colIndex = table.getColumnIndexFromLabel(columnName) if colIndex < 0 or colIndex >= len(columns): From 12ba8cf63e1e0c4514bb973f7a7d182466105669 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 19 Jun 2026 22:06:25 +0200 Subject: [PATCH 032/278] Add logical tables for persisted scipion sets --- ...ae69565b_add_scipion_set_logical_tables.py | 117 +++++++++ app/backend/mapper/scipion_set_mapper.py | 225 +++++++++++++++++- app/backend/viewers/postgresql_dao.py | 141 ++++++++++- 3 files changed, 463 insertions(+), 20 deletions(-) create mode 100644 alembic/versions/33ffae69565b_add_scipion_set_logical_tables.py diff --git a/alembic/versions/33ffae69565b_add_scipion_set_logical_tables.py b/alembic/versions/33ffae69565b_add_scipion_set_logical_tables.py new file mode 100644 index 00000000..27b34ab4 --- /dev/null +++ b/alembic/versions/33ffae69565b_add_scipion_set_logical_tables.py @@ -0,0 +1,117 @@ +"""add scipion set logical tables + +Revision ID: 33ffae69565b +Revises: c3d2b8f4a901 +Create Date: 2026-06-19 22:01:10.327665 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '33ffae69565b' +down_revision: Union[str, None] = 'c3d2b8f4a901' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "scipion_set_tables", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("setId", sa.Integer(), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("alias", sa.Text(), nullable=True), + sa.Column("tableKind", sa.Text(), server_default="root", nullable=False), + sa.Column("parentTableId", sa.Integer(), nullable=True), + sa.Column("parentItemId", sa.Integer(), nullable=True), + sa.Column("itemClassName", sa.Text(), nullable=True), + sa.Column( + "properties", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.CheckConstraint( + "\"tableKind\" IN ('root', 'child', 'properties')", + name="ck_scipion_set_tables_table_kind", + ), + sa.ForeignKeyConstraint(["setId"], ["scipion_sets.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["parentTableId"], ["scipion_set_tables.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("setId", "name", name="ux_scipion_set_tables_set_name"), + ) + op.create_index("idx_scipion_set_tables_set", "scipion_set_tables", ["setId"]) + op.create_index("idx_scipion_set_tables_parent", "scipion_set_tables", ["parentTableId"]) + op.create_index("idx_scipion_set_tables_properties_gin", "scipion_set_tables", ["properties"], postgresql_using="gin") + + op.create_table( + "scipion_set_table_columns", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("tableId", sa.Integer(), nullable=False), + sa.Column("labelProperty", sa.Text(), nullable=False), + sa.Column("columnName", sa.Text(), nullable=False), + sa.Column("className", sa.Text(), nullable=True), + sa.Column("valueType", sa.Text(), nullable=True), + sa.Column("position", sa.Integer(), nullable=False), + sa.Column("indexed", sa.Boolean(), server_default="false", nullable=False), + sa.Column( + "properties", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.ForeignKeyConstraint(["tableId"], ["scipion_set_tables.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("tableId", "labelProperty", name="ux_scipion_set_table_columns_table_label"), + sa.UniqueConstraint("tableId", "columnName", name="ux_scipion_set_table_columns_table_column"), + ) + op.create_index("idx_scipion_set_table_columns_table", "scipion_set_table_columns", ["tableId"]) + op.create_index("idx_scipion_set_table_columns_label", "scipion_set_table_columns", ["labelProperty"]) + + op.create_table( + "scipion_set_table_items", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("tableId", sa.Integer(), nullable=False), + sa.Column("scipionItemId", sa.Integer(), nullable=False), + sa.Column("parentItemId", sa.Integer(), nullable=True), + sa.Column("enabled", sa.Boolean(), server_default="true", nullable=False), + sa.Column("label", sa.Text(), nullable=True), + sa.Column("comment", sa.Text(), nullable=True), + sa.Column("creation", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "values", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["tableId"], ["scipion_set_tables.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("tableId", "scipionItemId", name="ux_scipion_set_table_items_table_item"), + ) + op.create_index("idx_scipion_set_table_items_table", "scipion_set_table_items", ["tableId"]) + op.create_index("idx_scipion_set_table_items_parent", "scipion_set_table_items", ["parentItemId"]) + op.create_index("idx_scipion_set_table_items_values_gin", "scipion_set_table_items", ["values"], postgresql_using="gin") + + +def downgrade() -> None: + op.drop_index("idx_scipion_set_table_items_values_gin", table_name="scipion_set_table_items") + op.drop_index("idx_scipion_set_table_items_parent", table_name="scipion_set_table_items") + op.drop_index("idx_scipion_set_table_items_table", table_name="scipion_set_table_items") + op.drop_table("scipion_set_table_items") + + op.drop_index("idx_scipion_set_table_columns_label", table_name="scipion_set_table_columns") + op.drop_index("idx_scipion_set_table_columns_table", table_name="scipion_set_table_columns") + op.drop_table("scipion_set_table_columns") + + op.drop_index("idx_scipion_set_tables_properties_gin", table_name="scipion_set_tables") + op.drop_index("idx_scipion_set_tables_parent", table_name="scipion_set_tables") + op.drop_index("idx_scipion_set_tables_set", table_name="scipion_set_tables") + op.drop_table("scipion_set_tables") \ No newline at end of file diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index bddc3316..eca7874c 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -65,8 +65,12 @@ def storeSet( existingSet = self._getExistingSet(projectId, protocolDbId, outputName) if existingSet is not None: + existingSetId = int(existingSet["id"]) existingProperties = self._normalizeProperties(existingSet.get("properties")) - if self._shouldSkipSetSync(existingProperties, itemsCountHint, maxItemIdHint, sourceMTime): + if ( + self.hasStoredSetTables(existingSetId) + and self._shouldSkipSetSync(existingProperties, itemsCountHint, maxItemIdHint, sourceMTime) + ): skippedProperties = dict(existingProperties) skippedProperties["lastCheckedAt"] = syncTimestamp skippedProperties["lastSkipReason"] = "unchanged_signature" @@ -135,11 +139,27 @@ def storeSet( ) self._upsertSetColumns(setId, columns) + rootTableId = self._upsertSetTable( + setId=setId, + name="objects", + alias=self._getClassName(scipionSet) or outputName, + tableKind="root", + parentTableId=None, + parentItemId=None, + itemClassName=itemClassName, + properties={ + "source": "postgresql", + "legacySetTable": True, + }, + ) + self._upsertSetTableColumns(rootTableId, columns) + itemsCount = 0 maxItemId = None if firstItem is not None: itemsCount, maxItemId = self._upsertSetItems( setId=setId, + tableId=rootTableId, firstItem=firstItem, remainingItems=itemIterator, batchSize=batchSize, @@ -391,13 +411,15 @@ def _upsertSetProperties(self, setId: int, properties: Dict[str, Any]) -> None: ) def _upsertSetItems( - self, - setId: int, - firstItem: Any, - remainingItems: Iterator[Any], - batchSize: int, + self, + setId: int, + tableId: Optional[int], + firstItem: Any, + remainingItems: Iterator[Any], + batchSize: int, ) -> Tuple[int, Optional[int]]: rows: List[Tuple[Any, ...]] = [] + tableRows: List[Tuple[Any, ...]] = [] itemsCount = 0 maxItemId: Optional[int] = None @@ -407,6 +429,8 @@ def _upsertSetItems( raise ValueError("Cannot store a Scipion set item without getObjId()/getId()") maxItemId = itemId if maxItemId is None else max(maxItemId, itemId) + itemValues = self._getItemValues(item) + rows.append( ( setId, @@ -415,20 +439,207 @@ def _upsertSetItems( self._getObjectLabel(item), self._getObjectComment(item), self._getObjectCreation(item), - self._jsonParam(self._getItemValues(item)), + self._jsonParam(itemValues), ) ) + + if tableId is not None: + tableRows.append( + ( + tableId, + itemId, + None, + self._getItemEnabled(item), + self._getObjectLabel(item), + self._getObjectComment(item), + self._getObjectCreation(item), + self._jsonParam(itemValues), + ) + ) itemsCount += 1 if len(rows) >= batchSize: self._flushSetItems(rows) rows = [] + if tableRows: + self._flushSetTableItems(tableRows) + tableRows = [] + if rows: self._flushSetItems(rows) + if tableRows: + self._flushSetTableItems(tableRows) + return itemsCount, maxItemId + def hasStoredSetTables(self, setId: int) -> bool: + row = self.db.fetchOne( + """ + SELECT id + FROM scipion_set_tables + WHERE "setId" = %s + LIMIT 1 + """, + (setId,), + ) + return row is not None + + def listStoredSetTables(self, setId: int) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT id, "setId", name, alias, "tableKind", "parentTableId", + "parentItemId", "itemClassName", properties, "createdAt", "updatedAt" + FROM scipion_set_tables + WHERE "setId" = %s + ORDER BY + CASE WHEN "tableKind" = 'root' THEN 0 ELSE 1 END, + name ASC + """, + (setId,), + ) + + def getStoredSetTableColumns(self, tableId: int) -> List[Dict[str, Any]]: + return self.db.fetchAll( + """ + SELECT id, "tableId", "labelProperty", "columnName", "className", + "valueType", position, indexed, properties + FROM scipion_set_table_columns + WHERE "tableId" = %s + ORDER BY position ASC + """, + (tableId,), + ) + + def getStoredSetTableItems( + self, + tableId: int, + limit: Optional[int] = None, + offset: int = 0, + ) -> List[Dict[str, Any]]: + if limit is None: + return self.db.fetchAll( + """ + SELECT id, "tableId", "scipionItemId", "parentItemId", enabled, + label, comment, creation, "values", "createdAt", "updatedAt" + FROM scipion_set_table_items + WHERE "tableId" = %s + ORDER BY "scipionItemId" ASC + """, + (tableId,), + ) + + return self.db.fetchAll( + """ + SELECT id, "tableId", "scipionItemId", "parentItemId", enabled, + label, comment, creation, "values", "createdAt", "updatedAt" + FROM scipion_set_table_items + WHERE "tableId" = %s + ORDER BY "scipionItemId" ASC + LIMIT %s OFFSET %s + """, + (tableId, limit, offset), + ) + + def _upsertSetTable( + self, + setId: int, + name: str, + alias: Optional[str], + tableKind: str, + parentTableId: Optional[int], + parentItemId: Optional[int], + itemClassName: Optional[str], + properties: Optional[Dict[str, Any]] = None, + ) -> int: + cur = self.db.execute( + """ + INSERT INTO scipion_set_tables ( + "setId", name, alias, "tableKind", "parentTableId", + "parentItemId", "itemClassName", properties + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb) + ON CONFLICT ON CONSTRAINT ux_scipion_set_tables_set_name + DO UPDATE SET + alias = EXCLUDED.alias, + "tableKind" = EXCLUDED."tableKind", + "parentTableId" = EXCLUDED."parentTableId", + "parentItemId" = EXCLUDED."parentItemId", + "itemClassName" = EXCLUDED."itemClassName", + properties = EXCLUDED.properties, + "updatedAt" = NOW() + RETURNING id + """, + ( + setId, + name, + alias, + tableKind, + parentTableId, + parentItemId, + itemClassName, + self._jsonParam(properties or {}), + ), + commit=False, + ) + return int(cur.fetchone()["id"]) + + def _upsertSetTableColumns(self, tableId: int, columns: List[Dict[str, Any]]) -> None: + for column in columns: + self.db.execute( + """ + INSERT INTO scipion_set_table_columns ( + "tableId", "labelProperty", "columnName", "className", + "valueType", position, indexed, properties + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb) + ON CONFLICT ON CONSTRAINT ux_scipion_set_table_columns_table_label + DO UPDATE SET + "columnName" = EXCLUDED."columnName", + "className" = EXCLUDED."className", + "valueType" = EXCLUDED."valueType", + position = EXCLUDED.position, + indexed = EXCLUDED.indexed, + properties = EXCLUDED.properties + """, + ( + tableId, + column["labelProperty"], + column["columnName"], + column["className"], + column["valueType"], + column["position"], + column["indexed"], + self._jsonParam(column.get("properties") or {}), + ), + commit=False, + ) + + def _flushSetTableItems(self, rows: List[Tuple[Any, ...]]) -> None: + psycopg2.extras.execute_values( + self.db.cursor, + """ + INSERT INTO scipion_set_table_items ( + "tableId", "scipionItemId", "parentItemId", enabled, + label, comment, creation, "values" + ) + VALUES %s + ON CONFLICT ON CONSTRAINT ux_scipion_set_table_items_table_item + DO UPDATE SET + "parentItemId" = EXCLUDED."parentItemId", + enabled = EXCLUDED.enabled, + label = EXCLUDED.label, + comment = EXCLUDED.comment, + creation = EXCLUDED.creation, + "values" = EXCLUDED."values", + "updatedAt" = NOW() + """, + rows, + template="(%s, %s, %s, %s, %s, %s, %s, %s::jsonb)", + page_size=len(rows), + ) + def _flushSetItems(self, rows: List[Tuple[Any, ...]]) -> None: psycopg2.extras.execute_values( self.db.cursor, diff --git a/app/backend/viewers/postgresql_dao.py b/app/backend/viewers/postgresql_dao.py index 7b32aa9e..53ecabff 100644 --- a/app/backend/viewers/postgresql_dao.py +++ b/app/backend/viewers/postgresql_dao.py @@ -142,6 +142,10 @@ def __init__( self._columns: List[Dict[str, Any]] = [] self._storedSet: Optional[Dict[str, Any]] = None + self._logicalTables: Dict[str, Dict[str, Any]] = {} + self._tableColumns: Dict[str, List[Dict[str, Any]]] = {} + self._useLogicalTables = False + # ------------------------------------------------------------------------- # metadata-viewer DAO API # ------------------------------------------------------------------------- @@ -163,6 +167,32 @@ def getTables(self): return self._tables storedSet = self._requireStoredSet(limit=0, offset=0) + setId = int(storedSet["id"]) + + logicalTables = [] + try: + logicalTables = self.setMapper.listStoredSetTables(setId) or [] + except Exception: + logicalTables = [] + + if logicalTables: + self._useLogicalTables = True + + for logicalTable in logicalTables: + tableName = logicalTable.get("name") + if not tableName: + continue + + table = Table(tableName) + table.setAlias(logicalTable.get("alias") or tableName) + + self._tables[tableName] = table + self._logicalTables[tableName] = logicalTable + self._tableColumns[tableName] = self.setMapper.getStoredSetTableColumns( + int(logicalTable["id"]) + ) + + return self._tables table = Table(OBJECT_TABLE) table.setAlias(storedSet.get("setClassName") or self.outputName) @@ -178,6 +208,7 @@ def fillTable(self, table, objectManager): return firstRow = self.getTableRow(tableName, 0) + columns = self._getColumnsForTable(tableName) if "id" not in firstRow: table.setHasColumnId(False) @@ -285,6 +316,16 @@ def addShiftColumn(name, offset, position): if table.getColumns(): table.setSortingColumn(table.getColumns()[0].getName()) + def _getColumnsForTable(self, tableName: str) -> List[Dict[str, Any]]: + if self._useLogicalTables: + return self._normalizeColumns(self._tableColumns.get(tableName) or []) + return self._columns + + def _getLogicalTable(self, tableName: str) -> Optional[Dict[str, Any]]: + if not self._useLogicalTables: + return None + return self._logicalTables.get(tableName) + def fillPage(self, page, actualColumn: str, orderAsc=True): table = page.getTable() tableName = table.getName() @@ -316,10 +357,19 @@ def fillPage(self, page, actualColumn: str, orderAsc=True): idValue = row.get("id", firstRow + rowCount + 1) page.addRow((int(idValue), values)) - def getTableRowCount(self, tableName): - if tableName not in self._tableCount: - self._tableCount[tableName] = self._getStoredSetItemsCount() - return self._tableCount[tableName] + def getTableRow(self, tableName, rowIndex): + rows = self._getRows( + tableName=tableName, + start=max(0, int(rowIndex or 0)), + limit=1, + orderBy="id", + orderAsc=True, + ) + + if rows: + return rows[0] + + return self._emptyRow(tableName) def getSelectedRangeRowsIds( self, @@ -392,9 +442,6 @@ def close(self): # ------------------------------------------------------------------------- def iterTable(self, tableName, **kwargs): - if tableName != OBJECT_TABLE: - return - start = max(0, int(kwargs.get("start", 0) or 0)) limit = kwargs.get("limit", None) limit = int(limit) if limit is not None else None @@ -404,6 +451,7 @@ def iterTable(self, tableName, **kwargs): orderAsc = mode != "DESC" rows = self._getRows( + tableName=tableName, start=start, limit=limit, orderBy=orderBy, @@ -501,6 +549,7 @@ def _getStoredSetHeader(self) -> Dict[str, Any]: def _getRows( self, + tableName: str, start: int, limit: Optional[int], orderBy: str, @@ -508,19 +557,53 @@ def _getRows( ) -> List[Dict[str, Any]]: orderBy = str(orderBy or "id") + if self._useLogicalTables: + logicalTable = self._getLogicalTable(tableName) + if logicalTable is None: + return [] + + tableId = int(logicalTable["id"]) + columns = self._getColumnsForTable(tableName) + + canUsePagedRead = orderAsc and orderBy in ("id", "_objId", "SCIPION_OBJECT_ID") + + if canUsePagedRead: + items = self.setMapper.getStoredSetTableItems( + tableId=tableId, + limit=limit, + offset=start, + ) + return [self._itemToRow(item, columns) for item in items] + + items = self.setMapper.getStoredSetTableItems( + tableId=tableId, + limit=None, + offset=0, + ) + rows = [self._itemToRow(item, columns) for item in items] + rows.sort( + key=lambda row: self._sortValue(row.get(orderBy)), + reverse=not orderAsc, + ) + + if limit is None: + return rows[start:] + + return rows[start:start + limit] + canUsePagedRead = orderAsc and orderBy in ("id", "_objId", "SCIPION_OBJECT_ID") if canUsePagedRead: storedSet = self._requireStoredSet(limit=limit, offset=start) items = storedSet.get("items") or [] self._columns = self._normalizeColumns(storedSet.get("columns") or self._columns) - return [self._itemToRow(item) for item in items] + return [self._itemToRow(item, self._columns) for item in items] storedSet = self._requireStoredSet(limit=None, offset=0) items = storedSet.get("items") or [] self._columns = self._normalizeColumns(storedSet.get("columns") or self._columns) - rows = [self._itemToRow(item) for item in items] + rows = [self._itemToRow(item, self._columns) for item in items] rows.sort( key=lambda row: self._sortValue(row.get(orderBy)), reverse=not orderAsc, @@ -531,7 +614,11 @@ def _getRows( return rows[start:start + limit] - def _itemToRow(self, item: Dict[str, Any]) -> Dict[str, Any]: + def _itemToRow( + self, + item: Dict[str, Any], + columns: List[Dict[str, Any]], + ) -> Dict[str, Any]: values = item.get("values") or {} if not isinstance(values, dict): values = {} @@ -545,7 +632,7 @@ def _itemToRow(self, item: Dict[str, Any]) -> Dict[str, Any]: "enabled": bool(item.get("enabled", True)), } - for column in self._columns: + for column in columns: label = column.get("labelProperty") if not label: continue @@ -558,19 +645,47 @@ def _itemToRow(self, item: Dict[str, Any]) -> Dict[str, Any]: return row - def _emptyRow(self) -> Dict[str, Any]: + def _emptyRow(self, tableName: str) -> Dict[str, Any]: row = { "id": 0, "enabled": True, } - for column in self._columns: + for column in self._getColumnsForTable(tableName): label = column.get("labelProperty") if label: row[str(label)] = None return row + def getTableRowCount(self, tableName): + if tableName not in self._tableCount: + if self._useLogicalTables: + logicalTable = self._getLogicalTable(tableName) + if logicalTable is None: + self._tableCount[tableName] = 0 + else: + self._tableCount[tableName] = self._getStoredSetTableItemsCount( + int(logicalTable["id"]) + ) + else: + self._tableCount[tableName] = self._getStoredSetItemsCount() + + return self._tableCount[tableName] + + def _getStoredSetTableItemsCount(self, tableId: int) -> int: + row = self.db.fetchOne( + """ + SELECT COUNT(*) AS count + FROM scipion_set_table_items + WHERE "tableId" = %s + """, + (tableId,), + ) + if not row: + return 0 + return int(row.get("count") or 0) + # ------------------------------------------------------------------------- # Metadata helpers # ------------------------------------------------------------------------- From 8d908b4119af19e9253f5ba3dae0ab86470db672 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 19 Jun 2026 22:11:55 +0200 Subject: [PATCH 033/278] fix postgres metadata dao logical table handling --- app/backend/viewers/postgresql_dao.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/app/backend/viewers/postgresql_dao.py b/app/backend/viewers/postgresql_dao.py index 53ecabff..4b3ad996 100644 --- a/app/backend/viewers/postgresql_dao.py +++ b/app/backend/viewers/postgresql_dao.py @@ -204,7 +204,7 @@ def getTables(self): def fillTable(self, table, objectManager): tableName = table.getName() - if tableName != OBJECT_TABLE: + if tableName != OBJECT_TABLE and not self._useLogicalTables: return firstRow = self.getTableRow(tableName, 0) @@ -461,19 +461,6 @@ def iterTable(self, tableName, **kwargs): for row in rows: yield row - def getTableRow(self, tableName, rowIndex): - rows = self._getRows( - start=max(0, int(rowIndex or 0)), - limit=1, - orderBy="id", - orderAsc=True, - ) - - if rows: - return rows[0] - - return self._emptyRow() - def composeImageFilename(self, row, values): if len(values) < 2: values.append("") From 7dafe669447470b8bfc8c9f12fd53288632162fa Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 19 Jun 2026 22:35:34 +0200 Subject: [PATCH 034/278] persist nested logical tables for scipion classes --- app/backend/mapper/scipion_set_mapper.py | 135 +++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index eca7874c..00223cda 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -456,6 +456,15 @@ def _upsertSetItems( self._jsonParam(itemValues), ) ) + + self._upsertNestedLogicalTablesForItem( + setId=setId, + parentTableId=tableId, + parentItem=item, + parentItemId=itemId, + batchSize=batchSize, + ) + itemsCount += 1 if len(rows) >= batchSize: @@ -474,6 +483,132 @@ def _upsertSetItems( return itemsCount, maxItemId + + def _upsertNestedLogicalTablesForItem( + self, + setId: int, + parentTableId: int, + parentItem: Any, + parentItemId: int, + batchSize: int, + ) -> None: + """ + Persist child logical tables for complex Scipion set items. + + First supported case: + - Class2D/Class3D-like items exposing iterItems() + -> ClassXXX_Objects table with the class members. + """ + if not self._isClassLikeNestedItem(parentItem): + return + + childIterator = iter(self._iterNestedItems(parentItem)) + firstChild = self._nextOrNone(childIterator) + if firstChild is None: + return + + childSchema = self._getItemSchema(firstChild) + childColumns = self._getSetColumns(childSchema) + childItemClassName = self._getItemClassName(firstChild, childSchema) + + tableName = self._getNestedClassTableName(parentItemId) + tableAlias = self._getNestedClassTableAlias(tableName, childItemClassName) + + childTableId = self._upsertSetTable( + setId=setId, + name=tableName, + alias=tableAlias, + tableKind="child", + parentTableId=parentTableId, + parentItemId=parentItemId, + itemClassName=childItemClassName, + properties={ + "source": "postgresql", + "parentItemId": parentItemId, + "parentClassName": self._getClassName(parentItem), + }, + ) + + self._upsertSetTableColumns(childTableId, childColumns) + + self._upsertLogicalTableItems( + tableId=childTableId, + parentItemId=parentItemId, + firstItem=firstChild, + remainingItems=childIterator, + batchSize=batchSize, + ) + + def _isClassLikeNestedItem(self, item: Any) -> bool: + className = self._getClassName(item) or item.__class__.__name__ + if not str(className).startswith("Class"): + return False + + iterItems = getattr(item, "iterItems", None) + if callable(iterItems): + return True + + return False + + def _iterNestedItems(self, item: Any) -> Iterable[Any]: + iterItems = getattr(item, "iterItems", None) + if callable(iterItems): + try: + return iterItems(iterate=False) + except TypeError: + return iterItems() + + return iter(()) + + def _getNestedClassTableName(self, parentItemId: int) -> str: + try: + return "Class%03d_Objects" % int(parentItemId) + except Exception: + return "Class%s_Objects" % str(parentItemId) + + def _getNestedClassTableAlias(self, tableName: str, childItemClassName: str) -> str: + cleanClassName = str(childItemClassName or "Objects") + return tableName.replace("_Objects", "_%s" % cleanClassName) + + def _upsertLogicalTableItems( + self, + tableId: int, + parentItemId: Optional[int], + firstItem: Any, + remainingItems: Iterator[Any], + batchSize: int, + ) -> int: + rows: List[Tuple[Any, ...]] = [] + itemsCount = 0 + + for item in self._chainFirst(firstItem, remainingItems): + itemId = self._getSourceObjId(item) + if itemId is None: + continue + + rows.append( + ( + tableId, + itemId, + parentItemId, + self._getItemEnabled(item), + self._getObjectLabel(item), + self._getObjectComment(item), + self._getObjectCreation(item), + self._jsonParam(self._getItemValues(item)), + ) + ) + itemsCount += 1 + + if len(rows) >= batchSize: + self._flushSetTableItems(rows) + rows = [] + + if rows: + self._flushSetTableItems(rows) + + return itemsCount + def hasStoredSetTables(self, setId: int) -> bool: row = self.db.fetchOne( """ From 1bd82140d9b9e65fdfaf336122e6303b0999c9ea Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 09:09:13 +0200 Subject: [PATCH 035/278] Reuse request-scoped database mapper --- app/backend/api/routers/project_router.py | 8 +- app/backend/api/services/project_service.py | 97 +++++++++++++++------ app/backend/database.py | 21 ++++- 3 files changed, 96 insertions(+), 30 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 66e519df..544dc01b 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -26,7 +26,7 @@ WorkflowImportRequest, ) from app.backend.api.schemas.tags_schema import ProtocolTagCreateIn, ProtocolTagUpdateIn, ProtocolTagsSetIn -from app.backend.database import getMapper +from app.backend.database import getMapperDependency as getMapper from app.backend.api.schemas.project_schema import (ProjectCreate, ProjectOut, ProjectUpdate, ProjectShareCreate, ApplyWorkflowToProjectRequest, TiltSeriesNewSetRequest, ProjectImportIn, ProtocolWizardExecuteResponse, @@ -2528,6 +2528,7 @@ def listOutputMetadataTables( projectId=projectId, protocolId=protocolId, outputName=outputName, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -2565,6 +2566,7 @@ def getMetadataTableSchema( protocolId=protocolId, outputName=outputName, tableName=tableName, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -2698,6 +2700,7 @@ def getMetadataTablePage( sortBy=sortBy, asc=asc, selectionOnly=selectionOnly, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -2764,6 +2767,7 @@ def exportMetadataTable( fmt=fmt, selectionOnly=selectionOnly, ids=idList, + mapper=mapper, ) resp.headers["X-Debug-Auth"] = "ok" resp.headers["X-Debug-UserId"] = str(getattr(currentUser, "id", currentUser.get("id", ""))) @@ -2843,6 +2847,7 @@ def renderMetadataImageCell( applyTransform=applyTransform, inline=inline, fmt=fmt, + mapper=mapper, ) resp.headers["X-Debug-Auth"] = "ok" resp.headers["X-Debug-UserId"] = str(getattr(currentUser, "id", currentUser.get("id", ""))) @@ -2903,6 +2908,7 @@ def getMetadataTableWindow( selectionOnly=selectionOnly, sortBy=sortBy, asc=asc, + mapper=mapper, ) from fastapi.responses import JSONResponse diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index ef62525e..ec4f818c 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -8466,11 +8466,13 @@ def _getPostgresqlDAOIfAvailable( projectId: int, protocolId: int, outputName: str, + mapper=None, ): - from app.backend.database import getMapper from app.backend.viewers.postgresql_dao import PostgresqlDAO - mapper = getMapper() + if mapper is None: + return None + dao = PostgresqlDAO( db=mapper.db, projectId=projectId, @@ -8488,11 +8490,13 @@ def _getMetadataObjectManagerForOutput( projectId: int, protocolId: int, outputName: str, + mapper=None, ) -> ObjectManager: pgDao = self._getPostgresqlDAOIfAvailable( projectId=projectId, protocolId=protocolId, outputName=outputName, + mapper=mapper, ) if pgDao is not None: @@ -8516,6 +8520,7 @@ def _openMetadataTable( protocolId: int, outputName: str, tableName: str, + mapper=None, ): """ Resolve (ObjectManager, Table) for a given output + tableName. @@ -8527,6 +8532,7 @@ def _openMetadataTable( projectId=projectId, protocolId=protocolId, outputName=outputName, + mapper=mapper, ) table = objMgr.getTable(tableName) @@ -8600,7 +8606,8 @@ def _convertCellForPage(self, renderer, rawValue, rowValues): def listOutputMetadataTablesService(self, projectId: int, protocolId: int, - outputName: str): + outputName: str, + mapper=None): """ List logical metadata tables associated with an output. """ @@ -8609,6 +8616,7 @@ def listOutputMetadataTablesService(self, projectId: int, projectId=projectId, protocolId=protocolId, outputName=outputName, + mapper=mapper, ) tables = objMgr.getTables() or {} @@ -8638,12 +8646,19 @@ def listOutputMetadataTablesService(self, projectId: int, def getMetadataTableSchemaService(self, projectId: int, protocolId: int, outputName: str, - tableName: str): + tableName: str, + mapper=None): """ Return logical schema for one metadata table: columns, renderers, flags. """ with _metadataLock: - objMgr, table = self._openMetadataTable(projectId, protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable( + projectId, + protocolId, + outputName, + tableName, + mapper=mapper, + ) visibleLabels = [] orderLabels = [] @@ -8818,23 +8833,30 @@ def runMetadataTableActionService( return False def getMetadataTablePageService( - self, - projectId: int, - protocolId: int, - outputName: str, - tableName: str, - page: int, - pageSize: int, - sortBy: str, - asc: bool, - selectionOnly: bool, + self, + projectId: int, + protocolId: int, + outputName: str, + tableName: str, + page: int, + pageSize: int, + sortBy: str, + asc: bool, + selectionOnly: bool, + mapper=None, ): """ Return one logical page of rows for a metadata table. Sorting is currently delegated to the underlying DAO default order. """ with _metadataLock: - objMgr, table = self._openMetadataTable(projectId, protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable( + projectId, + protocolId, + outputName, + tableName, + mapper=mapper, + ) columns = list(table.getColumns()) if selectionOnly: @@ -8904,14 +8926,15 @@ def getMetadataTablePageService( } def exportMetadataTableService( - self, - projectId: int, - protocolId: int, - outputName: str, - tableName: str, - fmt: str, - selectionOnly: bool, - ids: Optional[List[int]], + self, + projectId: int, + protocolId: int, + outputName: str, + tableName: str, + fmt: str, + selectionOnly: bool, + ids: Optional[List[int]], + mapper=None, ) -> Response: """ Export metadata table as CSV or XLSX. @@ -8923,7 +8946,13 @@ def exportMetadataTableService( import csv with _metadataLock: - objMgr, table = self._openMetadataTable(projectId, protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable( + projectId, + protocolId, + outputName, + tableName, + mapper=mapper, + ) columns = list(table.getColumns()) colNames = [c.getName() for c in columns] @@ -9101,6 +9130,7 @@ def renderMetadataImageCellService( inline: bool, fmt: str, rowIndex: Optional[int] = None, + mapper=None, ) -> Response: """ Render one image cell from a metadata table using ImageRenderer. @@ -9114,7 +9144,13 @@ def renderMetadataImageCellService( from pathlib import Path as LocalPath # Resolve metadata root path for relative image paths - objMgr, table = self._openMetadataTable(projectId, protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable( + projectId, + protocolId, + outputName, + tableName, + mapper=mapper, + ) columns = list(table.getColumns()) metaDir = None @@ -9380,6 +9416,7 @@ def getMetadataTableWindowService( selectionOnly: bool, sortBy: str, asc: bool, + mapper=None, ): """ Return a window of rows for a metadata table using offset + limit. @@ -9394,7 +9431,13 @@ def getMetadataTableWindowService( - number of selected rows if selectionOnly == True """ with _metadataLock: - objMgr, table = self._openMetadataTable(projectId, protocolId, outputName, tableName) + objMgr, table = self._openMetadataTable( + projectId, + protocolId, + outputName, + tableName, + mapper=mapper, + ) columns = list(table.getColumns()) table.setSortingColumn(sortBy) table.setSortingAsc(asc) diff --git a/app/backend/database.py b/app/backend/database.py index 1e2b7f45..a207a324 100644 --- a/app/backend/database.py +++ b/app/backend/database.py @@ -66,8 +66,8 @@ def _requireEnv(key: str) -> str: Base = declarative_base() -def getMapper(): - # getPostgresqlMapper +def _createMapper(): + # createPostgresqlMapper from app.backend.mapper.postgresql import PostgresqlFlatMapper, PostgresqlDb dbName = _requireEnv("DATABASE_NAME") @@ -76,3 +76,20 @@ def getMapper(): db = PostgresqlDb(dbName=dbName, user=dbUser, password=dbPass) return PostgresqlFlatMapper(db) + + +def getMapper(): + # keepDirectMapperFactory + return _createMapper() + + +def getMapperDependency(): + # requestScopedMapperDependency + mapper = _createMapper() + try: + yield mapper + finally: + try: + mapper.db.close() + except Exception: + pass From e8c0bce36081c1cc143f47e13f4f03e91cec3454 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 10:28:53 +0200 Subject: [PATCH 036/278] add properties table to postgres metadata dao --- app/backend/viewers/postgresql_dao.py | 136 +++++++++++++++++++++++++- 1 file changed, 134 insertions(+), 2 deletions(-) diff --git a/app/backend/viewers/postgresql_dao.py b/app/backend/viewers/postgresql_dao.py index 4b3ad996..0a140e85 100644 --- a/app/backend/viewers/postgresql_dao.py +++ b/app/backend/viewers/postgresql_dao.py @@ -32,9 +32,11 @@ # * # ****************************************************************************** +import json import logging from typing import Any, Dict, Iterable, List, Optional + import numpy from metadataviewer.dao.model import IDAO @@ -67,8 +69,12 @@ PERMANENT_COLUMNS = ["id", "enabled"] OBJECT_TABLE = "objects" +PROPERTIES_TABLE = "Properties" ENABLED_COLUMN = "enabled" EXTENDED_COLUMN_NAME = "stack" +PROPERTY_KEY_COLUMN = "key" +PROPERTY_VALUE_COLUMN = "value" +ADITIONAL_INFO_DISPLAY_COLUMN_LIST = ["_size", "id"] def _guessType(value): @@ -145,6 +151,7 @@ def __init__( self._logicalTables: Dict[str, Dict[str, Any]] = {} self._tableColumns: Dict[str, List[Dict[str, Any]]] = {} self._useLogicalTables = False + self._tableWithAdditionalInfo = None # ------------------------------------------------------------------------- # metadata-viewer DAO API @@ -192,6 +199,7 @@ def getTables(self): int(logicalTable["id"]) ) + self._addPropertiesTable() return self._tables table = Table(OBJECT_TABLE) @@ -200,8 +208,17 @@ def getTables(self): self._tables[OBJECT_TABLE] = table self._columns = self._normalizeColumns(storedSet.get("columns") or []) + self._addPropertiesTable() return self._tables + def _addPropertiesTable(self) -> None: + if PROPERTIES_TABLE in self._tables: + return + + table = Table(PROPERTIES_TABLE) + table.setAlias(PROPERTIES_TABLE) + self._tables[PROPERTIES_TABLE] = table + def fillTable(self, table, objectManager): tableName = table.getName() if tableName != OBJECT_TABLE and not self._useLogicalTables: @@ -317,10 +334,28 @@ def addShiftColumn(name, offset, position): table.setSortingColumn(table.getColumns()[0].getName()) def _getColumnsForTable(self, tableName: str) -> List[Dict[str, Any]]: + if tableName == PROPERTIES_TABLE: + return self._getPropertiesColumns() + if self._useLogicalTables: return self._normalizeColumns(self._tableColumns.get(tableName) or []) + return self._columns + def _getPropertiesColumns(self) -> List[Dict[str, Any]]: + return [ + { + "labelProperty": PROPERTY_KEY_COLUMN, + "position": 0, + "className": "String", + }, + { + "labelProperty": PROPERTY_VALUE_COLUMN, + "position": 1, + "className": "String", + }, + ] + def _getLogicalTable(self, tableName: str) -> Optional[Dict[str, Any]]: if not self._useLogicalTables: return None @@ -431,7 +466,7 @@ def getColumnsValues( return values def getTableWithAdditionalInfo(self): - return None + return self._tableWithAdditionalInfo, ADITIONAL_INFO_DISPLAY_COLUMN_LIST def close(self): # Do not close the shared PostgreSQL connection here. @@ -544,6 +579,18 @@ def _getRows( ) -> List[Dict[str, Any]]: orderBy = str(orderBy or "id") + if tableName == PROPERTIES_TABLE: + rows = self._getPropertiesRows() + rows.sort( + key=lambda row: self._sortValue(row.get(orderBy)), + reverse=not orderAsc, + ) + + if limit is None: + return rows[start:] + + return rows[start:start + limit] + if self._useLogicalTables: logicalTable = self._getLogicalTable(tableName) if logicalTable is None: @@ -633,6 +680,12 @@ def _itemToRow( return row def _emptyRow(self, tableName: str) -> Dict[str, Any]: + if tableName == PROPERTIES_TABLE: + return { + PROPERTY_KEY_COLUMN: "", + PROPERTY_VALUE_COLUMN: "", + } + row = { "id": 0, "enabled": True, @@ -647,7 +700,9 @@ def _emptyRow(self, tableName: str) -> Dict[str, Any]: def getTableRowCount(self, tableName): if tableName not in self._tableCount: - if self._useLogicalTables: + if tableName == PROPERTIES_TABLE: + self._tableCount[tableName] = len(self._getPropertiesRows()) + elif self._useLogicalTables: logicalTable = self._getLogicalTable(tableName) if logicalTable is None: self._tableCount[tableName] = 0 @@ -677,6 +732,83 @@ def _getStoredSetTableItemsCount(self, tableId: int) -> int: # Metadata helpers # ------------------------------------------------------------------------- + def _getPropertiesRows(self) -> List[Dict[str, Any]]: + storedSet = self._getStoredSetHeader() + + rows: List[Dict[str, Any]] = [] + seen = set() + + self._addPropertyRow( + rows, + seen, + "self", + storedSet.get("setClassName") or self.outputName, + ) + + self._addPropertyRow( + rows, + seen, + "_size", + self._getStoredSetItemsCount(), + ) + + for prop in storedSet.get("setProperties") or []: + if not isinstance(prop, dict): + continue + + self._addPropertyRow( + rows, + seen, + prop.get("key"), + prop.get("value"), + ) + + properties = storedSet.get("properties") or {} + if isinstance(properties, dict): + for key in sorted(properties.keys()): + self._addPropertyRow( + rows, + seen, + key, + properties.get(key), + ) + + return rows + + def _addPropertyRow( + self, + rows: List[Dict[str, Any]], + seen: set, + key: Any, + value: Any, + ) -> None: + keyText = str(key or "").strip() + if not keyText or keyText in seen: + return + + seen.add(keyText) + rows.append( + { + PROPERTY_KEY_COLUMN: keyText, + PROPERTY_VALUE_COLUMN: self._normalizePropertyValue(value), + } + ) + + def _normalizePropertyValue(self, value: Any) -> str: + if value is None: + return "" + + if isinstance(value, numpy.ndarray): + value = value.tolist() + + if isinstance(value, (dict, list, tuple)): + try: + return json.dumps(value, sort_keys=True, default=str) + except Exception: + return str(value) + + return str(value) + def _normalizeColumns(self, columns: List[Dict[str, Any]]) -> List[Dict[str, Any]]: def sortKey(column: Dict[str, Any]): try: From 144b574b7e1f8dcf87677356c47a9cf7c62a979e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 10:40:09 +0200 Subject: [PATCH 037/278] add metadata table actions to postgres dao --- app/backend/viewers/postgresql_dao.py | 176 +++++++++++++++++++++++++- 1 file changed, 174 insertions(+), 2 deletions(-) diff --git a/app/backend/viewers/postgresql_dao.py b/app/backend/viewers/postgresql_dao.py index 0a140e85..cb650946 100644 --- a/app/backend/viewers/postgresql_dao.py +++ b/app/backend/viewers/postgresql_dao.py @@ -152,6 +152,8 @@ def __init__( self._tableColumns: Dict[str, List[Dict[str, Any]]] = {} self._useLogicalTables = False self._tableWithAdditionalInfo = None + self._aliases: Dict[str, str] = {} + self._objectsType: Dict[str, str] = {} # ------------------------------------------------------------------------- # metadata-viewer DAO API @@ -190,25 +192,33 @@ def getTables(self): if not tableName: continue + tableAlias = logicalTable.get("alias") or tableName + table = Table(tableName) - table.setAlias(logicalTable.get("alias") or tableName) + table.setAlias(tableAlias) self._tables[tableName] = table + self._aliases[tableName] = tableAlias self._logicalTables[tableName] = logicalTable self._tableColumns[tableName] = self.setMapper.getStoredSetTableColumns( int(logicalTable["id"]) ) self._addPropertiesTable() + self.composeObjectType() return self._tables + tableAlias = storedSet.get("setClassName") or self.outputName + table = Table(OBJECT_TABLE) - table.setAlias(storedSet.get("setClassName") or self.outputName) + table.setAlias(tableAlias) self._tables[OBJECT_TABLE] = table + self._aliases[OBJECT_TABLE] = tableAlias self._columns = self._normalizeColumns(storedSet.get("columns") or []) self._addPropertiesTable() + self.composeObjectType() return self._tables def _addPropertiesTable(self) -> None: @@ -218,6 +228,7 @@ def _addPropertiesTable(self) -> None: table = Table(PROPERTIES_TABLE) table.setAlias(PROPERTIES_TABLE) self._tables[PROPERTIES_TABLE] = table + self._aliases[PROPERTIES_TABLE] = PROPERTIES_TABLE def fillTable(self, table, objectManager): tableName = table.getName() @@ -330,9 +341,170 @@ def addShiftColumn(name, offset, position): addShiftColumn(colNamePrefix + "_shiftY", -5, 1) computedColsCount += 1 + self.generateTableActions(table, objectManager) + if table.getColumns(): table.setSortingColumn(table.getColumns()[0].getName()) + def composeObjectType(self) -> Dict[str, str]: + self._objectsType = {} + + rootAlias = self._getActionAliasForTableName(OBJECT_TABLE) + rootObjectType = self._getRootObjectType() + + if rootAlias and rootObjectType: + self._objectsType[rootAlias] = rootObjectType + + for tableName in list(self._tables.keys()): + if tableName == PROPERTIES_TABLE: + continue + + aliasText = self._getActionAliasForTableName(tableName) + if not aliasText: + continue + + aliasParts = aliasText.split("_") + if len(aliasParts) != 2: + continue + + objectName = aliasParts[1].strip() + if not objectName: + continue + + objectType = self._composeSetObjectTypeFromAliasPart(objectName) + if objectName not in self._objectsType: + self._objectsType[objectName] = objectType + + return self._objectsType + + def _getRootObjectType(self) -> str: + try: + storedSet = self._getStoredSetHeader() + except Exception: + return self.outputName + + setClassName = storedSet.get("setClassName") + if setClassName: + return str(setClassName) + + return self.outputName + + def _getActionAliasForTableName(self, tableName: str) -> str: + if tableName == PROPERTIES_TABLE: + return PROPERTIES_TABLE + + logicalTable = self._logicalTables.get(tableName) + if logicalTable: + itemClassName = str(logicalTable.get("itemClassName") or "").strip() + if itemClassName: + if tableName == OBJECT_TABLE: + return itemClassName + + if tableName.endswith("_Objects"): + prefix = tableName[: -len("_Objects")] + return "%s_%s" % (prefix, itemClassName) + + if tableName == OBJECT_TABLE: + try: + storedSet = self._getStoredSetHeader() + itemClassName = str(storedSet.get("itemClassName") or "").strip() + if itemClassName: + return itemClassName + except Exception: + pass + + return str(self._aliases.get(tableName) or tableName) + + def _composeSetObjectTypeFromAliasPart(self, objectName: str) -> str: + objectName = str(objectName or "").strip() + if not objectName: + return "" + + normalizedName = "ParticlesFlex" if objectName == "ParticleFlex" else objectName + lastChar = normalizedName[-1] + + suffix = ( + "s" + if lastChar in "aeiouAEIOU" or (lastChar != "s" and lastChar != "x") + else "" + ) + + return "SetOf%s%s" % (normalizedName, suffix) + + def generateTableActions(self, table, objectManager) -> None: + if table.getName() == PROPERTIES_TABLE: + return + + self.composeObjectType() + + alias = self._getActionAliasForTableName(table.getName()) + if not alias: + return + + aliasParts = alias.split("_") + + if alias.startswith("Class") and len(aliasParts) == 1: + rootObjectType = self._objectsType.get(alias) + if rootObjectType: + self._addTableAction(table, alias, rootObjectType, objectManager) + + for actionName, objectType in self._objectsType.items(): + if actionName == alias: + continue + + self._addTableAction(table, actionName, objectType, objectManager) + break + + if alias == "Class2D": + self._addTableAction(table, "Averages", "SetOfAverages", objectManager) + elif alias == "Class3D": + self._addTableAction(table, "Volumes", "SetOfVolumes", objectManager) + + elif alias.startswith("Class") and len(aliasParts) > 1: + actionName = aliasParts[1] + objectType = self._objectsType.get(actionName) + if objectType: + self._addTableAction(table, actionName, objectType, objectManager) + + elif alias in self._objectsType and str(self._objectsType[alias]).startswith("SetOf"): + self._addTableAction(table, alias, self._objectsType[alias], objectManager) + + def _addTableAction(self, table, actionName: str, objectType: str, objectManager) -> None: + actionName = str(actionName or "").strip() + objectType = str(objectType or "").strip() + + if not actionName or not objectType: + return + + try: + for action in table.getActions() or []: + if action.getName() == actionName: + return + except Exception: + pass + + try: + table.addAction( + actionName, + lambda table=table, objectType=objectType, objectManager=objectManager: + self.createSubsetCallback(table, objectType, objectManager), + ) + except Exception: + logger.debug( + "Could not add PostgreSQL metadata action '%s' for object type '%s'", + actionName, + objectType, + exc_info=True, + ) + + def createSubsetCallback(self, table: Table, objectType: str, objectManager) -> bool: + logger.info( + "PostgreSQL metadata subset action requested. table=%s objectType=%s", + table.getName(), + objectType, + ) + return False + def _getColumnsForTable(self, tableName: str) -> List[Dict[str, Any]]: if tableName == PROPERTIES_TABLE: return self._getPropertiesColumns() From 30c1de24e4dfa3cf11b864c5d267142431db8f05 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 10:49:51 +0200 Subject: [PATCH 038/278] Expose metadata actions for all tables --- app/backend/api/services/project_service.py | 44 +++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index ec4f818c..aa331485 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -8544,6 +8544,45 @@ def _openMetadataTable( return objMgr, table + def _isPropertiesMetadataTable(self, table) -> bool: + try: + tableName = str(table.getName() or "").strip() + except Exception: + tableName = "" + + try: + tableAlias = str(table.getAlias() or "").strip() + except Exception: + tableAlias = "" + + return tableName == "Properties" or tableAlias == "Properties" + + def _getMetadataTableActionNames(self, table) -> List[str]: + if self._isPropertiesMetadataTable(table): + return [] + + try: + tableActions = table.getActions() or [] + except Exception: + return [] + + actionNames: List[str] = [] + seen = set() + + for action in tableActions: + try: + actionName = str(action.getName() or "").strip() + except Exception: + actionName = "" + + if not actionName or actionName in seen: + continue + + seen.add(actionName) + actionNames.append(actionName) + + return actionNames + def _rendererTypeFromInstance(self, renderer) -> str: """ Map renderer class name to a simple type label for the API. @@ -8663,7 +8702,6 @@ def getMetadataTableSchemaService(self, projectId: int, visibleLabels = [] orderLabels = [] renderLabels = [] - actions = [] if table.getName() == SQLITE_OBJECT_TABLE: from pwem.viewers.viewers_data import RegistryViewerConfig @@ -8689,8 +8727,8 @@ def getMetadataTableSchemaService(self, projectId: int, visibleLabels = visibleLabelsStr.split() orderLabels = orderLabelsStr.split() renderLabels = renderLabelsStr.split() - for action in table.getActions(): - actions.append(action.getName()) + + actions = self._getMetadataTableActionNames(table) try: hasColumnId = table.hasColumnId() From f87fd68fc70dbe9a7b80f66ab7737c482d1c7ae1 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 11:08:25 +0200 Subject: [PATCH 039/278] Resolve postgres metadata volume actions --- app/backend/api/services/project_service.py | 203 ++++++++++++++++---- 1 file changed, 164 insertions(+), 39 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index aa331485..c5d4d012 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -8794,6 +8794,122 @@ def getMetadataTableSchemaService(self, projectId: int, return schema + def _normalizeMetadataSelectionIds(self, ids: List[int]) -> List[int]: + normalizedIds: List[int] = [] + + for rowId in ids or []: + try: + normalizedIds.append(int(rowId)) + except Exception: + continue + + if not normalizedIds: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Missing ids", + ) + + return normalizedIds + + def _writeMetadataSelectionFile(self, ids: List[int]) -> str: + timeFormat = "%Y%m%d%H%M%S" + timestamp = datetime.now().strftime(timeFormat) + relativePath = "Logs/selection_%s.txt" % timestamp + + projectPath = None + try: + projectPath = self.currentProject.getPath() + except Exception: + projectPath = None + + if projectPath: + absolutePath = os.path.join(projectPath, relativePath) + else: + absolutePath = relativePath + + directoryPath = os.path.dirname(absolutePath) + if directoryPath: + os.makedirs(directoryPath, exist_ok=True) + + try: + with open(absolutePath, "w") as file: + for rowId in ids: + file.write(str(rowId) + " ") + + logger.debug("Metadata selection file was created: %s", absolutePath) + except Exception as e: + logger.exception("Error creating metadata selection file: %s", absolutePath) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error creating selection file: {e}", + ) + + return relativePath + + def _buildMetadataSelectionArgument(self, selectionPath: str, tableName: str) -> str: + selectionArg = selectionPath + "," + + if tableName != OBJECT_TABLE: + selectionArg += tableName.split("_Objects")[0] + + return selectionArg + + def _getMetadataActionAliasForTable(self, dao, table) -> str: + getActionAliasFn = getattr(dao, "_getActionAliasForTableName", None) + if callable(getActionAliasFn): + try: + actionAlias = str(getActionAliasFn(table.getName()) or "").strip() + if actionAlias: + return actionAlias + except Exception: + pass + + try: + return str(table.getAlias() or "").strip() + except Exception: + return "" + + def _resolveMetadataActionOutputClassName(self, dao, table, action: str) -> str: + actionName = str(action or "").strip() + if not actionName: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Missing action", + ) + + validActions = self._getMetadataTableActionNames(table) + if actionName not in validActions: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Unsupported action '{actionName}' for table '{table.getName()}'", + ) + + actionAlias = self._getMetadataActionAliasForTable(dao, table) + + if actionAlias == "Class2D" and actionName == "Averages": + return "SetOfAverages" + + if actionAlias == "Class3D" and actionName == "Volumes": + return "SetOfVolumes" + + objectsType = getattr(dao, "_objectsType", {}) or {} + + if actionAlias == "Class2D": + objectsType.setdefault("Averages", "SetOfAverages") + + if actionAlias == "Class3D": + objectsType.setdefault("Volumes", "SetOfVolumes") + + outputClassName = objectsType.get(actionName) + + if outputClassName: + return str(outputClassName) + + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Could not resolve output class for action '{actionName}'", + ) + def runMetadataTableActionService( self, projectId: int, @@ -8806,21 +8922,7 @@ def runMetadataTableActionService( currentUser: Any, mapper: Any, ) -> Any: - timeFormat = '%Y%m%d%H%M%S' - now = datetime.now() - timestamp = now.strftime(timeFormat) - path = 'Logs/selection_%s.txt' % timestamp - try: - with open(path, 'w') as file: - for rowId in ids: - file.write(str(rowId) + ' ') - file.close() - logger.debug(f"The file: {path} was created correctly.") - except Exception as e: - logger.error(f"Error creating the file: {e}") - path += "," # Always add a comma, it is expected by the user subset protocol - if tableName != OBJECT_TABLE: - path += tableName.split('_Objects')[0] + selectionIds = self._normalizeMetadataSelectionIds(ids) try: protocol = self.currentProject.getProtocol(int(protocolId)) @@ -8830,45 +8932,68 @@ def runMetadataTableActionService( if not hasattr(protocol, outputName): raise HTTPException( status_code=404, - detail=f"Output '{outputName}' not found in protocol" + detail=f"Output '{outputName}' not found in protocol", ) output = getattr(protocol, outputName) if output is None: raise HTTPException( status_code=404, - detail=f"Output '{outputName}' is None" + detail=f"Output '{outputName}' is None", ) - getFileNameFn = getattr(output, "getFileName", None) - if not callable(getFileNameFn): - raise HTTPException( - status_code=404, - detail="Output has no metadata file (getFileName not available)" + with _metadataLock: + objMgr, table = self._openMetadataTable( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + tableName=tableName, + mapper=mapper, ) - objMgr = self._getMetadataObjectManager(str(output.getFileName())) + dao = objMgr.getDAO() + outputClassName = self._resolveMetadataActionOutputClassName( + dao=dao, + table=table, + action=action, + ) - dao = objMgr.getDAO() - table = objMgr.getTable(tableName) - dao.fillTable(table, objMgr) - if table.getAlias() == 'Class2D': - dao._objectsType['Averages'] = 'SetOfAverages' - elif table.getAlias() == 'Class3D': - dao._objectsType['Volumes'] = 'SetOfVolumes' - outputClassName = dao._objectsType[action] + selectionPath = self._writeMetadataSelectionFile(selectionIds) + selectionArg = self._buildMetadataSelectionArgument( + selectionPath=selectionPath, + tableName=tableName, + ) try: - batchProt = self.currentProject.newProtocol(ProtUserSubSet, - inputObject=output, - sqliteFile=path, - outputClassName=outputClassName, - other='', - label=subsetName) + batchProt = self.currentProject.newProtocol( + ProtUserSubSet, + inputObject=output, + sqliteFile=selectionArg, + outputClassName=outputClassName, + other="", + label=subsetName, + ) + self.currentProject.launchProtocol(batchProt) - return True + + return { + "success": True, + "message": "Subset protocol was launched successfully", + } + except Exception as e: - return False + logger.exception( + "Failed to launch metadata table action. projectId=%s protocolId=%s outputName=%s tableName=%s action=%s", + projectId, + protocolId, + outputName, + tableName, + action, + ) + return { + "success": False, + "errors": [str(e)], + } def getMetadataTablePageService( self, From 3266cc0a153b21e5a5afca9f4095c9bed362b862 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 12:20:57 +0200 Subject: [PATCH 040/278] add metadata action service tests --- tests/conftest.py | 1 + .../services/test_project_service_metadata.py | 319 ++++++++++++++++-- 2 files changed, 287 insertions(+), 33 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 55cc7028..69c2bdc9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -308,6 +308,7 @@ def exportMetadataTableService( fmt, selectionOnly, ids, + mapper, ): self.lastExportMetadataTableCall = { "projectId": projectId, diff --git a/tests/unit/backend/api/services/test_project_service_metadata.py b/tests/unit/backend/api/services/test_project_service_metadata.py index 2b2b2d10..322204a5 100644 --- a/tests/unit/backend/api/services/test_project_service_metadata.py +++ b/tests/unit/backend/api/services/test_project_service_metadata.py @@ -25,7 +25,6 @@ # ****************************************************************************** import importlib -from pathlib import Path import numpy as np import pytest @@ -192,8 +191,9 @@ def setSortingAsc(self, value): class FakeDao: # fakeDao - def __init__(self): - self._objectsType = {"create subset": "SetOfParticles"} + def __init__(self, objectsType=None, actionAliases=None): + self._objectsType = objectsType or {"create subset": "SetOfParticles"} + self._actionAliases = actionAliases or {} self.fillTableCalls = [] def fillTable(self, table, objMgr): @@ -204,14 +204,18 @@ def fillTable(self, table, objMgr): } ) + def _getActionAliasForTableName(self, tableName): + return self._actionAliases.get(tableName, "") + class FakeObjectManager: # fakeObjectManager - def __init__(self, tables, rowsByTable, rowCounts=None, dao=None): + def __init__(self, tables, rowsByTable, rowCounts=None, dao=None, fileName=""): self._tables = tables self._rowsByTable = rowsByTable self._rowCounts = rowCounts or {} self._dao = dao or FakeDao() + self._fileName = fileName def getTables(self): return self._tables @@ -260,13 +264,19 @@ def newProtocol(self, *args, **kwargs): class FakeCurrentProject: # fakeCurrentProject - def __init__(self, protocol): + def __init__(self, protocol, projectPath=None): self._protocol = protocol + self._projectPath = projectPath self.launchedProtocols = [] def getProtocol(self, protocolId): return self._protocol + def getPath(self): + if self._projectPath is None: + raise AttributeError("Project path is not set") + return str(self._projectPath) + def newProtocol(self, *args, **kwargs): return self._protocol.newProtocol(*args, **kwargs) @@ -290,11 +300,50 @@ def service(projectServiceModule, tmp_path): protocol = FakeProtocol("outputParticles", output) instance = object.__new__(projectServiceModule.ProjectService) - instance.currentProject = FakeCurrentProject(protocol) + instance.currentProject = FakeCurrentProject(protocol, projectPath=tmp_path) instance.tomoList = {} return instance +def patchOpenMetadataTable(service, monkeypatch, objMgr, table, calls=None): + # patchOpenMetadataTable + def openMetadataTable(projectId, protocolId, outputName, tableName, mapper=None): + if calls is not None: + calls.append( + { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tableName": tableName, + "mapper": mapper, + } + ) + return objMgr, table + + monkeypatch.setattr(service, "_openMetadataTable", openMetadataTable) + + +def patchObjectManagerForOutput(service, monkeypatch, objMgr, calls=None): + # patchObjectManagerForOutput + def getMetadataObjectManagerForOutput(projectId, protocolId, outputName, mapper=None): + if calls is not None: + calls.append( + { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + ) + return objMgr + + monkeypatch.setattr( + service, + "_getMetadataObjectManagerForOutput", + getMetadataObjectManagerForOutput, + ) + + def test_ListOutputMetadataTablesServiceReturnsSummaries(service, monkeypatch): tables = { "objects": FakeTable( @@ -315,20 +364,26 @@ def test_ListOutputMetadataTablesServiceReturnsSummaries(service, monkeypatch): rowsByTable={"objects": [], "classes": []}, rowCounts={"objects": 11, "classes": 3}, ) + mapper = object() + calls = [] - monkeypatch.setattr( - service, - "_resolveOutputForMetadata", - lambda protocolId, outputName: (None, None, "/tmp/fake.sqlite"), - ) - monkeypatch.setattr(service, "_getMetadataObjectManager", lambda metaPath: objMgr) + patchObjectManagerForOutput(service, monkeypatch, objMgr, calls=calls) result = service.listOutputMetadataTablesService( projectId=1, protocolId=10, outputName="outputParticles", + mapper=mapper, ) + assert calls == [ + { + "projectId": 1, + "protocolId": 10, + "outputName": "outputParticles", + "mapper": mapper, + } + ] assert result == [ { "name": "objects", @@ -358,16 +413,32 @@ def test_GetMetadataTableSchemaServiceBuildsColumns(service, monkeypatch): columns=columns, hasColumnId=True, ) + objMgr = FakeObjectManager( + tables={"objects": table}, + rowsByTable={"objects": []}, + ) + mapper = object() + calls = [] - monkeypatch.setattr(service, "_openMetadataTable", lambda protocolId, outputName, tableName: (object(), table)) + patchOpenMetadataTable(service, monkeypatch, objMgr, table, calls=calls) result = service.getMetadataTableSchemaService( projectId=1, protocolId=10, outputName="outputParticles", tableName="objects", + mapper=mapper, ) + assert calls == [ + { + "projectId": 1, + "protocolId": 10, + "outputName": "outputParticles", + "tableName": "objects", + "mapper": mapper, + } + ] assert result["name"] == "objects" assert result["alias"] == "Particles" assert result["hasColumnId"] is True @@ -416,6 +487,120 @@ def test_GetMetadataTableSchemaServiceBuildsColumns(service, monkeypatch): ] +def test_GetMetadataTableSchemaServiceReturnsActionsForChildTables(service, monkeypatch): + table = FakeTable( + name="Class001_Objects", + alias="Class001_Particle", + columns=[], + actions=[FakeAction("Particle"), FakeAction("Particle")], + ) + objMgr = FakeObjectManager( + tables={"Class001_Objects": table}, + rowsByTable={"Class001_Objects": []}, + ) + + patchOpenMetadataTable(service, monkeypatch, objMgr, table) + + result = service.getMetadataTableSchemaService( + projectId=1, + protocolId=10, + outputName="outputParticles", + tableName="Class001_Objects", + mapper=object(), + ) + + assert result["name"] == "Class001_Objects" + assert result["alias"] == "Class001_Particle" + assert result["actions"] == ["Particle"] + + +def test_GetMetadataTableSchemaServiceDoesNotReturnActionsForProperties(service, monkeypatch): + table = FakeTable( + name="Properties", + alias="Properties", + columns=[FakeColumn("key", "key", StrRenderer())], + actions=[FakeAction("Particle")], + ) + objMgr = FakeObjectManager( + tables={"Properties": table}, + rowsByTable={"Properties": []}, + ) + + patchOpenMetadataTable(service, monkeypatch, objMgr, table) + + result = service.getMetadataTableSchemaService( + projectId=1, + protocolId=10, + outputName="outputParticles", + tableName="Properties", + mapper=object(), + ) + + assert result["name"] == "Properties" + assert result["alias"] == "Properties" + assert result["actions"] == [] + + +def test_ResolveMetadataActionOutputClassNameReturnsSetOfVolumesForClass3D(service): + table = FakeTable( + name="objects", + alias="SetOfClasses3D", + columns=[], + actions=[FakeAction("Volumes")], + ) + dao = FakeDao( + objectsType={}, + actionAliases={"objects": "Class3D"}, + ) + + result = service._resolveMetadataActionOutputClassName( + dao=dao, + table=table, + action="Volumes", + ) + + assert result == "SetOfVolumes" + + +def test_ResolveMetadataActionOutputClassNameReturnsSetOfAveragesForClass2D(service): + table = FakeTable( + name="objects", + alias="SetOfClasses2D", + columns=[], + actions=[FakeAction("Averages")], + ) + dao = FakeDao( + objectsType={}, + actionAliases={"objects": "Class2D"}, + ) + + result = service._resolveMetadataActionOutputClassName( + dao=dao, + table=table, + action="Averages", + ) + + assert result == "SetOfAverages" + + +def test_ResolveMetadataActionOutputClassNameUsesDaoObjectsType(service): + table = FakeTable( + name="objects", + alias="Particles", + columns=[], + actions=[FakeAction("Particle")], + ) + dao = FakeDao(objectsType={"Particle": "SetOfParticles"}) + + result = service._resolveMetadataActionOutputClassName( + dao=dao, + table=table, + action="Particle", + ) + + assert result == "SetOfParticles" + + def test_GetMetadataTablePageServiceConvertsCells(service, monkeypatch): columns = [ FakeColumn("id", "Id", IntRenderer()), @@ -431,7 +616,7 @@ def test_GetMetadataTablePageServiceConvertsCells(service, monkeypatch): rowCounts={"objects": 1}, ) - monkeypatch.setattr(service, "_openMetadataTable", lambda protocolId, outputName, tableName: (objMgr, table)) + patchOpenMetadataTable(service, monkeypatch, objMgr, table) result = service.getMetadataTablePageService( projectId=1, @@ -443,6 +628,7 @@ def test_GetMetadataTablePageServiceConvertsCells(service, monkeypatch): sortBy="id", asc=True, selectionOnly=False, + mapper=object(), ) assert result == { @@ -480,7 +666,7 @@ def test_GetMetadataTableWindowServiceReturnsOffsetWindow(service, monkeypatch): rowCounts={"objects": 3}, ) - monkeypatch.setattr(service, "_openMetadataTable", lambda protocolId, outputName, tableName: (objMgr, table)) + patchOpenMetadataTable(service, monkeypatch, objMgr, table) result = service.getMetadataTableWindowService( projectId=1, @@ -492,6 +678,7 @@ def test_GetMetadataTableWindowServiceReturnsOffsetWindow(service, monkeypatch): selectionOnly=False, sortBy="label", asc=False, + mapper=object(), ) assert table.sortBy == "label" @@ -533,7 +720,7 @@ def test_ExportMetadataTableServiceReturnsCsv(service, monkeypatch): rowCounts={"objects": 2}, ) - monkeypatch.setattr(service, "_openMetadataTable", lambda protocolId, outputName, tableName: (objMgr, table)) + patchOpenMetadataTable(service, monkeypatch, objMgr, table) response = service.exportMetadataTableService( projectId=1, @@ -543,6 +730,7 @@ def test_ExportMetadataTableServiceReturnsCsv(service, monkeypatch): fmt="csv", selectionOnly=False, ids=None, + mapper=object(), ) text = response.body.decode("utf-8") @@ -560,14 +748,10 @@ def test_RenderMetadataImageCellServiceReturnsPlaceholderWhenRowMissing(service, tables={"objects": table}, rowsByTable={"objects": []}, rowCounts={"objects": 0}, + fileName="postgresql://project/1/protocol/10/output/outputParticles", ) - monkeypatch.setattr( - service, - "_resolveOutputForMetadata", - lambda protocolId, outputName: (None, None, "/tmp/fake.sqlite"), - ) - monkeypatch.setattr(service, "_openMetadataTable", lambda protocolId, outputName, tableName: (objMgr, table)) + patchOpenMetadataTable(service, monkeypatch, objMgr, table) response = service.renderMetadataImageCellService( projectId=1, @@ -581,6 +765,7 @@ def test_RenderMetadataImageCellServiceReturnsPlaceholderWhenRowMissing(service, applyTransform=False, inline=True, fmt="png", + mapper=object(), ) assert response.status_code == 200 @@ -599,7 +784,7 @@ def test_RunMetadataTableActionServiceLaunchesSubsetProtocol( output = FakeOutput(str(outputFile)) protocol = FakeProtocol("outputParticles", output) - service.currentProject = FakeCurrentProject(protocol) + service.currentProject = FakeCurrentProject(protocol, projectPath=tmp_path) table = FakeTable( name="objects", @@ -607,22 +792,19 @@ def test_RunMetadataTableActionServiceLaunchesSubsetProtocol( columns=[], actions=[FakeAction("create subset")], ) - dao = FakeDao() + dao = FakeDao(objectsType={"create subset": "SetOfParticles"}) objMgr = FakeObjectManager( tables={"objects": table}, rowsByTable={"objects": []}, dao=dao, ) + mapper = object() + calls = [] - monkeypatch.setattr(service, "_getMetadataObjectManager", lambda metaPath: objMgr) + patchOpenMetadataTable(service, monkeypatch, objMgr, table, calls=calls) monkeypatch.setattr(projectServiceModule, "OBJECT_TABLE", "objects") monkeypatch.setattr(projectServiceModule, "ProtUserSubSet", object()) - oldCwd = Path.cwd() - monkeypatch.chdir(tmp_path) - logsDir = tmp_path / "Logs" - logsDir.mkdir(parents=True, exist_ok=True) - result = service.runMetadataTableActionService( projectId=1, protocolId=10, @@ -632,15 +814,86 @@ def test_RunMetadataTableActionServiceLaunchesSubsetProtocol( subsetName="subset A", ids=[3, 5, 7], currentUser={"id": 1}, - mapper=object(), + mapper=mapper, ) - assert result is True + assert result == { + "success": True, + "message": "Subset protocol was launched successfully", + } + assert calls == [ + { + "projectId": 1, + "protocolId": 10, + "outputName": "outputParticles", + "tableName": "objects", + "mapper": mapper, + } + ] assert len(protocol.newProtocolCalls) == 1 call = protocol.newProtocolCalls[0] assert call["kwargs"]["inputObject"] is output assert call["kwargs"]["outputClassName"] == "SetOfParticles" assert call["kwargs"]["label"] == "subset A" + assert call["kwargs"]["sqliteFile"].startswith("Logs/selection_") + assert call["kwargs"]["sqliteFile"].endswith(".txt,") assert service.currentProject.launchedProtocols == [ {"batchProtocol": True, "kwargs": call["kwargs"]} - ] \ No newline at end of file + ] + + selectionFiles = list((tmp_path / "Logs").glob("selection_*.txt")) + assert len(selectionFiles) == 1 + assert selectionFiles[0].read_text(encoding="utf-8") == "3 5 7 " + + +def test_RunMetadataTableActionServiceBuildsChildTableSelectionArgument( + service, + projectServiceModule, + monkeypatch, + tmp_path, +): + outputFile = tmp_path / "metadata.sqlite" + outputFile.write_text("placeholder", encoding="utf-8") + + output = FakeOutput(str(outputFile)) + protocol = FakeProtocol("outputParticles", output) + service.currentProject = FakeCurrentProject(protocol, projectPath=tmp_path) + + table = FakeTable( + name="Class001_Objects", + alias="Class001_Particle", + columns=[], + actions=[FakeAction("Particle")], + ) + dao = FakeDao( + objectsType={"Particle": "SetOfParticles"}, + actionAliases={"Class001_Objects": "Class001_Particle"}, + ) + objMgr = FakeObjectManager( + tables={"Class001_Objects": table}, + rowsByTable={"Class001_Objects": []}, + dao=dao, + ) + + patchOpenMetadataTable(service, monkeypatch, objMgr, table) + monkeypatch.setattr(projectServiceModule, "OBJECT_TABLE", "objects") + monkeypatch.setattr(projectServiceModule, "ProtUserSubSet", object()) + + result = service.runMetadataTableActionService( + projectId=1, + protocolId=10, + outputName="outputParticles", + tableName="Class001_Objects", + action="Particle", + subsetName="class particles", + ids=[1, 2], + currentUser={"id": 1}, + mapper=object(), + ) + + assert result["success"] is True + assert len(protocol.newProtocolCalls) == 1 + call = protocol.newProtocolCalls[0] + assert call["kwargs"]["outputClassName"] == "SetOfParticles" + assert call["kwargs"]["sqliteFile"].startswith("Logs/selection_") + assert call["kwargs"]["sqliteFile"].endswith(".txt,Class001") \ No newline at end of file From b9f5a8a26da7c72cda7d9481e46bd9334a80525c Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 12:32:30 +0200 Subject: [PATCH 041/278] Add postgresql metadata dao tests --- tests/unit/backend/viewers/__init__.py | 25 ++ .../backend/viewers/test_postgresql_dao.py | 374 ++++++++++++++++++ 2 files changed, 399 insertions(+) create mode 100644 tests/unit/backend/viewers/__init__.py create mode 100644 tests/unit/backend/viewers/test_postgresql_dao.py diff --git a/tests/unit/backend/viewers/__init__.py b/tests/unit/backend/viewers/__init__.py new file mode 100644 index 00000000..4cdb527e --- /dev/null +++ b/tests/unit/backend/viewers/__init__.py @@ -0,0 +1,25 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** diff --git a/tests/unit/backend/viewers/test_postgresql_dao.py b/tests/unit/backend/viewers/test_postgresql_dao.py new file mode 100644 index 00000000..998307e9 --- /dev/null +++ b/tests/unit/backend/viewers/test_postgresql_dao.py @@ -0,0 +1,374 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +import importlib + +import pytest + + +class FakeDb: + def __init__(self, countsByTableId=None): + self.countsByTableId = countsByTableId or {} + self.fetchOneCalls = [] + + def fetchOne(self, sql, params=None): + self.fetchOneCalls.append( + { + "sql": sql, + "params": params, + } + ) + + tableId = None + if params: + try: + tableId = int(params[0]) + except Exception: + tableId = None + + return {"count": self.countsByTableId.get(tableId, 0)} + + +class FakeSetMapper: + def __init__( + self, + storedSet, + logicalTables=None, + columnsByTableId=None, + itemsByTableId=None, + ): + self.storedSet = storedSet + self.logicalTables = logicalTables or [] + self.columnsByTableId = columnsByTableId or {} + self.itemsByTableId = itemsByTableId or {} + self.getStoredSetCalls = [] + self.listStoredSetTablesCalls = [] + self.getStoredSetTableColumnsCalls = [] + self.getStoredSetTableItemsCalls = [] + + def getStoredSet(self, projectId, protocolDbId, outputName, limit=None, offset=0): + self.getStoredSetCalls.append( + { + "projectId": projectId, + "protocolDbId": protocolDbId, + "outputName": outputName, + "limit": limit, + "offset": offset, + } + ) + + storedSet = dict(self.storedSet) + items = list(storedSet.get("items") or []) + start = max(0, int(offset or 0)) + + if limit is None: + storedSet["items"] = items[start:] + else: + storedSet["items"] = items[start:start + int(limit)] + + return storedSet + + def listStoredSetTables(self, setId): + self.listStoredSetTablesCalls.append(setId) + return list(self.logicalTables) + + def getStoredSetTableColumns(self, tableId): + self.getStoredSetTableColumnsCalls.append(tableId) + return list(self.columnsByTableId.get(int(tableId), [])) + + def getStoredSetTableItems(self, tableId, limit=None, offset=0): + self.getStoredSetTableItemsCalls.append( + { + "tableId": tableId, + "limit": limit, + "offset": offset, + } + ) + + items = list(self.itemsByTableId.get(int(tableId), [])) + start = max(0, int(offset or 0)) + + if limit is None: + return items[start:] + + return items[start:start + int(limit)] + + +class FakeObjectManager: + def __init__(self, hiddenLabels=None): + self.hiddenLabels = set(hiddenLabels or []) + + def isLabelVisible(self, label): + return label not in self.hiddenLabels + + +@pytest.fixture +def postgresqlDaoModule(authTestEnv): + return importlib.import_module("app.backend.viewers.postgresql_dao") + + +def _makeStoredSet(setClassName="SetOfClasses3D", itemClassName="Class3D"): + return { + "id": 100, + "setClassName": setClassName, + "itemClassName": itemClassName, + "columns": [ + { + "labelProperty": "score", + "position": 0, + "className": "Float", + }, + ], + "items": [ + { + "id": 1, + "scipionItemId": 1, + "enabled": True, + "values": { + "score": 0.5, + }, + }, + { + "id": 2, + "scipionItemId": 2, + "enabled": True, + "values": { + "score": 0.7, + }, + }, + ], + "properties": { + "boxSize": 128, + "itemsCount": 2, + }, + "setProperties": [ + { + "key": "samplingRate", + "value": 1.25, + }, + ], + } + + +def _makeLogicalTables(rootItemClassName="Class3D", childItemClassName="Particle"): + return [ + { + "id": 1001, + "name": "objects", + "alias": "Root objects", + "itemClassName": rootItemClassName, + }, + { + "id": 1002, + "name": "Class001_Objects", + "alias": "Class001 objects", + "itemClassName": childItemClassName, + }, + ] + + +def _makeColumnsByTableId(): + return { + 1001: [ + { + "labelProperty": "score", + "position": 0, + "className": "Float", + }, + ], + 1002: [ + { + "labelProperty": "particleId", + "position": 0, + "className": "Integer", + }, + ], + } + + +def _makeItemsByTableId(): + return { + 1001: [ + { + "id": 1, + "scipionItemId": 1, + "enabled": True, + "values": { + "score": 0.5, + }, + }, + ], + 1002: [ + { + "id": 11, + "scipionItemId": 11, + "enabled": True, + "values": { + "particleId": 101, + }, + }, + ], + } + + +def _buildDao(postgresqlDaoModule, monkeypatch, setMapper, db=None): + db = db or FakeDb(countsByTableId={1001: 1, 1002: 1}) + monkeypatch.setattr( + postgresqlDaoModule, + "ScipionSetPostgresqlMapper", + lambda dbConnection: setMapper, + ) + + return postgresqlDaoModule.PostgresqlDAO( + db=db, + projectId=1, + protocolId=10, + outputName="outputClasses", + ) + + +def _buildLogicalDao( + postgresqlDaoModule, + monkeypatch, + setClassName="SetOfClasses3D", + rootItemClassName="Class3D", + childItemClassName="Particle", +): + storedSet = _makeStoredSet( + setClassName=setClassName, + itemClassName=rootItemClassName, + ) + logicalTables = _makeLogicalTables( + rootItemClassName=rootItemClassName, + childItemClassName=childItemClassName, + ) + setMapper = FakeSetMapper( + storedSet=storedSet, + logicalTables=logicalTables, + columnsByTableId=_makeColumnsByTableId(), + itemsByTableId=_makeItemsByTableId(), + ) + dao = _buildDao(postgresqlDaoModule, monkeypatch, setMapper) + + return dao, setMapper + + +def _getActionNames(table): + return [ + action.getName() + for action in table.getActions() + ] + + +def test_GetTablesAddsPropertiesTable(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao(postgresqlDaoModule, monkeypatch) + + tables = dao.getTables() + + assert set(tables.keys()) == {"objects", "Class001_Objects", "Properties"} + assert tables["Properties"].getAlias() == "Properties" + assert dao._useLogicalTables is True + assert dao._objectsType["Class3D"] == "SetOfClasses3D" + assert dao._objectsType["Particle"] == "SetOfParticles" + + +def test_GetPropertiesTableRowsIncludesSelfAndSize(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao(postgresqlDaoModule, monkeypatch) + dao.getTables() + + rowsByKey = { + row["key"]: row["value"] + for row in dao._getPropertiesRows() + } + + assert rowsByKey["self"] == "SetOfClasses3D" + assert rowsByKey["_size"] == "2" + assert rowsByKey["samplingRate"] == "1.25" + assert rowsByKey["boxSize"] == "128" + + +def test_GetPropertiesTableHasNoActions(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao(postgresqlDaoModule, monkeypatch) + table = dao.getTables()["Properties"] + + dao.fillTable(table, FakeObjectManager()) + + assert _getActionNames(table) == [] + + +def test_GenerateTableActionsAddsParticleAction(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao(postgresqlDaoModule, monkeypatch) + table = dao.getTables()["Class001_Objects"] + + dao.fillTable(table, FakeObjectManager()) + + assert _getActionNames(table) == ["Particle"] + + +def test_GenerateTableActionsAddsVolumesForClass3D(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao( + postgresqlDaoModule, + monkeypatch, + setClassName="SetOfClasses3D", + rootItemClassName="Class3D", + childItemClassName="Particle", + ) + table = dao.getTables()["objects"] + + dao.fillTable(table, FakeObjectManager()) + + actionNames = _getActionNames(table) + assert "Class3D" in actionNames + assert "Particle" in actionNames + assert "Volumes" in actionNames + + +def test_GenerateTableActionsAddsAveragesForClass2D(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao( + postgresqlDaoModule, + monkeypatch, + setClassName="SetOfClasses2D", + rootItemClassName="Class2D", + childItemClassName="Particle", + ) + table = dao.getTables()["objects"] + + dao.fillTable(table, FakeObjectManager()) + + actionNames = _getActionNames(table) + assert "Class2D" in actionNames + assert "Particle" in actionNames + assert "Averages" in actionNames + + +def test_GetTableWithAdditionalInfoReturnsCompatibilityTuple(postgresqlDaoModule, monkeypatch): + dao, _setMapper = _buildLogicalDao(postgresqlDaoModule, monkeypatch) + + table, displayColumns = dao.getTableWithAdditionalInfo() + + assert table is None + assert displayColumns == postgresqlDaoModule.ADITIONAL_INFO_DISPLAY_COLUMN_LIST + assert displayColumns == ["_size", "id"] \ No newline at end of file From 53feab4d639e0e529ca7a1251c45a7952a5b2830 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 12:54:51 +0200 Subject: [PATCH 042/278] test metadata router mapper propagation --- tests/conftest.py | 131 ++++++++++++++++++ tests/integration/api/test_projects_router.py | 112 +++++++++++++++ 2 files changed, 243 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 69c2bdc9..2db8445a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -149,6 +149,46 @@ def __init__(self): ) self.lastExportMetadataTableCall = None + + self.metadataTablesResult = [ + { + "name": "objects", + "alias": "Particles", + "rowCount": 1, + "hasColumnId": True, + } + ] + + self.lastListOutputMetadataTablesCall = None + + self.metadataTableSchemaResult = { + "name": "objects", + "alias": "Particles", + "hasColumnId": True, + "actions": ["Particle"], + "columns": [], + } + self.lastGetMetadataTableSchemaCall = None + + self.metadataTablePageResult = { + "pageNumber": 1, + "pageSize": 20, + "totalRows": 1, + "rows": [ + { + "id": 1, + "values": ["row-1"], + } + ], + } + self.lastGetMetadataTablePageCall = None + + self.renderMetadataImageCellResponse = PlainTextResponse( + "image-bytes", + media_type="image/png", + ) + self.lastRenderMetadataImageCellCall = None + self.listProjectsResult = [makeProjectOut()] self.lastListProjectsCall = None @@ -274,6 +314,96 @@ def resolveAnalyzeViewerDecision(self, projectId, protocolId, ctx): raise self.resolveViewerError return self.resolveViewerResult + def listOutputMetadataTablesService( + self, + projectId, + protocolId, + outputName, + mapper, + ): + self.lastListOutputMetadataTablesCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + return self.metadataTablesResult + + def getMetadataTableSchemaService( + self, + projectId, + protocolId, + outputName, + tableName, + mapper, + ): + self.lastGetMetadataTableSchemaCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tableName": tableName, + "mapper": mapper, + } + return self.metadataTableSchemaResult + + def getMetadataTablePageService( + self, + projectId, + protocolId, + outputName, + tableName, + page, + pageSize, + sortBy, + asc, + selectionOnly, + mapper, + ): + self.lastGetMetadataTablePageCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tableName": tableName, + "page": page, + "pageSize": pageSize, + "sortBy": sortBy, + "asc": asc, + "selectionOnly": selectionOnly, + "mapper": mapper, + } + return self.metadataTablePageResult + + def renderMetadataImageCellService( + self, + projectId, + protocolId, + outputName, + tableName, + rowId, + rowIndex, + columnName, + size, + applyTransform, + inline, + fmt, + mapper, + ): + self.lastRenderMetadataImageCellCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tableName": tableName, + "rowId": rowId, + "rowIndex": rowIndex, + "columnName": columnName, + "size": size, + "applyTransform": applyTransform, + "inline": inline, + "fmt": fmt, + "mapper": mapper, + } + return self.renderMetadataImageCellResponse + def runMetadataTableActionService( self, projectId, @@ -318,6 +448,7 @@ def exportMetadataTableService( "fmt": fmt, "selectionOnly": selectionOnly, "ids": ids, + "mapper": mapper, } return self.exportMetadataTableResponse diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 888bc1cb..95ae7ad9 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -181,6 +181,60 @@ def test_ResolveAnalyzeViewerReturnsHandledFalseOnUnexpectedError(projectClient, } +def test_ListMetadataTablesDelegatesMapperToService(projectClient, fakeProjectService): + fakeProjectService.metadataTablesResult = [ + { + "name": "objects", + "alias": "Particles", + "rowCount": 3, + "hasColumnId": True, + }, + { + "name": "Properties", + "alias": "Properties", + "rowCount": 2, + "hasColumnId": False, + }, + ] + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/metadata/tables" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.metadataTablesResult + assert fakeProjectService.lastListOutputMetadataTablesCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastListOutputMetadataTablesCall["mapper"], + } + + +def test_GetMetadataTableSchemaDelegatesMapperToService(projectClient, fakeProjectService): + fakeProjectService.metadataTableSchemaResult = { + "name": "objects", + "alias": "Particles", + "hasColumnId": True, + "actions": ["Particle"], + "columns": [], + } + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/schema" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.metadataTableSchemaResult + assert fakeProjectService.lastGetMetadataTableSchemaCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tableName": "objects", + "mapper": fakeProjectService.lastGetMetadataTableSchemaCall["mapper"], + } + + def test_RunMetadataTableActionRejectsMissingIds(projectClient): response = projectClient.post( "/projects/1/protocols/2/outputs/out/metadata/tables/table/actions", @@ -229,6 +283,35 @@ def test_RunMetadataTableActionUsesDefaultSubsetNameAndNormalizesServiceResult( } +def test_GetMetadataTablePageDelegatesMapperToService(projectClient, fakeProjectService): + fakeProjectService.metadataTablePageResult = { + "pageNumber": 2, + "pageSize": 50, + "totalRows": 1, + "rows": [{"id": 1, "values": ["row-1"]}], + } + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/page" + "?page=2&pageSize=50&sortBy=id&asc=false&selectionOnly=true" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.metadataTablePageResult + assert fakeProjectService.lastGetMetadataTablePageCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tableName": "objects", + "page": 2, + "pageSize": 50, + "sortBy": "id", + "asc": False, + "selectionOnly": True, + "mapper": fakeProjectService.lastGetMetadataTablePageCall["mapper"], + } + + def test_ExportMetadataTableRejectsInvalidIds(projectClient): response = projectClient.get( "/projects/1/protocols/2/outputs/out/metadata/tables/table/export?ids=1,abc,3" @@ -254,4 +337,33 @@ def test_ExportMetadataTableParsesIdsAndDelegatesToService(projectClient, fakePr "fmt": "csv", "selectionOnly": False, "ids": [1, 2, 3], + "mapper": fakeProjectService.lastExportMetadataTableCall["mapper"], + } + + +def test_RenderMetadataImageCellDelegatesMapperToService(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/image" + "?rowId=7&rowIndex=3&column=stack&size=128" + "&applyTransform=true&inline=false&fmt=jpeg" + ) + + assert response.status_code == 200 + assert response.text == "image-bytes" + + call = fakeProjectService.lastRenderMetadataImageCellCall + assert str(call["rowId"]) == "7" + assert call == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tableName": "objects", + "rowId": call["rowId"], + "rowIndex": 3, + "columnName": "stack", + "size": 128, + "applyTransform": True, + "inline": False, + "fmt": "jpeg", + "mapper": call["mapper"], } \ No newline at end of file From cc6341caeab4344a0d692b75468a329bd42c3ed9 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 14:02:19 +0200 Subject: [PATCH 043/278] support nested metadata tables for complex sets --- app/backend/mapper/scipion_set_mapper.py | 94 ++++++++++++++++++++---- app/backend/viewers/postgresql_dao.py | 9 ++- 2 files changed, 85 insertions(+), 18 deletions(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index 00223cda..4c884a04 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -34,6 +34,7 @@ SELF_LABEL = "self" +NESTED_LOGICAL_TABLES_VERSION = 2 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): @@ -76,6 +77,7 @@ def storeSet( skippedProperties["lastSkipReason"] = "unchanged_signature" skippedProperties["skippedLastSync"] = True skippedProperties["incremental"] = True + skippedProperties["nestedTablesVersion"] = NESTED_LOGICAL_TABLES_VERSION if sourceMTime is not None: skippedProperties["sourceMTime"] = sourceMTime @@ -113,6 +115,7 @@ def storeSet( itemClassName = self._getItemClassName(firstItem, itemSchema) columns = self._getSetColumns(itemSchema) initialProperties = self._getSetProperties(scipionSet) + initialProperties["nestedTablesVersion"] = NESTED_LOGICAL_TABLES_VERSION storedPaths: List[str] = [] with self.db.transaction(): @@ -174,6 +177,7 @@ def storeSet( finalProperties["lastSkipReason"] = None finalProperties["skippedLastSync"] = False finalProperties["incremental"] = True + finalProperties["nestedTablesVersion"] = NESTED_LOGICAL_TABLES_VERSION if sourceMTime is not None: finalProperties["sourceMTime"] = sourceMTime self._updateSetProperties(setId, finalProperties) @@ -495,11 +499,12 @@ def _upsertNestedLogicalTablesForItem( """ Persist child logical tables for complex Scipion set items. - First supported case: - - Class2D/Class3D-like items exposing iterItems() - -> ClassXXX_Objects table with the class members. + Supported examples: + - Class2D/Class3D items expose class members. + - TiltSeries items expose tilt images. + - Any item exposing iterItems() can expose a child logical table. """ - if not self._isClassLikeNestedItem(parentItem): + if not self._hasNestedLogicalItems(parentItem): return childIterator = iter(self._iterNestedItems(parentItem)) @@ -511,8 +516,8 @@ def _upsertNestedLogicalTablesForItem( childColumns = self._getSetColumns(childSchema) childItemClassName = self._getItemClassName(firstChild, childSchema) - tableName = self._getNestedClassTableName(parentItemId) - tableAlias = self._getNestedClassTableAlias(tableName, childItemClassName) + tableName = self._getNestedLogicalTableName(parentItem, parentItemId) + tableAlias = self._getNestedLogicalTableAlias(tableName, childItemClassName) childTableId = self._upsertSetTable( setId=setId, @@ -539,16 +544,12 @@ def _upsertNestedLogicalTablesForItem( batchSize=batchSize, ) - def _isClassLikeNestedItem(self, item: Any) -> bool: - className = self._getClassName(item) or item.__class__.__name__ - if not str(className).startswith("Class"): + def _hasNestedLogicalItems(self, item: Any) -> bool: + if item is None: return False iterItems = getattr(item, "iterItems", None) - if callable(iterItems): - return True - - return False + return callable(iterItems) def _iterNestedItems(self, item: Any) -> Iterable[Any]: iterItems = getattr(item, "iterItems", None) @@ -560,15 +561,70 @@ def _iterNestedItems(self, item: Any) -> Iterable[Any]: return iter(()) + def _getNestedLogicalTableName(self, parentItem: Any, parentItemId: int) -> str: + className = self._getClassName(parentItem) or parentItem.__class__.__name__ + + if str(className).startswith("Class"): + return self._getNestedClassTableName(parentItemId) + + stableName = self._getNestedItemStableName( + parentItem=parentItem, + parentItemId=parentItemId, + className=className, + ) + return "%s_Objects" % self._sanitizeLogicalTableNamePart(stableName) + + def _getNestedItemStableName( + self, + parentItem: Any, + parentItemId: int, + className: str, + ) -> str: + for getterName in ("getTsId", "getTomoId", "getObjLabel", "getName"): + value = self._callOptionalGetter(parentItem, getterName) + valueText = str(value or "").strip() + if valueText: + return valueText + + try: + return "%s%03d" % (className or "Item", int(parentItemId)) + except Exception: + return "%s%s" % (className or "Item", str(parentItemId)) + + def _sanitizeLogicalTableNamePart(self, value: Any) -> str: + text = str(value or "").strip() + chars = [] + previousWasUnderscore = False + + for char in text: + if char.isalnum(): + chars.append(char) + previousWasUnderscore = False + continue + + if char == "_" and not previousWasUnderscore: + chars.append("_") + previousWasUnderscore = True + continue + + if not previousWasUnderscore: + chars.append("_") + previousWasUnderscore = True + + cleanText = "".join(chars).strip("_") + return cleanText or "Item" + def _getNestedClassTableName(self, parentItemId: int) -> str: try: return "Class%03d_Objects" % int(parentItemId) except Exception: return "Class%s_Objects" % str(parentItemId) - def _getNestedClassTableAlias(self, tableName: str, childItemClassName: str) -> str: - cleanClassName = str(childItemClassName or "Objects") - return tableName.replace("_Objects", "_%s" % cleanClassName) + def _getNestedLogicalTableAlias(self, tableName: str, childItemClassName: str) -> str: + cleanClassName = str(childItemClassName or "Objects").strip() or "Objects" + if tableName.endswith("_Objects"): + return "%s_%s" % (tableName[: -len("_Objects")], cleanClassName) + return "%s_%s" % (tableName, cleanClassName) def _upsertLogicalTableItems( self, @@ -819,6 +875,12 @@ def _shouldSkipSetSync( if not existingProperties or not existingProperties.get("incremental"): return False + storedNestedTablesVersion = self._toOptionalInt( + existingProperties.get("nestedTablesVersion") + ) + if storedNestedTablesVersion != NESTED_LOGICAL_TABLES_VERSION: + return False + if itemsCountHint is None: return False diff --git a/app/backend/viewers/postgresql_dao.py b/app/backend/viewers/postgresql_dao.py index cb650946..c8c06063 100644 --- a/app/backend/viewers/postgresql_dao.py +++ b/app/backend/viewers/postgresql_dao.py @@ -363,7 +363,7 @@ def composeObjectType(self) -> Dict[str, str]: if not aliasText: continue - aliasParts = aliasText.split("_") + aliasParts = aliasText.rsplit("_", 1) if len(aliasParts) != 2: continue @@ -441,7 +441,7 @@ def generateTableActions(self, table, objectManager) -> None: if not alias: return - aliasParts = alias.split("_") + aliasParts = alias.rsplit("_", 1) if alias.startswith("Class") and len(aliasParts) == 1: rootObjectType = self._objectsType.get(alias) @@ -465,6 +465,11 @@ def generateTableActions(self, table, objectManager) -> None: objectType = self._objectsType.get(actionName) if objectType: self._addTableAction(table, actionName, objectType, objectManager) + elif len(aliasParts) > 1: + actionName = aliasParts[1] + objectType = self._objectsType.get(actionName) + if objectType: + self._addTableAction(table, actionName, objectType, objectManager) elif alias in self._objectsType and str(self._objectsType[alias]).startswith("SetOf"): self._addTableAction(table, alias, self._objectsType[alias], objectManager) From cf584f7ad8595076be34442c2d712b38a0568072 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 15:24:53 +0200 Subject: [PATCH 044/278] serve tilt series list from postgresql --- app/backend/api/routers/project_router.py | 1 + app/backend/api/services/project_service.py | 44 +++++ .../viewers/postgresql_tiltseries_reader.py | 157 ++++++++++++++++++ 3 files changed, 202 insertions(+) create mode 100644 app/backend/viewers/postgresql_tiltseries_reader.py diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 544dc01b..57dadd89 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1753,6 +1753,7 @@ def listOutputTiltSeries( projectId=projectId, protocolId=protocolId, outputName=outputName, + mapper=mapper, ) from fastapi.responses import JSONResponse diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index c5d4d012..b7fc10cd 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6047,6 +6047,39 @@ def _resolveOutputForTiltSeries(self, protocolId: int, outputName: str): return protocol, output + def _getPostgresqlTiltSeriesReaderIfAvailable( + self, + mapper, + projectId: int, + protocolId: int, + outputName: str, + ): + if mapper is None: + return None + + try: + from app.backend.viewers.postgresql_tiltseries_reader import PostgresqlTiltSeriesReader + + reader = PostgresqlTiltSeriesReader( + db=mapper.db, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if reader.hasOutput(): + return reader + + except Exception: + logger.exception( + "Failed to initialize PostgreSQL TiltSeries reader. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + ) + + return None + # ====================================================================== # Analyze Results: Resolve viewer # ====================================================================== @@ -6748,6 +6781,7 @@ def listOutputTiltSeriesService( projectId: int, protocolId: int, outputName: str, + mapper=None, ): """ List all tilt series in a SetOfTiltSeries-like output. @@ -6765,6 +6799,16 @@ def listOutputTiltSeriesService( ... ] """ + pgReader = self._getPostgresqlTiltSeriesReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + return pgReader.listTiltSeries() + _, setOfTiltSeries = self._resolveOutputForTiltSeries(protocolId, outputName) seriesList: List[Dict[str, Any]] = [] diff --git a/app/backend/viewers/postgresql_tiltseries_reader.py b/app/backend/viewers/postgresql_tiltseries_reader.py new file mode 100644 index 00000000..78de5f71 --- /dev/null +++ b/app/backend/viewers/postgresql_tiltseries_reader.py @@ -0,0 +1,157 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from typing import Any, Dict, List, Optional + +from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + +class PostgresqlTiltSeriesReader: + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + self.setMapper = ScipionSetPostgresqlMapper(db) + self._storedSet = None + self._logicalTables = None + + def hasOutput(self) -> bool: + return self._getStoredSet() is not None + + def listTiltSeries(self) -> List[Dict[str, Any]]: + storedSet = self._getStoredSet() + if storedSet is None: + return [] + + result = [] + for index, item in enumerate(storedSet.get("items") or []): + result.append(self._buildTiltSeriesSummary(item, index)) + + return result + + def _getStoredSet(self) -> Optional[Dict[str, Any]]: + if self._storedSet is None: + self._storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=self.protocolId, + outputName=self.outputName, + ) + return self._storedSet + + def _getLogicalTables(self) -> List[Dict[str, Any]]: + if self._logicalTables is None: + storedSet = self._getStoredSet() + if storedSet is None: + self._logicalTables = [] + else: + self._logicalTables = self.setMapper.listStoredSetTables( + int(storedSet["id"]) + ) + return self._logicalTables + + def _buildTiltSeriesSummary(self, item: Dict[str, Any], index: int) -> Dict[str, Any]: + values = item.get("values") or {} + itemId = item.get("scipionItemId") + + tiltSeriesId = self._firstValue( + values, + ["_tsId", "tsId", "tiltSeriesId", "id"], + ) + if tiltSeriesId is None: + tiltSeriesId = item.get("label") or itemId or index + + summary: Dict[str, Any] = { + "tiltSeriesId": str(tiltSeriesId), + "label": "TiltSeries %s" % str(tiltSeriesId), + } + + childTable = self._findChildTableForParentItem(itemId) + if childTable is not None: + childItems = self.setMapper.getStoredSetTableItems(int(childTable["id"])) + summary["nViews"] = len(childItems) + + dims = self._firstValueBySuffix( + values, + ["dim", "dims", "dimensions"], + ) + if dims is not None: + summary["dims"] = dims + + pixelSize = self._firstValueBySuffix( + values, + ["samplingrate", "pixelSize", "pixel_size"], + ) + if pixelSize is not None: + summary["pixelSize"] = self._toOptionalFloat(pixelSize) + + tiltAxisAngle = self._firstValueBySuffix( + values, + ["tiltaxisangle"], + ) + if tiltAxisAngle is not None: + summary["tiltAxisAngle"] = self._toOptionalFloat(tiltAxisAngle) + + return summary + + def _findChildTableForParentItem(self, parentItemId: Any) -> Optional[Dict[str, Any]]: + if parentItemId is None: + return None + + for table in self._getLogicalTables(): + if table.get("tableKind") != "child": + continue + if str(table.get("parentItemId")) == str(parentItemId): + return table + + return None + + def _firstValue(self, values: Dict[str, Any], keys: List[str]) -> Any: + for key in keys: + if key in values: + return values.get(key) + return None + + def _firstValueBySuffix(self, values: Dict[str, Any], suffixes: List[str]) -> Any: + normalizedSuffixes = [ + str(suffix).replace("_", "").replace(".", "").lower() + for suffix in suffixes + ] + + for key, value in values.items(): + normalizedKey = str(key).replace("_", "").replace(".", "").lower() + for suffix in normalizedSuffixes: + if normalizedKey.endswith(suffix): + return value + + return None + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + try: + return float(value) + except Exception: + return None \ No newline at end of file From 9f8f326337e945bf603a11221d2bca96e7cd6f43 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 15:43:19 +0200 Subject: [PATCH 045/278] serve tilt series frames from postgresql --- app/backend/api/routers/project_router.py | 1 + app/backend/api/services/project_service.py | 13 ++ .../viewers/postgresql_tiltseries_reader.py | 219 +++++++++++++++++- 3 files changed, 232 insertions(+), 1 deletion(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 57dadd89..dee79029 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1792,6 +1792,7 @@ def getTiltSeriesFrames( protocolId=protocolId, outputName=outputName, tiltSeriesId=tiltSeriesId, + mapper=mapper, ) from fastapi.responses import JSONResponse diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index b7fc10cd..de6851d0 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6827,6 +6827,7 @@ def getTiltSeriesFramesService( protocolId: int, outputName: str, tiltSeriesId: Union[int, str], + mapper=None, ): """ Return metadata for all tilt images in a given tilt series. @@ -6853,6 +6854,18 @@ def getTiltSeriesFramesService( ] } """ + pgReader = self._getPostgresqlTiltSeriesReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.getTiltSeriesFrames(tiltSeriesId) + if payload is not None: + return payload + protocol, setOfTiltSeries = self._resolveOutputForTiltSeries(protocolId, outputName) targetKey = str(tiltSeriesId) selectedSummary: Optional[Dict[str, Any]] = None diff --git a/app/backend/viewers/postgresql_tiltseries_reader.py b/app/backend/viewers/postgresql_tiltseries_reader.py index 78de5f71..308d9f63 100644 --- a/app/backend/viewers/postgresql_tiltseries_reader.py +++ b/app/backend/viewers/postgresql_tiltseries_reader.py @@ -23,6 +23,10 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ****************************************************************************** +import ast +import json +import math + from typing import Any, Dict, List, Optional from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper @@ -52,6 +56,31 @@ def listTiltSeries(self) -> List[Dict[str, Any]]: return result + def getTiltSeriesFrames(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: + seriesItem = self._findTiltSeriesItem(tiltSeriesId) + if seriesItem is None: + return None + + seriesSummary = self._buildTiltSeriesSummary(seriesItem, 0) + childTable = self._findChildTableForParentItem(seriesItem.get("scipionItemId")) + + frames: List[Dict[str, Any]] = [] + if childTable is not None: + childItems = self.setMapper.getStoredSetTableItems(int(childTable["id"])) + for index, item in enumerate(childItems): + frames.append(self._buildTiltImageFrame(item, index)) + + payload: Dict[str, Any] = { + "tiltSeriesId": seriesSummary.get("tiltSeriesId") or str(tiltSeriesId), + "label": seriesSummary.get("label") or str(tiltSeriesId), + "frames": frames, + } + + if "tiltAxisAngle" in seriesSummary: + payload["tiltAxisAngle"] = seriesSummary["tiltAxisAngle"] + + return payload + def _getStoredSet(self) -> Optional[Dict[str, Any]]: if self._storedSet is None: self._storedSet = self.setMapper.getStoredSet( @@ -116,6 +145,78 @@ def _buildTiltSeriesSummary(self, item: Dict[str, Any], index: int) -> Dict[str, return summary + def _findTiltSeriesItem(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: + storedSet = self._getStoredSet() + if storedSet is None: + return None + + targetKey = str(tiltSeriesId) + + for index, item in enumerate(storedSet.get("items") or []): + itemTiltSeriesId = self._getTiltSeriesIdFromItem(item, index) + if str(itemTiltSeriesId) == targetKey: + return item + + return None + + def _getTiltSeriesIdFromItem(self, item: Dict[str, Any], index: int) -> Any: + values = item.get("values") or {} + tiltSeriesId = self._firstValue( + values, + ["_tsId", "tsId", "tiltSeriesId", "id"], + ) + + if tiltSeriesId is not None: + return tiltSeriesId + + return item.get("label") or item.get("scipionItemId") or index + + def _buildTiltImageFrame(self, item: Dict[str, Any], position: int) -> Dict[str, Any]: + values = item.get("values") or {} + + frame: Dict[str, Any] = { + "viewId": item.get("scipionItemId"), + "index": position, + } + + imageIndex = self._firstValueBySuffix(values, ["index"]) + imageIndexInt = self._toOptionalInt(imageIndex) + if imageIndexInt is not None: + frame["index"] = imageIndexInt + + order = self._firstValueBySuffix( + values, + ["acquisitionorder", "acqorder", "order"], + ) + orderInt = self._toOptionalInt(order) + if orderInt is not None: + frame["order"] = orderInt + + tiltAngle = self._firstValueBySuffix(values, ["tiltangle"]) + tiltAngleFloat = self._toOptionalFloat(tiltAngle) + if tiltAngleFloat is not None: + frame["tiltAngle"] = tiltAngleFloat + + excluded = self._getFrameExcluded(item, values) + frame["excluded"] = excluded + + dose = self._firstValueBySuffix(values, ["accumdose", "dose"]) + doseFloat = self._toOptionalFloat(dose) + if doseFloat is not None: + frame["dose"] = doseFloat + + imagePath = self._firstValueBySuffix( + values, + ["filename", "filepath", "path"], + ) + if imagePath: + frame["path"] = "%s@%s" % (str(frame["index"]), str(imagePath)) + + transformValues = self._getFrameTransform(values) + frame.update(transformValues) + + return frame + def _findChildTableForParentItem(self, parentItemId: Any) -> Optional[Dict[str, Any]]: if parentItemId is None: return None @@ -154,4 +255,120 @@ def _toOptionalFloat(self, value: Any) -> Optional[float]: try: return float(value) except Exception: - return None \ No newline at end of file + return None + + def _getFrameExcluded(self, item: Dict[str, Any], values: Dict[str, Any]) -> bool: + excludedValue = self._firstValueBySuffix( + values, + ["excluded", "skip"], + ) + if excludedValue is not None: + return bool(self._toOptionalBool(excludedValue)) + + enabledValue = item.get("enabled") + if enabledValue is None: + enabledValue = self._firstValueBySuffix(values, ["enabled", "isenabled"]) + + enabled = self._toOptionalBool(enabledValue) + if enabled is None: + return False + + return not enabled + + def _getFrameTransform(self, values: Dict[str, Any]) -> Dict[str, Any]: + matrixValue = self._firstValueBySuffix(values, ["matrix"]) + matrix = self._parseMatrix(matrixValue) + if not matrix: + return {} + + flat = self._flattenMatrix(matrix) + result: Dict[str, Any] = {} + + if len(flat) >= 6: + result["shiftX"] = flat[2] + result["shiftY"] = flat[5] + + try: + if isinstance(matrix, list) and len(matrix) >= 2: + m00 = float(matrix[0][0]) + m10 = float(matrix[1][0]) + result["rot"] = math.degrees(-math.atan2(m10, m00)) + except Exception: + pass + + return result + + def _parseMatrix(self, value: Any) -> Optional[Any]: + if value is None or value == "": + return None + + if isinstance(value, list): + return value + + if isinstance(value, tuple): + return list(value) + + if isinstance(value, str): + text = value.strip() + if not text: + return None + + try: + return json.loads(text) + except Exception: + pass + + try: + return ast.literal_eval(text) + except Exception: + return None + + return None + + def _flattenMatrix(self, matrix: Any) -> List[float]: + values: List[float] = [] + + if not isinstance(matrix, list): + return values + + for row in matrix: + if isinstance(row, list): + for value in row: + parsedValue = self._toOptionalFloat(value) + if parsedValue is not None: + values.append(parsedValue) + else: + parsedValue = self._toOptionalFloat(row) + if parsedValue is not None: + values.append(parsedValue) + + return values + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except Exception: + try: + return int(float(value)) + except Exception: + return None + + def _toOptionalBool(self, value: Any) -> Optional[bool]: + if value is None or value == "": + return None + + if isinstance(value, bool): + return value + + if isinstance(value, (int, float)): + return bool(value) + + text = str(value).strip().lower() + if text in ("1", "true", "yes", "y", "on", "enabled"): + return True + if text in ("0", "false", "no", "n", "off", "disabled"): + return False + + return None \ No newline at end of file From 843249a65abf15f692528eff2c5d757beaca94be Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 16:02:10 +0200 Subject: [PATCH 046/278] Render tilt series images from postgresql --- app/backend/api/routers/project_router.py | 1 + app/backend/api/services/project_service.py | 99 +++++++++++++++++++ .../viewers/postgresql_tiltseries_reader.py | 19 ++++ 3 files changed, 119 insertions(+) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index dee79029..498750e7 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1865,6 +1865,7 @@ def renderTiltSeriesImage( fmt=fmt, applyTransform=applyTransform, inline=inline, + mapper=mapper, ) resp.headers["X-Debug-Auth"] = "ok" diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index de6851d0..12126aa0 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6080,6 +6080,30 @@ def _getPostgresqlTiltSeriesReaderIfAvailable( return None + def _parseTiltSeriesFramePath( + self, + framePath: Any, + fallbackIndex: int, + ) -> Tuple[str, int]: + pathText = str(framePath or "").strip() + if not pathText: + raise HTTPException( + status_code=404, + detail="Tilt image path not found in PostgreSQL metadata", + ) + + imageIndex = int(fallbackIndex) + imagePath = pathText + + if "@" in pathText: + indexText, imagePath = pathText.split("@", 1) + try: + imageIndex = int(float(indexText)) + except Exception: + imageIndex = int(fallbackIndex) + + return os.path.abspath(imagePath), imageIndex + # ====================================================================== # Analyze Results: Resolve viewer # ====================================================================== @@ -7182,7 +7206,82 @@ def renderTiltSeriesImageService( applyTransform: bool = True, inline: bool = True, requestHeaders: Optional[Dict[str, str]] = None, + mapper=None, ): + + pgReader = self._getPostgresqlTiltSeriesReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + frame = pgReader.getTiltImageFrame(tiltSeriesId, index) + if frame is not None: + try: + imagePath, imageIndex = self._parseTiltSeriesFramePath( + frame.get("path"), + fallbackIndex=int(index), + ) + + cacheKey = self._buildTiltSeriesPreviewCacheKey( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + tiltSeriesId=tiltSeriesId, + index=imageIndex, + size=size, + fmt=fmt, + applyTransform=applyTransform, + inline=inline, + imagePath=imagePath, + ) + + cachedResponse = self._getTiltSeriesPreviewFromCache(cacheKey) + if cachedResponse is not None: + return cachedResponse + + rot = None + shifts = None + + if applyTransform: + rot = frame.get("rot") + shiftX = frame.get("shiftX") + shiftY = frame.get("shiftY") + if shiftX is not None and shiftY is not None: + shifts = (shiftX, shiftY) + + preview = OutputsPreview( + currentProject=self.currentProject, + protocol=None, + output=None, + requestHeaders=requestHeaders, + ) + + response = preview.renderImageFromFilePath( + imagePath, + size=size, + fmt=fmt, + index=imageIndex, + applyTransform=applyTransform, + inline=inline, + rot=rot, + shifts=shifts, + ) + + return self._storeTiltSeriesPreviewInCache(cacheKey, response) + + except Exception: + logger.exception( + "Failed to render TiltSeries image from PostgreSQL. projectId=%s protocolId=%s outputName=%s tiltSeriesId=%s index=%s", + projectId, + protocolId, + outputName, + tiltSeriesId, + index, + ) + protocol, setOfTiltSeries = self._resolveOutputForTiltSeries(protocolId, outputName) ts = setOfTiltSeries.getItem('_tsId', tiltSeriesId) diff --git a/app/backend/viewers/postgresql_tiltseries_reader.py b/app/backend/viewers/postgresql_tiltseries_reader.py index 308d9f63..af44bbf0 100644 --- a/app/backend/viewers/postgresql_tiltseries_reader.py +++ b/app/backend/viewers/postgresql_tiltseries_reader.py @@ -81,6 +81,25 @@ def getTiltSeriesFrames(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: return payload + def getTiltImageFrame(self, tiltSeriesId: Any, index: Any) -> Optional[Dict[str, Any]]: + payload = self.getTiltSeriesFrames(tiltSeriesId) + if not payload: + return None + + frames = payload.get("frames") or [] + targetIndex = self._toOptionalInt(index) + + if targetIndex is not None: + for frame in frames: + frameIndex = self._toOptionalInt(frame.get("index")) + if frameIndex == targetIndex: + return frame + + if 0 <= targetIndex < len(frames): + return frames[targetIndex] + + return None + def _getStoredSet(self) -> Optional[Dict[str, Any]]: if self._storedSet is None: self._storedSet = self.setMapper.getStoredSet( From 23e990d739c8bda1d6815b1628f5183b14263bac Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 16:05:54 +0200 Subject: [PATCH 047/278] Render tilt series batch from postgresql --- app/backend/api/routers/project_router.py | 1 + app/backend/api/services/project_service.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 498750e7..ca9f11ab 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1913,6 +1913,7 @@ def renderTiltSeriesImagesBatch( fmt=payload.fmt, applyTransform=payload.applyTransform, inline=payload.inline, + mapper=mapper ) resp = JSONResponse(result or {}) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 12126aa0..0182da26 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -7358,6 +7358,7 @@ def renderTiltSeriesImagesBatchService( applyTransform: bool = True, inline: bool = True, requestHeaders: Optional[Dict[str, str]] = None, + mapper=None, ) -> Dict[str, Any]: # renderTiltSeriesImagesBatchService cleanIndices: List[int] = [] @@ -7394,6 +7395,7 @@ def renderTiltSeriesImagesBatchService( applyTransform=applyTransform, inline=inline, requestHeaders=requestHeaders, + mapper=mapper, ) body = getattr(response, "body", None) or b"" From 8e559bf459739df66cadb62196050f2b83738fbd Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 17:48:09 +0200 Subject: [PATCH 048/278] serve ctftomo series list from postgresql --- app/backend/api/routers/project_router.py | 7 +- app/backend/api/services/project_service.py | 44 +++++ .../viewers/postgresql_ctftomo_reader.py | 160 ++++++++++++++++++ 3 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 app/backend/viewers/postgresql_ctftomo_reader.py diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index ca9f11ab..a45a46e6 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2018,7 +2018,12 @@ def listCtftomoSeries( if not project: raise HTTPException(status_code=404, detail="Project not found") - return service.listOutputCtftomoSeriesService(projectId, protocolId, outputName) + return service.listOutputCtftomoSeriesService( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + ) @router.get( diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 0182da26..904f7992 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6237,6 +6237,7 @@ def listOutputCtftomoSeriesService( projectId: int, protocolId: int, outputName: str, + mapper=None, ): """ List all CTFTomoSeries in a CTFTomo output. @@ -6254,6 +6255,16 @@ def listOutputCtftomoSeriesService( ... ] """ + pgReader = self._getPostgresqlCtftomoReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + return pgReader.listCtftomoSeries() + protocol, output = self._resolveOutputForCtftomoSeries(protocolId, outputName) seriesList: List[Dict[str, Any]] = [] @@ -6435,6 +6446,39 @@ def _buildTiltSeriesSummary(self, ts) -> Dict[str, Any]: return item + def _getPostgresqlCtftomoReaderIfAvailable( + self, + mapper, + projectId: int, + protocolId: int, + outputName: str, + ): + if mapper is None: + return None + + try: + from app.backend.viewers.postgresql_ctftomo_reader import PostgresqlCtftomoReader + + reader = PostgresqlCtftomoReader( + db=mapper.db, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if reader.hasOutput(): + return reader + + except Exception: + logger.exception( + "Failed to initialize PostgreSQL CTFTomo reader. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + ) + + return None + # ====================================================================== # Analyze Results: Volumes (Volume / VolumeMask / SetOfVolumes) # ====================================================================== diff --git a/app/backend/viewers/postgresql_ctftomo_reader.py b/app/backend/viewers/postgresql_ctftomo_reader.py new file mode 100644 index 00000000..2f340ce5 --- /dev/null +++ b/app/backend/viewers/postgresql_ctftomo_reader.py @@ -0,0 +1,160 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from typing import Any, Dict, List, Optional + +from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + +class PostgresqlCtftomoReader: + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + self.setMapper = ScipionSetPostgresqlMapper(db) + self._storedSet = None + self._logicalTables = None + + def hasOutput(self) -> bool: + return self._getStoredSet() is not None + + def listCtftomoSeries(self) -> List[Dict[str, Any]]: + storedSet = self._getStoredSet() + if storedSet is None: + return [] + + result = [] + for index, item in enumerate(storedSet.get("items") or []): + summary = self._buildCtftomoSeriesSummary(item, index) + result.append(summary) + + return result + + def _getStoredSet(self) -> Optional[Dict[str, Any]]: + if self._storedSet is None: + self._storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=self.protocolId, + outputName=self.outputName, + ) + return self._storedSet + + def _getLogicalTables(self) -> List[Dict[str, Any]]: + if self._logicalTables is None: + storedSet = self._getStoredSet() + if storedSet is None: + self._logicalTables = [] + else: + self._logicalTables = self.setMapper.listStoredSetTables( + int(storedSet["id"]) + ) + return self._logicalTables + + def _buildCtftomoSeriesSummary(self, item: Dict[str, Any], index: int) -> Dict[str, Any]: + values = item.get("values") or {} + itemId = item.get("scipionItemId") + + tiltSeriesId = self._firstValue( + values, + ["_tsId", "tsId", "tiltSeriesId", "id"], + ) + if tiltSeriesId is None: + tiltSeriesId = item.get("label") or itemId or index + + label = self._firstValueBySuffix( + values, + ["objlabel", "label", "name"], + ) + if label is None: + label = str(item.get("label") or "CTFTomoSeries %s" % str(tiltSeriesId)) + + summary: Dict[str, Any] = { + "tiltSeriesId": str(tiltSeriesId), + "label": str(label), + "index": index, + } + + childTable = self._findChildTableForParentItem(itemId) + if childTable is not None: + childItems = self.setMapper.getStoredSetTableItems(int(childTable["id"])) + summary["nViews"] = len(childItems) + + dims = self._firstValueBySuffix(values, ["dim", "dims", "dimensions"]) + if dims is not None: + summary["dims"] = dims + + pixelSize = self._firstValueBySuffix( + values, + ["samplingrate", "pixelSize", "pixel_size"], + ) + if pixelSize is not None: + summary["pixelSize"] = self._toOptionalFloat(pixelSize) + + tiltAxisAngle = self._firstValueBySuffix(values, ["tiltaxisangle"]) + if tiltAxisAngle is not None: + summary["tiltAxisAngle"] = self._toOptionalFloat(tiltAxisAngle) + + return summary + + def _findChildTableForParentItem(self, parentItemId: Any) -> Optional[Dict[str, Any]]: + if parentItemId is None: + return None + + for table in self._getLogicalTables(): + if table.get("tableKind") != "child": + continue + if str(table.get("parentItemId")) == str(parentItemId): + return table + + return None + + def _firstValue(self, values: Dict[str, Any], keys: List[str]) -> Any: + for key in keys: + if key in values: + return values.get(key) + return None + + def _firstValueBySuffix(self, values: Dict[str, Any], suffixes: List[str]) -> Any: + normalizedSuffixes = [ + str(suffix).replace("_", "").replace(".", "").lower() + for suffix in suffixes + ] + + for key, value in values.items(): + normalizedKey = str(key).replace("_", "").replace(".", "").lower() + for suffix in normalizedSuffixes: + if normalizedKey.endswith(suffix): + return value + + return None + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + try: + return float(value) + except Exception: + return None \ No newline at end of file From 8dadb94bb276416e25e3c1b9e6fd68114d70a3b0 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 17:55:15 +0200 Subject: [PATCH 049/278] serve ctftomo views from postgresql --- app/backend/api/routers/project_router.py | 8 +- app/backend/api/services/project_service.py | 13 ++ .../viewers/postgresql_ctftomo_reader.py | 144 +++++++++++++++++- 3 files changed, 163 insertions(+), 2 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index a45a46e6..8316c8b1 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2046,7 +2046,13 @@ def getCtftomoSeriesViews( if not project: raise HTTPException(status_code=404, detail="Project not found") - return service.getCtftomoSeriesViewsService(projectId, protocolId, outputName, tiltSeriesId) + return service.getCtftomoSeriesViewsService( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + tiltSeriesId=tiltSeriesId, + mapper=mapper, + ) @router.post( diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 904f7992..a36864a2 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6285,6 +6285,7 @@ def getCtftomoSeriesViewsService( protocolId: int, outputName: str, tiltSeriesId: Union[int, str], + mapper=None, ): """ Return all CTF measurements for one tilt series inside a CTFTomo output. @@ -6318,6 +6319,18 @@ def getCtftomoSeriesViewsService( ] } """ + pgReader = self._getPostgresqlCtftomoReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.getCtftomoSeriesViews(tiltSeriesId) + if payload is not None: + return payload + protocol, output = self._resolveOutputForCtftomoSeries(protocolId, outputName) targetKey = str(tiltSeriesId) diff --git a/app/backend/viewers/postgresql_ctftomo_reader.py b/app/backend/viewers/postgresql_ctftomo_reader.py index 2f340ce5..f943e1d0 100644 --- a/app/backend/viewers/postgresql_ctftomo_reader.py +++ b/app/backend/viewers/postgresql_ctftomo_reader.py @@ -53,6 +53,26 @@ def listCtftomoSeries(self) -> List[Dict[str, Any]]: return result + def getCtftomoSeriesViews(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: + seriesItem = self._findCtftomoSeriesItem(tiltSeriesId) + if seriesItem is None: + return None + + summary = self._buildCtftomoSeriesSummary(seriesItem, 0) + childTable = self._findChildTableForParentItem(seriesItem.get("scipionItemId")) + + frames: List[Dict[str, Any]] = [] + if childTable is not None: + childItems = self.setMapper.getStoredSetTableItems(int(childTable["id"])) + for index, item in enumerate(childItems): + frames.append(self._buildCtftomoMeasurementFrame(item, index)) + + summary["frames"] = frames + summary["tiltSeriesId"] = summary.get("tiltSeriesId") or str(tiltSeriesId) + summary["nViews"] = len(frames) + + return summary + def _getStoredSet(self) -> Optional[Dict[str, Any]]: if self._storedSet is None: self._storedSet = self.setMapper.getStoredSet( @@ -119,6 +139,99 @@ def _buildCtftomoSeriesSummary(self, item: Dict[str, Any], index: int) -> Dict[s return summary + def _findCtftomoSeriesItem(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: + storedSet = self._getStoredSet() + if storedSet is None: + return None + + targetKey = str(tiltSeriesId) + + for index, item in enumerate(storedSet.get("items") or []): + itemTiltSeriesId = self._getTiltSeriesIdFromItem(item, index) + if str(itemTiltSeriesId) == targetKey: + return item + + return None + + def _getTiltSeriesIdFromItem(self, item: Dict[str, Any], index: int) -> Any: + values = item.get("values") or {} + tiltSeriesId = self._firstValue( + values, + ["_tsId", "tsId", "tiltSeriesId", "id"], + ) + + if tiltSeriesId is not None: + return tiltSeriesId + + return item.get("label") or item.get("scipionItemId") or index + + def _buildCtftomoMeasurementFrame(self, item: Dict[str, Any], position: int) -> Dict[str, Any]: + values = item.get("values") or {} + + frame: Dict[str, Any] = { + "index": item.get("scipionItemId") or position, + "viewIndex": item.get("scipionItemId") or position, + } + + tiltAngle = self._firstValueBySuffix(values, ["tiltangle"]) + tiltAngleFloat = self._toOptionalFloat(tiltAngle) + if tiltAngleFloat is not None: + frame["tiltAngle"] = tiltAngleFloat + + dose = self._firstValueBySuffix(values, ["accumdose", "dose"]) + doseFloat = self._toOptionalFloat(dose) + if doseFloat is not None: + frame["dose"] = doseFloat + + defocusU = self._firstValueBySuffix(values, ["defocusu"]) + defocusUFloat = self._toOptionalFloat(defocusU) + if defocusUFloat is not None: + frame["defocusU"] = defocusUFloat + + defocusV = self._firstValueBySuffix(values, ["defocusv"]) + defocusVFloat = self._toOptionalFloat(defocusV) + if defocusVFloat is not None: + frame["defocusV"] = defocusVFloat + + if defocusUFloat is not None and defocusVFloat is not None: + frame["astigmatism"] = defocusUFloat - defocusVFloat + + defocusAngle = self._firstValueBySuffix(values, ["defocusangle"]) + defocusAngleFloat = self._toOptionalFloat(defocusAngle) + if defocusAngleFloat is not None: + frame["defocusAngle"] = defocusAngleFloat + + resolution = self._firstValueBySuffix(values, ["resolution"]) + resolutionFloat = self._toOptionalFloat(resolution) + if resolutionFloat is not None: + frame["resolution"] = resolutionFloat + + phaseShift = self._firstValueBySuffix(values, ["phaseshift"]) + phaseShiftFloat = self._toOptionalFloat(phaseShift) + if phaseShiftFloat is not None: + frame["phaseShift"] = phaseShiftFloat + + acquisitionOrder = self._firstValueBySuffix( + values, + ["acquisitionorder", "acqorder", "order"], + ) + acquisitionOrderInt = self._toOptionalInt(acquisitionOrder) + if acquisitionOrderInt is not None: + frame["order"] = acquisitionOrderInt + + psdFile = self._firstValueBySuffix( + values, + ["psdfile", "psdfilename", "psdpath"], + ) + if psdFile: + frame["psdFile"] = str(psdFile) + + enabled = self._firstValueBySuffix(values, ["enabled", "isenabled"]) + enabledBool = self._toOptionalBool(enabled) + frame["excluded"] = False if enabledBool is None else not enabledBool + + return frame + def _findChildTableForParentItem(self, parentItemId: Any) -> Optional[Dict[str, Any]]: if parentItemId is None: return None @@ -157,4 +270,33 @@ def _toOptionalFloat(self, value: Any) -> Optional[float]: try: return float(value) except Exception: - return None \ No newline at end of file + return None + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except Exception: + try: + return int(float(value)) + except Exception: + return None + + def _toOptionalBool(self, value: Any) -> Optional[bool]: + if value is None or value == "": + return None + + if isinstance(value, bool): + return value + + if isinstance(value, (int, float)): + return bool(value) + + text = str(value).strip().lower() + if text in ("1", "true", "yes", "y", "on", "enabled"): + return True + if text in ("0", "false", "no", "n", "off", "disabled"): + return False + + return None \ No newline at end of file From b2c6b32131e5146d30a25f5dba667bffae115a8f Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 18:07:59 +0200 Subject: [PATCH 050/278] render ctftomo psd from postgresql metadata --- app/backend/api/routers/project_router.py | 1 + app/backend/api/services/project_service.py | 45 +++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 8316c8b1..ce3006c4 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2171,6 +2171,7 @@ def renderCtftomoPsdImage( fmt=fmt, applyTransform=applyTransform, inline=inline, + mapper=mapper, ) resp.headers["X-Debug-Auth"] = "ok" resp.headers["X-Debug-UserId"] = str( diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index a36864a2..b9031339 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6374,6 +6374,7 @@ def renderCtfTomoPsdImageService( applyTransform: bool = False, rot=None, shifts=None, + mapper=None, ) -> Response: """ Render a single CTFtomo PSD image using the OutputsPreview pipeline. @@ -6384,6 +6385,50 @@ def renderCtfTomoPsdImageService( - index is used when the PSD is stored in a stack file. - applyTransform/rot/shifts are optional alignment parameters. """ + if mapper is not None and psdPath: + try: + splitPath = str(psdPath).split("@") + imageIndex = int(index) + + if len(splitPath) > 1: + try: + imageIndex = int(float(splitPath[0])) + except Exception: + imageIndex = int(index) + + candidatePath = Path(splitPath[-1]).expanduser() + + if candidatePath.is_absolute(): + absPath = candidatePath.resolve() + + if absPath.exists() and absPath.is_file(): + preview = OutputsPreview( + currentProject=self.currentProject, + protocol=None, + output=None, + ) + + return preview.renderImageFromFilePath( + filePath=str(absPath), + size=size, + fmt=fmt, + index=imageIndex, + inline=inline, + quality=quality, + applyTransform=applyTransform and rot is not None and shifts is not None, + rot=rot, + shifts=shifts, + ) + + except Exception: + logger.exception( + "Failed to render CTFTomo PSD from PostgreSQL metadata. projectId=%s protocolId=%s outputName=%s psdPath=%s", + projectId, + protocolId, + outputName, + psdPath, + ) + protocol, output = self._resolveOutputForCtftomoSeries(protocolId, outputName) if protocol is None: raise HTTPException( From cbe3a9e651a20740255b03040b9adc35355d8653 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 18:24:08 +0200 Subject: [PATCH 051/278] fallback incomplete ctftomo postgresql views --- .../viewers/postgresql_ctftomo_reader.py | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/app/backend/viewers/postgresql_ctftomo_reader.py b/app/backend/viewers/postgresql_ctftomo_reader.py index f943e1d0..b9ff73ba 100644 --- a/app/backend/viewers/postgresql_ctftomo_reader.py +++ b/app/backend/viewers/postgresql_ctftomo_reader.py @@ -71,6 +71,9 @@ def getCtftomoSeriesViews(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: summary["tiltSeriesId"] = summary.get("tiltSeriesId") or str(tiltSeriesId) summary["nViews"] = len(frames) + if not self._hasCtftomoViewerContract(summary): + return None + return summary def _getStoredSet(self) -> Optional[Dict[str, Any]]: @@ -122,7 +125,7 @@ def _buildCtftomoSeriesSummary(self, item: Dict[str, Any], index: int) -> Dict[s childItems = self.setMapper.getStoredSetTableItems(int(childTable["id"])) summary["nViews"] = len(childItems) - dims = self._firstValueBySuffix(values, ["dim", "dims", "dimensions"]) + dims = self._firstValueBySuffix(values, ["dim", "dims", "dimensions", "getDim"]) if dims is not None: summary["dims"] = dims @@ -167,10 +170,12 @@ def _getTiltSeriesIdFromItem(self, item: Dict[str, Any], index: int) -> Any: def _buildCtftomoMeasurementFrame(self, item: Dict[str, Any], position: int) -> Dict[str, Any]: values = item.get("values") or {} + viewId = item.get("scipionItemId") or position frame: Dict[str, Any] = { - "index": item.get("scipionItemId") or position, - "viewIndex": item.get("scipionItemId") or position, + "viewId": viewId, + "index": position, + "viewIndex": position, } tiltAngle = self._firstValueBySuffix(values, ["tiltangle"]) @@ -299,4 +304,31 @@ def _toOptionalBool(self, value: Any) -> Optional[bool]: if text in ("0", "false", "no", "n", "off", "disabled"): return False - return None \ No newline at end of file + return None + + def _hasCtftomoViewerContract(self, payload: Dict[str, Any]) -> bool: + frames = payload.get("frames") or [] + if not frames: + return False + + if not payload.get("dims"): + return False + + requiredNumericKeys = ( + "tiltAngle", + "defocusU", + "defocusV", + "defocusAngle", + "resolution", + "order", + ) + + for frame in frames: + for key in requiredNumericKeys: + if self._toOptionalFloat(frame.get(key)) is None: + return False + + if not frame.get("psdFile"): + return False + + return True \ No newline at end of file From 66c9bf241c7e241d6a73851159ef677ee9bd0b10 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 18:44:26 +0200 Subject: [PATCH 052/278] sync generated ctftomo set to postgresql --- app/backend/api/routers/project_router.py | 1 + app/backend/api/services/project_service.py | 32 +++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index ce3006c4..e038d471 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2091,6 +2091,7 @@ def createNewSetOfCtftomoSeries( outputName=outputName, exclusions=payload.exclusions, restack=payload.restack, + mapper=mapper, ) from fastapi.responses import JSONResponse diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index b9031339..41e08195 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -7096,6 +7096,7 @@ def createNewSetOfCtftomoSeriesService( outputName: str, exclusions: Dict[str, Any], restack: bool, + mapper=None, ) -> Dict[str, Any]: """ Create a new SetOfCTFTomoSeries applying per-series and per-tilt exclusions. @@ -7157,20 +7158,47 @@ def createNewSetOfCtftomoSeriesService( "restack": bool(restack), "message": "No output was generated because it cannot be empty", } + postgresqlSync = None + try: protocol._defineOutputs(**{newOutputName: outputSet}) protocol._store() + + if mapper is not None: + try: + from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + setMapper = ScipionSetPostgresqlMapper(mapper.db) + postgresqlSync = setMapper.storeSet( + projectId=projectId, + protocolDbId=protocolId, + outputName=newOutputName, + scipionSet=outputSet, + ) + + except Exception: + logger.exception( + "Failed to persist created CTFTomo output to PostgreSQL. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + newOutputName, + ) + except Exception: - logger.exception("Error attaching Ctftomo filtered set '%s' to protocol", newOutputName,) + logger.exception("Error attaching Ctftomo filtered set '%s' to protocol", newOutputName) logger.info( - "The new Ctftomo set (%s) has been created successfully with %d series", newOutputName, createdCount,) + "The new Ctftomo set (%s) has been created successfully with %d series", + newOutputName, + createdCount, + ) return { "status": 0, "outputName": newOutputName, "createdSeries": createdCount, "restack": bool(restack), + "postgresqlSync": postgresqlSync, } def _buildTiltSeriesPreviewCacheKey( From b776d6f8101a70ba48dee1f42f10f5a87ac9a9d9 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 22:27:21 +0200 Subject: [PATCH 053/278] store linked tomograms for coords3d sets --- app/backend/api/routers/project_router.py | 1 + app/backend/api/services/project_service.py | 55 ++ app/backend/mapper/scipion_set_mapper.py | 118 ++- .../viewers/postgresql_coords3d_reader.py | 775 ++++++++++++++++++ 4 files changed, 948 insertions(+), 1 deletion(-) create mode 100644 app/backend/viewers/postgresql_coords3d_reader.py diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index e038d471..a9e1fcc4 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2216,6 +2216,7 @@ def listCoordinates3dTomograms( projectId=projectId, protocolId=protocolId, outputName=outputName, + mapper=mapper, ) from fastapi.responses import JSONResponse diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 41e08195..c12a91bb 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -7791,11 +7791,45 @@ def _cloneTiltImage(self, ti, included): # ---------------------------------------------------------------------- # Analyze Results: Coordinates3D # ---------------------------------------------------------------------- + def _getPostgresqlCoords3dReaderIfAvailable( + self, + mapper, + projectId: int, + protocolId: int, + outputName: str, + ): + if mapper is None: + return None + + try: + from app.backend.viewers.postgresql_coords3d_reader import PostgresqlCoords3dReader + + reader = PostgresqlCoords3dReader( + db=mapper.db, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if reader.hasOutput(): + return reader + + except Exception: + logger.exception( + "Failed to initialize PostgreSQL Coordinates3D reader. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + ) + + return None + def listCoordinates3dTomogramsService( self, projectId: int, protocolId: int, outputName: str, + mapper=None, ): """ Return a list of tomograms referenced by the SetOfCoordinates3D output. @@ -7806,6 +7840,27 @@ def listCoordinates3dTomogramsService( ... ] """ + pgReader = self._getPostgresqlCoords3dReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.listTomograms() + if payload is not None: + self.tomoList = {} + return payload + + logger.info( + "Skipping PostgreSQL Coordinates3D tomograms reader. projectId=%s protocolId=%s outputName=%s reason=%s", + projectId, + protocolId, + outputName, + getattr(pgReader, "lastSkipReason", None), + ) + try: protocol = self.currentProject.getProtocol(int(protocolId)) except Exception: diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index 4c884a04..e86d4f92 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -34,7 +34,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 2 +NESTED_LOGICAL_TABLES_VERSION = 3 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): @@ -1089,6 +1089,10 @@ def _getSetProperties(self, scipionSet: Any) -> Dict[str, Any]: if streamState is not None: properties["streamState"] = self._toJsonValue(streamState) + linkedTomograms = self._getLinkedTomogramsSummary(scipionSet) + if linkedTomograms: + properties["linkedTomograms"] = linkedTomograms + for attrName, attrValue in self._getAttributesToStore(scipionSet): if self._getAttributesToStore(attrValue): continue @@ -1096,6 +1100,118 @@ def _getSetProperties(self, scipionSet: Any) -> Dict[str, Any]: return {key: value for key, value in properties.items() if value is not None} + def _getLinkedTomogramsSummary(self, scipionSet: Any) -> List[Dict[str, Any]]: + tomograms = [] + + for index, tomogram in enumerate(self._iterLinkedTomograms(scipionSet)): + item = self._buildLinkedTomogramSummary(tomogram, index) + if item is not None: + tomograms.append(item) + + return tomograms + + def _iterLinkedTomograms(self, scipionSet: Any) -> Iterable[Any]: + for methodName in ("iterTomograms", "iterVolumes"): + iteratorGetter = getattr(scipionSet, methodName, None) + if not callable(iteratorGetter): + continue + + try: + return iteratorGetter() + except Exception: + continue + + getTomograms = getattr(scipionSet, "getTomograms", None) + if callable(getTomograms): + try: + tomograms = getTomograms() + iterItems = getattr(tomograms, "iterItems", None) + if callable(iterItems): + try: + return iterItems(iterate=False) + except TypeError: + return iterItems() + return iter(tomograms) + except Exception: + pass + + return iter(()) + + def _buildLinkedTomogramSummary(self, tomogram: Any, index: int) -> Optional[Dict[str, Any]]: + objectId = self._callOptionalGetter(tomogram, "getObjId") + tsId = self._callOptionalGetter(tomogram, "getTsId") + tomoId = self._callOptionalGetter(tomogram, "getTomoId") + + stableId = tsId or tomoId or objectId or index + if stableId is None: + return None + + name = None + for methodName in ("getObjLabel", "getNameId", "getFileName"): + value = self._callOptionalGetter(tomogram, methodName) + if value: + name = value + break + + if not name: + name = stableId + + dims = self._normalizeLinkedTomogramDims( + self._callOptionalGetter(tomogram, "getDim") + ) + + samplingRate = self._toOptionalFloat( + self._callOptionalGetter(tomogram, "getSamplingRate") + ) + + item: Dict[str, Any] = { + "id": str(stableId), + "tomoId": str(stableId), + "label": str(stableId), + "name": str(name), + } + + if objectId is not None: + item["objectId"] = str(objectId) + item["volumeId"] = str(objectId) + + if tsId is not None: + item["tsId"] = str(tsId) + item["tiltSeriesId"] = str(tsId) + + fileName = self._callOptionalGetter(tomogram, "getFileName") + if fileName: + item["fileName"] = str(fileName) + + if dims is not None: + item["dims"] = dims + + if samplingRate is not None: + item["voxelSize"] = [samplingRate, samplingRate, samplingRate] + + return item + + def _normalizeLinkedTomogramDims(self, dims: Any) -> Optional[List[int]]: + if dims is None: + return None + + try: + values = list(dims) + except Exception: + return None + + if len(values) < 3: + return None + + out = [] + for value in values[:3]: + intValue = self._toOptionalInt(value) + if intValue is None or intValue <= 0: + return None + out.append(intValue) + + return out + def _getItemEnabled(self, item: Any) -> bool: isEnabled = getattr(item, "isEnabled", None) if not callable(isEnabled): diff --git a/app/backend/viewers/postgresql_coords3d_reader.py b/app/backend/viewers/postgresql_coords3d_reader.py new file mode 100644 index 00000000..c7dfd3d6 --- /dev/null +++ b/app/backend/viewers/postgresql_coords3d_reader.py @@ -0,0 +1,775 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +import json +import os +import re +from typing import Any, Dict, List, Optional, Set + +from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + +class PostgresqlCoords3dReader: + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + self.setMapper = ScipionSetPostgresqlMapper(db) + self._storedSet = None + self.lastSkipReason = None + + def hasOutput(self) -> bool: + return self._getStoredSet() is not None + + def listTomograms(self) -> Optional[List[Dict[str, Any]]]: + self.lastSkipReason = None + + storedSet = self._getStoredSet() + if storedSet is None: + self.lastSkipReason = "stored_set_not_found" + return None + + countsByKey = self._countCoordinatesByTomogramKey(storedSet) + coordinateKeys = set(countsByKey.keys()) + + linkedTomograms = self._getLinkedTomogramsFromProperties(storedSet) + payload = self._buildTomogramPayloadFromLinkedTomograms( + linkedTomograms=linkedTomograms, + countsByKey=countsByKey, + ) + + if payload is not None: + return payload + + payload = self._buildTomogramPayloadFromProjectSets( + coordinateKeys=coordinateKeys, + countsByKey=countsByKey, + ) + + if payload is not None: + return payload + + if not linkedTomograms: + self.lastSkipReason = ( + "linked_tomograms_not_found " + "and_project_tomograms_not_resolved " + "coordinateKeys=%s" + ) % sorted(coordinateKeys) + else: + self.lastSkipReason = ( + "linked_tomograms_invalid " + "and_project_tomograms_not_resolved " + "coordinateKeys=%s" + ) % sorted(coordinateKeys) + + return None + + def _getStoredSet(self) -> Optional[Dict[str, Any]]: + if self._storedSet is None: + self._storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=self.protocolId, + outputName=self.outputName, + ) + return self._storedSet + + def _buildTomogramPayloadFromLinkedTomograms( + self, + linkedTomograms: List[Dict[str, Any]], + countsByKey: Dict[str, int], + ) -> Optional[List[Dict[str, Any]]]: + if not linkedTomograms: + return None + + result = [] + for item in linkedTomograms: + normalized = self._normalizeLinkedTomogramItem(item) + if normalized is None: + continue + + count = self._findTomogramCount(normalized, countsByKey) + if count is not None: + normalized["nCoords"] = count + normalized["count"] = count + + result.append(normalized) + + if not result: + return None + + if not self._hasTomogramViewerContract(result): + return None + + return result + + def _buildTomogramPayloadFromProjectSets( + self, + coordinateKeys: Set[str], + countsByKey: Dict[str, int], + ) -> Optional[List[Dict[str, Any]]]: + if not coordinateKeys: + return None + + rows = self._getProjectTomogramRows() + if not rows: + return None + + resultById: Dict[str, Dict[str, Any]] = {} + + for row in rows: + summary = self._buildTomogramSummaryFromStoredItem(row) + if summary is None: + continue + + matchKeys = self._getTomogramMatchKeys(summary) + if not matchKeys.intersection(coordinateKeys): + continue + + count = self._findTomogramCount(summary, countsByKey) + if count is not None: + summary["nCoords"] = count + summary["count"] = count + + resultKey = str(summary.get("id")) + existing = resultById.get(resultKey) + + if existing is None: + resultById[resultKey] = summary + continue + + resultById[resultKey] = self._mergeTomogramSummaries(existing, summary) + + result = list(resultById.values()) + if not result: + return None + + if not self._hasTomogramViewerContract(result): + return None + + return result + + def _getProjectTomogramRows(self) -> List[Dict[str, Any]]: + try: + return self.db.fetchAll( + """ + SELECT + s.id AS "setId", + s."projectId", + s."protocolDbId", + s."outputName", + s."setClassName", + s."itemClassName", + s.properties AS "setProperties", + i.id AS "itemRowId", + i."scipionItemId", + i.enabled, + i.label, + i.comment, + i.creation, + i."values", + i."createdAt", + i."updatedAt" + FROM scipion_sets s + JOIN scipion_set_items i + ON i."setId" = s.id + WHERE s."projectId" = %s + AND ( + LOWER(COALESCE(s."setClassName", '')) LIKE '%%tomogram%%' + OR LOWER(COALESCE(s."itemClassName", '')) LIKE '%%tomogram%%' + OR LOWER(COALESCE(s."setClassName", '')) LIKE '%%volume%%' + OR LOWER(COALESCE(s."itemClassName", '')) LIKE '%%volume%%' + ) + ORDER BY + CASE + WHEN LOWER(COALESCE(s."itemClassName", '')) LIKE '%%tomogram%%' THEN 0 + WHEN LOWER(COALESCE(s."setClassName", '')) LIKE '%%tomogram%%' THEN 1 + ELSE 2 + END, + s."protocolDbId" ASC, + s."outputName" ASC, + i."scipionItemId" ASC + """, + (self.projectId,), + ) + except Exception: + return [] + + def _buildTomogramSummaryFromStoredItem(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]: + values = row.get("values") or {} + objectId = row.get("scipionItemId") + objectIdText = self._toTextCandidate(objectId) + + tsId = self._firstValueBySuffix( + values, + ["tsid", "tiltseriesid", "tilt_series_id"], + ) + + tomoId = self._firstValueBySuffix( + values, + ["tomoid", "tomogramid", "tomo_id", "tomogram_id"], + ) + + nameId = self._firstValueBySuffix( + values, + ["nameid", "name_id"], + ) + + labelValue = self._firstValueBySuffix( + values, + ["objlabel", "label"], + ) + + stableId = ( + self._toTextCandidate(tsId) + or self._toTextCandidate(tomoId) + or self._toTextCandidate(labelValue) + or self._toTextCandidate(nameId) + or objectIdText + ) + + if not stableId: + return None + + dims = self._extractDims(values) + if dims is None: + return None + + name = ( + self._toTextCandidate(nameId) + or self._toTextCandidate(labelValue) + or self._extractFileBasename(values) + or stableId + ) + + summary: Dict[str, Any] = { + "id": stableId, + "tomoId": stableId, + "label": str(tsId or stableId), + "name": str(name), + "dims": dims, + } + + voxelSize = self._extractVoxelSize(values) + if voxelSize is not None: + summary["voxelSize"] = voxelSize + + fileName = self._extractTomogramFile(values) + if fileName: + summary["fileName"] = str(fileName) + + if objectIdText: + summary["objectId"] = objectIdText + summary["volumeId"] = objectIdText + + if tsId is not None: + summary["tsId"] = str(tsId) + summary["tiltSeriesId"] = str(tsId) + + if tomoId is not None: + summary["sourceTomoId"] = str(tomoId) + + summary["sourceOutputName"] = str(row.get("outputName") or "") + summary["sourceProtocolId"] = str(row.get("protocolDbId") or "") + + return summary + + def _mergeTomogramSummaries( + self, + current: Dict[str, Any], + candidate: Dict[str, Any], + ) -> Dict[str, Any]: + currentScore = self._getTomogramSummaryScore(current) + candidateScore = self._getTomogramSummaryScore(candidate) + + if candidateScore > currentScore: + base = dict(candidate) + for key in ("nCoords", "count"): + if key in current and key not in base: + base[key] = current[key] + return base + + base = dict(current) + for key, value in candidate.items(): + if value is not None and key not in base: + base[key] = value + return base + + def _getTomogramSummaryScore(self, item: Dict[str, Any]) -> int: + score = 0 + + if item.get("dims"): + score += 10 + + if item.get("voxelSize"): + score += 5 + + if item.get("fileName"): + score += 3 + + classText = " ".join( + [ + str(item.get("sourceOutputName") or ""), + str(item.get("sourceProtocolId") or ""), + ] + ).lower() + + if "tomogram" in classText: + score += 2 + + return score + + def _hasTomogramViewerContract(self, items: List[Dict[str, Any]]) -> bool: + for item in items: + dims = item.get("dims") + if not isinstance(dims, list) or len(dims) < 3: + return False + + for value in dims[:3]: + intValue = self._toOptionalInt(value) + if intValue is None or intValue <= 0: + return False + + return True + + def _getLinkedTomogramsFromProperties(self, storedSet: Dict[str, Any]) -> List[Dict[str, Any]]: + properties = self._normalizeJsonObject(storedSet.get("properties")) + + linkedTomograms = properties.get("linkedTomograms") + if isinstance(linkedTomograms, list): + return [ + item + for item in linkedTomograms + if isinstance(item, dict) + ] + + setProperties = storedSet.get("setProperties") or [] + for item in setProperties: + if str(item.get("key")) != "linkedTomograms": + continue + + parsed = self._parseJsonValue(item.get("value")) + if isinstance(parsed, list): + return [ + entry + for entry in parsed + if isinstance(entry, dict) + ] + + return [] + + def _normalizeLinkedTomogramItem(self, item: Dict[str, Any]) -> Optional[Dict[str, Any]]: + tomoId = item.get("tomoId") or item.get("id") or item.get("tsId") or item.get("label") + if tomoId is None: + return None + + dims = self._normalizeDims(item.get("dims")) + if dims is None: + return None + + normalized: Dict[str, Any] = { + "id": str(tomoId), + "tomoId": str(tomoId), + "label": str(item.get("label") or tomoId), + "name": str(item.get("name") or item.get("label") or tomoId), + "dims": dims, + } + + voxelSize = self._normalizeVoxelSize(item.get("voxelSize")) + if voxelSize is not None: + normalized["voxelSize"] = voxelSize + + for key in ("objectId", "volumeId", "tsId", "tiltSeriesId", "fileName"): + value = item.get(key) + if value is not None: + normalized[key] = str(value) + + return normalized + + def _countCoordinatesByTomogramKey(self, storedSet: Dict[str, Any]) -> Dict[str, int]: + counts: Dict[str, int] = {} + + for item in storedSet.get("items") or []: + values = item.get("values") or {} + keys = self._extractCoordinateTomogramKeys(values) + + for key in keys: + counts[key] = counts.get(key, 0) + 1 + + return counts + + def _extractCoordinateTomogramKeys(self, values: Dict[str, Any]) -> List[str]: + candidates = [] + + for suffixes in ( + ["volid", "volumeid", "volumeobjid", "volume_obj_id"], + ["tomoid", "tomogramid", "tomo_id", "tomogram_id"], + ["tsid", "tiltseriesid", "tilt_series_id"], + ["volname", "volumename", "tomoname", "tomogramname"], + ): + value = self._firstValueBySuffix(values, suffixes) + text = self._toTextCandidate(value) + if text: + candidates.append(text) + + for key, value in values.items(): + normalizedKey = self._normalizeKey(key) + if normalizedKey not in { + "volid", + "volumeid", + "volumeobjid", + "tomoid", + "tomogramid", + "tsid", + "tiltseriesid", + "volname", + "volumename", + "tomoname", + "tomogramname", + }: + continue + + text = self._toTextCandidate(value) + if text: + candidates.append(text) + + return self._uniqueStrings(candidates) + + def _findTomogramCount(self, item: Dict[str, Any], countsByKey: Dict[str, int]) -> Optional[int]: + candidates = [ + item.get("id"), + item.get("tomoId"), + item.get("label"), + item.get("name"), + item.get("tsId"), + item.get("tiltSeriesId"), + item.get("objectId"), + item.get("volumeId"), + item.get("sourceTomoId"), + ] + + for candidate in candidates: + if candidate is None: + continue + + key = str(candidate) + if key in countsByKey: + return countsByKey[key] + + return None + + def _getTomogramMatchKeys(self, item: Dict[str, Any]) -> Set[str]: + keys = set() + + for key in ( + "id", + "tomoId", + "label", + "name", + "tsId", + "tiltSeriesId", + "objectId", + "volumeId", + "sourceTomoId", + ): + value = item.get(key) + if value is not None and str(value).strip(): + keys.add(str(value).strip()) + + return keys + + def _extractTomogramId(self, values: Dict[str, Any]) -> Optional[Any]: + value = self._firstValueBySuffix( + values, + [ + "tomoid", + "tomogramid", + "tomoName", + "tomogramName", + "volumeid", + "volid", + "volname", + "tsid", + "tiltseriesid", + ], + ) + return value + + def _extractTomogramLabel(self, values: Dict[str, Any], fallback: str) -> str: + label = self._firstValueBySuffix( + values, + [ + "nameid", + "tomoname", + "tomogramname", + "volumename", + "volname", + "objlabel", + "label", + "name", + ], + ) + + if label: + return str(label) + + fileName = self._extractTomogramFile(values) + if fileName: + return str(fileName).split("/")[-1] + + return str(fallback) + + def _extractTomogramFile(self, values: Dict[str, Any]) -> Optional[Any]: + return self._firstValueBySuffix( + values, + [ + "filename", + "fileName", + "filepath", + "filePath", + "volumefile", + "tomogramfile", + "location", + ], + ) + + def _extractFileBasename(self, values: Dict[str, Any]) -> Optional[str]: + fileName = self._extractTomogramFile(values) + if not fileName: + return None + + try: + return os.path.basename(str(fileName)) + except Exception: + return str(fileName) + + def _extractDims(self, values: Dict[str, Any]) -> Optional[List[int]]: + raw = self._firstValueBySuffix( + values, + [ + "dim", + "dims", + "dimensions", + "volumedim", + "tomogramdim", + "getdim", + ], + ) + + return self._normalizeDims(raw) + + def _extractVoxelSize(self, values: Dict[str, Any]) -> Optional[List[float]]: + raw = self._firstValueBySuffix( + values, + [ + "voxelsize", + "voxel_size", + "samplingrate", + "samplingRate", + "pixelSize", + "pixel_size", + ], + ) + + return self._normalizeVoxelSize(raw) + + def _normalizeDims(self, value: Any) -> Optional[List[int]]: + parsed = self._parseNumberList(value) + if parsed is None or len(parsed) < 3: + return None + + dims = [] + for item in parsed[:3]: + intValue = self._toOptionalInt(item) + if intValue is None or intValue <= 0: + return None + dims.append(intValue) + + return dims + + def _normalizeVoxelSize(self, value: Any) -> Optional[List[float]]: + parsed = self._parseNumberList(value) + if parsed is None: + floatValue = self._toOptionalFloat(value) + if floatValue is None: + return None + return [floatValue, floatValue, floatValue] + + if len(parsed) == 1: + floatValue = self._toOptionalFloat(parsed[0]) + if floatValue is None: + return None + return [floatValue, floatValue, floatValue] + + if len(parsed) >= 3: + voxelSize = [] + for item in parsed[:3]: + floatValue = self._toOptionalFloat(item) + if floatValue is None: + return None + voxelSize.append(floatValue) + return voxelSize + + return None + + def _parseNumberList(self, value: Any) -> Optional[List[Any]]: + if value is None or value == "": + return None + + parsed = self._parseJsonValue(value) + if isinstance(parsed, (list, tuple)): + return list(parsed) + + if isinstance(value, (list, tuple)): + return list(value) + + text = str(value).strip() + if not text: + return None + + numbers = re.findall(r"[-+]?\d*\.?\d+(?:[eE][-+]?\d+)?", text) + if not numbers: + return None + + return numbers + + def _firstValue(self, values: Dict[str, Any], keys: List[str]) -> Any: + for key in keys: + if key in values: + return values.get(key) + return None + + def _firstValueBySuffix(self, values: Dict[str, Any], suffixes: List[str]) -> Any: + normalizedSuffixes = [ + self._normalizeKey(suffix) + for suffix in suffixes + ] + + for key, value in values.items(): + if value is None: + continue + + normalizedKey = self._normalizeKey(key) + for suffix in normalizedSuffixes: + if normalizedKey.endswith(suffix): + return value + + return None + + def _normalizeKey(self, value: Any) -> str: + return str(value).replace("_", "").replace(".", "").replace("-", "").lower() + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except Exception: + try: + return int(float(value)) + except Exception: + return None + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + try: + return float(value) + except Exception: + return None + + def _toTextCandidate(self, value: Any) -> Optional[str]: + if value is None: + return None + + if isinstance(value, dict): + for key in ( + "id", + "objId", + "objectId", + "volumeId", + "volId", + "tomoId", + "tomogramId", + "tsId", + "name", + "label", + ): + if key in value: + text = self._toTextCandidate(value.get(key)) + if text: + return text + return None + + if isinstance(value, (list, tuple)): + if len(value) == 1: + return self._toTextCandidate(value[0]) + return None + + text = str(value).strip() + return text or None + + def _uniqueStrings(self, values: List[str]) -> List[str]: + seen = set() + out = [] + + for value in values: + text = str(value).strip() + if not text or text in seen: + continue + + seen.add(text) + out.append(text) + + return out + + def _parseJsonValue(self, value: Any) -> Any: + if isinstance(value, (dict, list, tuple)): + return value + + if not isinstance(value, str): + return value + + text = value.strip() + if not text: + return value + + if not ( + text.startswith("{") + or text.startswith("[") + or text.startswith('"') + ): + return value + + try: + return json.loads(text) + except Exception: + return value + + def _normalizeJsonObject(self, value: Any) -> Dict[str, Any]: + parsed = self._parseJsonValue(value) + if isinstance(parsed, dict): + return parsed + return {} \ No newline at end of file From 9f37380e6e4e2ee89611e5f8c6857b8d8922dd5f Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 23:17:15 +0200 Subject: [PATCH 054/278] store coords3d bottom-left coordinates in postgresql --- app/backend/api/routers/project_router.py | 1 + app/backend/api/services/project_service.py | 22 ++ app/backend/mapper/scipion_set_mapper.py | 107 ++++++++- .../viewers/postgresql_coords3d_reader.py | 223 +++++++++++++++++- 4 files changed, 346 insertions(+), 7 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index a9e1fcc4..6da88e6f 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2256,6 +2256,7 @@ def getCoordinates3dPoints( protocolId=protocolId, outputName=outputName, tomogramId=tomogramId, + mapper=mapper, ) from fastapi.responses import JSONResponse diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index c12a91bb..a865398e 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -7958,6 +7958,7 @@ def getCoordinates3dPointsService( protocolId: int, outputName: str, tomogramId: Union[int, str], + mapper=None, ): """ Return a flat list of 3D points for one tomogram. @@ -7981,6 +7982,27 @@ def getCoordinates3dPointsService( ... ] """ + pgReader = self._getPostgresqlCoords3dReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.getPoints(tomogramId) + if payload is not None: + return payload + + logger.info( + "Skipping PostgreSQL Coordinates3D points reader. projectId=%s protocolId=%s outputName=%s tomogramId=%s reason=%s", + projectId, + protocolId, + outputName, + tomogramId, + getattr(pgReader, "lastSkipReason", None), + ) + try: protocol = self.currentProject.getProtocol(int(protocolId)) except Exception: diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index e86d4f92..1c626ec7 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -32,9 +32,14 @@ from app.backend.mapper.scipion_object_mapper import ScipionObjectPostgresqlMapper +try: + from tomo.constants import BOTTOM_LEFT_CORNER +except Exception: + BOTTOM_LEFT_CORNER = None + SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 3 +NESTED_LOGICAL_TABLES_VERSION = 6 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): @@ -998,16 +1003,67 @@ def _getItemSchema(self, item: Any) -> Dict[str, Any]: return self._getObjDict(item, includeClass=True) def _getItemValues(self, item: Any) -> Dict[str, Any]: - values = self._getObjDict(item, includeClass=False) - if not values: - return {} + rawValues = self._getObjDict(item, includeClass=False) - return { + values = { str(label): self._toJsonValue(value) - for label, value in values.items() + for label, value in (rawValues or {}).items() if str(label) != SELF_LABEL } + self._addCoordinate3dBottomLeftCoordinates(item, values) + + return values + + def _addCoordinate3dBottomLeftCoordinates(self, item: Any, values: Dict[str, Any]) -> None: + coords = self._getCoordinate3dBottomLeftCoordinates(item) + if coords is None: + return + + x, y, z = coords + + values["bottomLeftX"] = x + values["bottomLeftY"] = y + values["bottomLeftZ"] = z + values["coordinateConvention"] = "BOTTOM_LEFT_CORNER" + + if "_x" in values and "rawX" not in values: + values["rawX"] = values.get("_x") + if "_y" in values and "rawY" not in values: + values["rawY"] = values.get("_y") + if "_z" in values and "rawZ" not in values: + values["rawZ"] = values.get("_z") + + def _getCoordinate3dBottomLeftCoordinates(self, item: Any) -> Optional[Tuple[float, float, float]]: + if BOTTOM_LEFT_CORNER is None: + return None + + getX = getattr(item, "getX", None) + getY = getattr(item, "getY", None) + getZ = getattr(item, "getZ", None) + + if not callable(getX) or not callable(getY) or not callable(getZ): + return None + + try: + return ( + float(getX(BOTTOM_LEFT_CORNER)), + float(getY(BOTTOM_LEFT_CORNER)), + float(getZ(BOTTOM_LEFT_CORNER)), + ) + except Exception: + return None + + def _callCoordinateGetter(self, item: Any, getterName: str) -> Optional[float]: + getter = getattr(item, getterName, None) + if not callable(getter): + return None + + try: + return float(getter(BOTTOM_LEFT_CORNER)) + except Exception: + return None + def _getObjDict(self, scipionObj: Any, includeClass: bool) -> Dict[str, Any]: getter = getattr(scipionObj, "getObjDict", None) if not callable(getter): @@ -1280,3 +1336,42 @@ def _chainFirst(self, firstItem: Any, remainingItems: Iterator[Any]) -> Iterator yield firstItem for item in remainingItems: yield item + + def _addCoordinate3dBottomLeftCoordinates(self, item: Any, values: Dict[str, Any]) -> None: + coords = self._getCoordinate3dBottomLeftCoordinates(item) + if coords is None: + return + + x, y, z = coords + + if "_x" in values and "rawX" not in values: + values["rawX"] = values.get("_x") + if "_y" in values and "rawY" not in values: + values["rawY"] = values.get("_y") + if "_z" in values and "rawZ" not in values: + values["rawZ"] = values.get("_z") + + values["bottomLeftX"] = x + values["bottomLeftY"] = y + values["bottomLeftZ"] = z + values["coordinateConvention"] = "BOTTOM_LEFT_CORNER" + + def _getCoordinate3dBottomLeftCoordinates(self, item: Any) -> Optional[Tuple[float, float, float]]: + if BOTTOM_LEFT_CORNER is None: + return None + + getX = getattr(item, "getX", None) + getY = getattr(item, "getY", None) + getZ = getattr(item, "getZ", None) + + if not callable(getX) or not callable(getY) or not callable(getZ): + return None + + try: + x = float(getX(BOTTOM_LEFT_CORNER)) + y = float(getY(BOTTOM_LEFT_CORNER)) + z = float(getZ(BOTTOM_LEFT_CORNER)) + except Exception: + return None + + return x, y, z \ No newline at end of file diff --git a/app/backend/viewers/postgresql_coords3d_reader.py b/app/backend/viewers/postgresql_coords3d_reader.py index c7dfd3d6..ff11a599 100644 --- a/app/backend/viewers/postgresql_coords3d_reader.py +++ b/app/backend/viewers/postgresql_coords3d_reader.py @@ -87,6 +87,48 @@ def listTomograms(self) -> Optional[List[Dict[str, Any]]]: return None + def getPoints(self, tomogramId: Any) -> Optional[List[Dict[str, Any]]]: + self.lastSkipReason = None + + storedSet = self._getStoredSet() + if storedSet is None: + self.lastSkipReason = "stored_set_not_found" + return None + + targetKeys = self._resolveTomogramTargetKeys(storedSet, tomogramId) + if not targetKeys: + self.lastSkipReason = "tomogram_target_keys_not_resolved tomogramId=%s" % str(tomogramId) + return None + + boxSize = self._extractBoxSize(storedSet) + points: List[Dict[str, Any]] = [] + + for item in storedSet.get("items") or []: + values = item.get("values") or {} + coordinateKeys = set(self._extractCoordinateTomogramKeys(values)) + + if not coordinateKeys.intersection(targetKeys): + continue + + point = self._buildCoordinatePoint( + item=item, + values=values, + tomogramId=tomogramId, + boxSize=boxSize, + ) + + if point is not None: + points.append(point) + + if not points: + self.lastSkipReason = ( + "coordinates_not_found_for_tomogram " + "tomogramId=%s targetKeys=%s" + ) % (str(tomogramId), sorted(targetKeys)) + return None + + return points + def _getStoredSet(self) -> Optional[Dict[str, Any]]: if self._storedSet is None: self._storedSet = self.setMapper.getStoredSet( @@ -772,4 +814,183 @@ def _normalizeJsonObject(self, value: Any) -> Dict[str, Any]: parsed = self._parseJsonValue(value) if isinstance(parsed, dict): return parsed - return {} \ No newline at end of file + return {} + + def _resolveTomogramTargetKeys( + self, + storedSet: Dict[str, Any], + tomogramId: Any, + ) -> Set[str]: + requested = self._toTextCandidate(tomogramId) + if not requested: + return set() + + targetKeys = {requested} + + linkedTomograms = self._getLinkedTomogramsFromProperties(storedSet) + for item in linkedTomograms: + normalized = self._normalizeLinkedTomogramItem(item) + if normalized is None: + continue + + matchKeys = self._getTomogramMatchKeys(normalized) + if requested in matchKeys: + targetKeys.update(matchKeys) + + countsByKey = self._countCoordinatesByTomogramKey(storedSet) + coordinateKeys = set(countsByKey.keys()) + + payload = self._buildTomogramPayloadFromProjectSets( + coordinateKeys=coordinateKeys, + countsByKey=countsByKey, + ) + + for item in payload or []: + matchKeys = self._getTomogramMatchKeys(item) + if requested in matchKeys: + targetKeys.update(matchKeys) + + return { + str(value) + for value in targetKeys + if value is not None and str(value).strip() + } + + def _buildCoordinatePoint( + self, + item: Dict[str, Any], + values: Dict[str, Any], + tomogramId: Any, + boxSize: Optional[float], + ) -> Optional[Dict[str, Any]]: + x = self._getCoordinateValue(values, "x") + y = self._getCoordinateValue(values, "y") + z = self._getCoordinateValue(values, "z") + + if x is None or y is None or z is None: + return None + + point: Dict[str, Any] = { + "x": float(x), + "y": float(y), + "z": float(z), + "tomoId": tomogramId, + } + + objId = item.get("scipionItemId") + if objId is not None: + point["id"] = objId + + classId = self._extractPointClassId(values) + if classId is not None: + point["classId"] = classId + + score = self._extractPointScore(values) + if score is not None: + point["score"] = score + + label = item.get("label") or self._firstValueBySuffix(values, ["objlabel", "label"]) + if label not in (None, ""): + point["label"] = str(label) + + matrix = self._extractPointMatrix(values) + if matrix is not None: + point["matrix"] = matrix + else: + point["matrix"] = [] + + if boxSize is not None: + point["radius"] = float(boxSize) + + return point + + def _getCoordinateValue(self, values: Dict[str, Any], axis: str) -> Optional[float]: + normalizedAxis = self._normalizeKey(axis) + + preferredKeys = { + "x": "bottomleftx", + "y": "bottomlefty", + "z": "bottomleftz", + } + + preferredKey = preferredKeys.get(normalizedAxis) + if preferredKey is None: + return None + + for key, value in values.items(): + if self._normalizeKey(key) == preferredKey: + return self._toOptionalFloat(value) + + return None + + def _extractPointClassId(self, values: Dict[str, Any]) -> Optional[Any]: + return self._firstValueBySuffix( + values, + [ + "classid", + "class", + "groupid", + "group", + ], + ) + + def _extractPointScore(self, values: Dict[str, Any]) -> Optional[float]: + raw = self._firstValueBySuffix( + values, + [ + "score", + "weight", + "prob", + "probability", + "confidence", + ], + ) + + return self._toOptionalFloat(raw) + + def _extractPointMatrix(self, values: Dict[str, Any]) -> Optional[Any]: + raw = self._firstValueBySuffix( + values, + [ + "matrix", + "transform", + "transformmatrix", + "transformationmatrix", + ], + ) + + if raw is None: + return None + + parsed = self._parseJsonValue(raw) + if isinstance(parsed, list): + return parsed + + return None + + def _extractBoxSize(self, storedSet: Dict[str, Any]) -> Optional[float]: + properties = self._normalizeJsonObject(storedSet.get("properties")) + + raw = self._firstValueBySuffix( + properties, + [ + "boxsize", + "box_size", + "radius", + ], + ) + + value = self._toOptionalFloat(raw) + if value is not None: + return value + + for item in storedSet.get("setProperties") or []: + key = item.get("key") + if self._normalizeKey(key) not in {"boxsize", "radius"}: + continue + + value = self._toOptionalFloat(item.get("value")) + if value is not None: + return value + + return None From fb5fc0b770f42a6798688fddae50869801008f3e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 23:31:53 +0200 Subject: [PATCH 055/278] attach coords3d tomograms before bottom-left conversion --- app/backend/mapper/scipion_set_mapper.py | 237 ++++++++++++++++++----- 1 file changed, 185 insertions(+), 52 deletions(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index 1c626ec7..c263bc23 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -27,10 +27,12 @@ import os from datetime import datetime, timezone from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple +import logging import psycopg2.extras from app.backend.mapper.scipion_object_mapper import ScipionObjectPostgresqlMapper +logger = logging.getLogger(__name__) try: from tomo.constants import BOTTOM_LEFT_CORNER @@ -39,7 +41,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 6 +NESTED_LOGICAL_TABLES_VERSION = 7 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): @@ -171,6 +173,7 @@ def storeSet( firstItem=firstItem, remainingItems=itemIterator, batchSize=batchSize, + scipionSet=scipionSet, ) finalProperties = dict(initialProperties) @@ -426,6 +429,7 @@ def _upsertSetItems( firstItem: Any, remainingItems: Iterator[Any], batchSize: int, + scipionSet: Optional[Any] = None, ) -> Tuple[int, Optional[int]]: rows: List[Tuple[Any, ...]] = [] tableRows: List[Tuple[Any, ...]] = [] @@ -438,7 +442,7 @@ def _upsertSetItems( raise ValueError("Cannot store a Scipion set item without getObjId()/getId()") maxItemId = itemId if maxItemId is None else max(maxItemId, itemId) - itemValues = self._getItemValues(item) + itemValues = self._getItemValues(item, scipionSet=scipionSet) rows.append( ( @@ -1002,7 +1006,7 @@ def _iterSetItems(self, scipionSet: Any) -> Iterable[Any]: def _getItemSchema(self, item: Any) -> Dict[str, Any]: return self._getObjDict(item, includeClass=True) - def _getItemValues(self, item: Any) -> Dict[str, Any]: + def _getItemValues(self, item: Any, scipionSet: Optional[Any] = None) -> Dict[str, Any]: rawValues = self._getObjDict(item, includeClass=False) values = { @@ -1011,22 +1015,30 @@ def _getItemValues(self, item: Any) -> Dict[str, Any]: if str(label) != SELF_LABEL } - self._addCoordinate3dBottomLeftCoordinates(item, values) + self._addCoordinate3dBottomLeftCoordinates( + item=item, + values=values, + scipionSet=scipionSet, + ) return values - def _addCoordinate3dBottomLeftCoordinates(self, item: Any, values: Dict[str, Any]) -> None: - coords = self._getCoordinate3dBottomLeftCoordinates(item) + def _addCoordinate3dBottomLeftCoordinates( + self, + item: Any, + values: Dict[str, Any], + scipionSet: Optional[Any] = None, + ) -> None: + coords = self._getCoordinate3dBottomLeftCoordinates( + item=item, + values=values, + scipionSet=scipionSet, + ) if coords is None: return x, y, z = coords - values["bottomLeftX"] = x - values["bottomLeftY"] = y - values["bottomLeftZ"] = z - values["coordinateConvention"] = "BOTTOM_LEFT_CORNER" - if "_x" in values and "rawX" not in values: values["rawX"] = values.get("_x") if "_y" in values and "rawY" not in values: @@ -1034,10 +1046,29 @@ def _addCoordinate3dBottomLeftCoordinates(self, item: Any, values: Dict[str, Any if "_z" in values and "rawZ" not in values: values["rawZ"] = values.get("_z") - def _getCoordinate3dBottomLeftCoordinates(self, item: Any) -> Optional[Tuple[float, float, float]]: + values["bottomLeftX"] = x + values["bottomLeftY"] = y + values["bottomLeftZ"] = z + values["coordinateConvention"] = "BOTTOM_LEFT_CORNER" + + def _getCoordinate3dBottomLeftCoordinates( + self, + item: Any, + values: Optional[Dict[str, Any]] = None, + scipionSet: Optional[Any] = None, + ) -> Optional[Tuple[float, float, float]]: if BOTTOM_LEFT_CORNER is None: return None + self._attachCoordinate3dTomogram( + item=item, + values=values or {}, + scipionSet=scipionSet, + ) + + return self._readCoordinate3dBottomLeftCoordinates(item) + + def _readCoordinate3dBottomLeftCoordinates(self, item: Any) -> Optional[Tuple[float, float, float]]: getX = getattr(item, "getX", None) getY = getattr(item, "getY", None) getZ = getattr(item, "getZ", None) @@ -1054,6 +1085,147 @@ def _getCoordinate3dBottomLeftCoordinates(self, item: Any) -> Optional[Tuple[flo except Exception: return None + def _attachCoordinate3dTomogram( + self, + item: Any, + values: Dict[str, Any], + scipionSet: Optional[Any], + ) -> bool: + if scipionSet is None: + return False + + tomogram = self._resolveCoordinate3dTomogram( + item=item, + values=values, + scipionSet=scipionSet, + ) + if tomogram is None: + return False + + setVolume = getattr(item, "setVolume", None) + if callable(setVolume): + try: + setVolume(tomogram) + return True + except Exception: + pass + + return False + + def _resolveCoordinate3dTomogram( + self, + item: Any, + values: Dict[str, Any], + scipionSet: Any, + ) -> Optional[Any]: + candidateKeys = self._getCoordinate3dTomogramCandidateKeys(item, values) + if not candidateKeys: + return None + + getTomogram = getattr(scipionSet, "_getTomogram", None) + if callable(getTomogram): + for key in candidateKeys: + for candidate in self._expandTomogramLookupKey(key): + try: + tomogram = getTomogram(candidate) + except Exception: + tomogram = None + + if tomogram is not None: + return tomogram + + for tomogram in self._iterLinkedTomograms(scipionSet): + tomogramKeys = self._getTomogramObjectMatchKeys(tomogram) + if tomogramKeys.intersection(candidateKeys): + return tomogram + + return None + + def _getCoordinate3dTomogramCandidateKeys( + self, + item: Any, + values: Dict[str, Any], + ) -> set: + candidates = [] + + for keyName in ( + "_tomoId", + "_volId", + "_volumeId", + "tomoId", + "tomogramId", + "volId", + "volumeId", + "tsId", + "tiltSeriesId", + ): + value = self._getValueByNormalizedKey(values, keyName) + text = self._toMatchText(value) + if text: + candidates.append(text) + + for getterName in ("getTomoId", "getVolId", "getVolumeId", "getTsId"): + value = self._callOptionalGetter(item, getterName) + text = self._toMatchText(value) + if text: + candidates.append(text) + + return { + str(value) + for value in candidates + if value is not None and str(value).strip() + } + + def _getTomogramObjectMatchKeys(self, tomogram: Any) -> set: + candidates = [] + + for getterName in ("getObjId", "getTsId", "getTomoId", "getNameId", "getObjLabel"): + value = self._callOptionalGetter(tomogram, getterName) + text = self._toMatchText(value) + if text: + candidates.append(text) + + return { + str(value) + for value in candidates + if value is not None and str(value).strip() + } + + def _expandTomogramLookupKey(self, key: Any) -> List[Any]: + values = [key] + + intValue = self._toOptionalInt(key) + if intValue is not None: + values.append(intValue) + + return values + + def _getValueByNormalizedKey(self, values: Dict[str, Any], keyName: str) -> Any: + targetKey = self._normalizeMatchKey(keyName) + + for key, value in values.items(): + if self._normalizeMatchKey(key) == targetKey: + return value + + return None + + def _normalizeMatchKey(self, value: Any) -> str: + return str(value).replace("_", "").replace(".", "").replace("-", "").lower() + + def _toMatchText(self, value: Any) -> Optional[str]: + if value is None: + return None + + getter = getattr(value, "get", None) + if callable(getter): + try: + value = getter() + except Exception: + return None + + text = str(value).strip() + return text or None + def _callCoordinateGetter(self, item: Any, getterName: str) -> Optional[float]: getter = getattr(item, getterName, None) if not callable(getter): @@ -1335,43 +1507,4 @@ def _nextOrNone(self, iterator: Iterator[Any]) -> Any: def _chainFirst(self, firstItem: Any, remainingItems: Iterator[Any]) -> Iterator[Any]: yield firstItem for item in remainingItems: - yield item - - def _addCoordinate3dBottomLeftCoordinates(self, item: Any, values: Dict[str, Any]) -> None: - coords = self._getCoordinate3dBottomLeftCoordinates(item) - if coords is None: - return - - x, y, z = coords - - if "_x" in values and "rawX" not in values: - values["rawX"] = values.get("_x") - if "_y" in values and "rawY" not in values: - values["rawY"] = values.get("_y") - if "_z" in values and "rawZ" not in values: - values["rawZ"] = values.get("_z") - - values["bottomLeftX"] = x - values["bottomLeftY"] = y - values["bottomLeftZ"] = z - values["coordinateConvention"] = "BOTTOM_LEFT_CORNER" - - def _getCoordinate3dBottomLeftCoordinates(self, item: Any) -> Optional[Tuple[float, float, float]]: - if BOTTOM_LEFT_CORNER is None: - return None - - getX = getattr(item, "getX", None) - getY = getattr(item, "getY", None) - getZ = getattr(item, "getZ", None) - - if not callable(getX) or not callable(getY) or not callable(getZ): - return None - - try: - x = float(getX(BOTTOM_LEFT_CORNER)) - y = float(getY(BOTTOM_LEFT_CORNER)) - z = float(getZ(BOTTOM_LEFT_CORNER)) - except Exception: - return None - - return x, y, z \ No newline at end of file + yield item \ No newline at end of file From 7f9bfc4f3cc6856d009ba2c4b1d02ede8f947092 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 23:45:21 +0200 Subject: [PATCH 056/278] serve coords3d tomogram slices from postgresql --- app/backend/api/routers/project_router.py | 1 + app/backend/api/services/project_service.py | 245 ++++++++++++++++++ .../viewers/postgresql_coords3d_reader.py | 65 +++++ 3 files changed, 311 insertions(+) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 6da88e6f..73ee8aac 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2348,6 +2348,7 @@ def renderCoords3dTomogramSlice( thumb=thumb, fast=fast, quality=quality, + mapper=mapper ) resp.headers["X-Debug-Auth"] = "ok" diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index a865398e..0fa5b114 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -8234,6 +8234,7 @@ def renderCoords3dTomogramSliceService( thumb: Optional[int] = 128, fast: bool = True, quality: int = 75, + mapper=None, ) -> Response: """ Render a 2D slice from a tomogram referenced by a SetOfCoordinates3D. @@ -8249,6 +8250,42 @@ def renderCoords3dTomogramSliceService( """ from PIL import Image as PILImage + pgReader = self._getPostgresqlCoords3dReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + tomogramInfo = pgReader.getTomogramFile(tomogramId) + if tomogramInfo is not None: + volumePath = tomogramInfo.get("fileName") + if volumePath and os.path.exists(volumePath): + return self._renderCoords3dTomogramSliceFromPath( + volumePath=volumePath, + tomogramId=tomogramId, + sliceIndex=sliceIndex, + axis=axis, + colormap=colormap, + normalize=normalize, + scale=scale, + inline=inline, + fmt=fmt, + thumb=thumb, + fast=fast, + quality=quality, + ) + + logger.info( + "Skipping PostgreSQL Coordinates3D tomogram slice reader. projectId=%s protocolId=%s outputName=%s tomogramId=%s reason=%s", + projectId, + protocolId, + outputName, + tomogramId, + getattr(pgReader, "lastSkipReason", None), + ) + try: protocol = self.currentProject.getProtocol(int(protocolId)) except Exception: @@ -8478,6 +8515,214 @@ def renderCoords3dTomogramSliceService( return Response(content=buf.getvalue(), media_type=mediaType, headers=headers) + def _renderCoords3dTomogramSliceFromPath( + self, + volumePath: str, + tomogramId: Union[int, str], + sliceIndex: int, + axis: str = "z", + colormap: Optional[str] = None, + normalize: Optional[str] = "minmax", + scale: float = 1.0, + inline: bool = True, + fmt: str = "webp", + thumb: Optional[int] = 128, + fast: bool = True, + quality: int = 75, + ) -> Response: + + from PIL import Image as PILImage + + axis = (axis or "z").lower() + if axis not in ("x", "y", "z"): + axis = "z" + + fmtLower = (fmt or "png").lower() + if fmtLower in ("jpg", "jpeg"): + pilFormat = "JPEG" + mediaType = "image/jpeg" + saveKw = {"quality": int(quality or 75)} + elif fmtLower == "webp": + pilFormat = "WEBP" + mediaType = "image/webp" + saveKw = {"quality": int(quality or 75)} + else: + pilFormat = "PNG" + mediaType = "image/png" + saveKw = {} + + usedColormap = colormap + gray: Optional[np.ndarray] = None + depth = 1 + + try: + requestedIndex = int(sliceIndex or 0) + except Exception: + requestedIndex = 0 + requestedIndex = max(0, requestedIndex) + + sliceUsed = requestedIndex + if axis == "z" and fast: + try: + reader = ImageReadersRegistry.open(volumePath) + + try: + images = reader.getImages() + if hasattr(images, "ndim") and images.ndim == 3: + zdim, ydim, xdim = int(images.shape[0]), int(images.shape[1]), int(images.shape[2]) + elif hasattr(images, "ndim") and images.ndim == 2: + zdim, ydim, xdim = 1, int(images.shape[0]), int(images.shape[1]) + else: + zdim, ydim, xdim = 1, 0, 0 + except Exception: + zdim, ydim, xdim = 1, 0, 0 + + depth = max(zdim, 1) + + k = requestedIndex + if zdim > 0: + k = max(0, min(k, zdim - 1)) + + try: + pilImg = reader.getImage(index=k, pilImage=True) + except Exception: + try: + pilImg = reader.getCentralImage(pilImage=True) + if zdim > 0: + k = max(0, min(zdim // 2, max(zdim - 1, 0))) + else: + k = 0 + except Exception: + pilImg = reader.getImage(index=0, pilImage=True) + k = 0 + + arr2d = self._coords3dPilTo2dTile(reader, pilImg) + if arr2d is None: + arrRaw = np.asarray(pilImg) + if arrRaw.ndim == 3: + arr2d = arrRaw.mean(axis=-1) + else: + arr2d = arrRaw.astype(np.float32, copy=False) + + gray = self._normalize2dSlice(arr2d, mode=normalize) + sliceUsed = k + except Exception: + gray = None + + if gray is None: + try: + vol3d, _props = readVolumeArray3d(str(volumePath)) # Z, Y, X + except HTTPException: + raise + except FileNotFoundError: + raise HTTPException( + status_code=404, + detail="Tomogram file not found on disk", + ) + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"Failed to read tomogram volume: {e}", + ) + + if vol3d.ndim != 3: + raise HTTPException( + status_code=500, + detail=f"Invalid tomogram volume shape {vol3d.shape}", + ) + + zdim, ydim, xdim = int(vol3d.shape[0]), int(vol3d.shape[1]), int(vol3d.shape[2]) + depth = max(zdim, 1) + + if axis == "z": + dim = zdim + elif axis == "y": + dim = ydim + else: + dim = xdim + + if dim <= 0: + raise HTTPException(status_code=500, detail="Empty tomogram volume") + + k = max(0, min(requestedIndex, dim - 1)) + + if axis == "z": + slice2d = vol3d[k, :, :] + elif axis == "y": + slice2d = vol3d[:, k, :] + else: + slice2d = vol3d[:, :, k] + + gray = self._normalize2dSlice(slice2d, mode=normalize) + sliceUsed = k + + if thumb is not None and thumb > 0: + pilTmp = PILImage.fromarray(gray.astype(np.uint8), mode="L") + pilTmp.thumbnail((thumb, thumb)) + gray = np.asarray(pilTmp) + + if gray.dtype != np.uint8: + gray = gray.astype(np.uint8, copy=False) + + imgArray = gray.astype(np.uint8, copy=False) + pilMode = "L" + + if usedColormap: + try: + import matplotlib.cm as cm + sliceNorm = imgArray.astype(np.float32) / 255.0 + cmapObj = cm.get_cmap(usedColormap) + rgba = cmapObj(sliceNorm) + rgb = (rgba[..., :3] * 255.0).clip(0, 255).astype(np.uint8) + imgArray = rgb + pilMode = "RGB" + except Exception: + usedColormap = None + imgArray = gray.astype(np.uint8, copy=False) + pilMode = "L" + + if scale is not None and scale != 1.0: + try: + pilScale = PILImage.fromarray(imgArray, mode=pilMode) + newW = max(1, int(round(pilScale.width * float(scale)))) + newH = max(1, int(round(pilScale.height * float(scale)))) + pilScale = pilScale.resize((newW, newH), resample=PILImage.Resampling.BILINEAR) + imgArray = np.asarray(pilScale, copy=False) + except Exception: + pass + + img = PILImage.fromarray(imgArray, mode=pilMode) + + buf = io.BytesIO() + img.save(buf, format=pilFormat, **saveKw) + + disp = "inline" if inline else "attachment" + filename = f"coords3d_{tomogramId}_axis-{axis}_slice-{sliceUsed}.{fmtLower}" + + headers = { + "Content-Disposition": f'{disp}; filename="{filename}"', + "Access-Control-Expose-Headers": ( + "Content-Disposition, " + "X-Preview-Mime, " + "X-Preview-Width, " + "X-Preview-Height, " + "X-Preview-Depth, " + "X-Preview-Colormap, " + "X-Preview-Format, " + "X-Preview-TomogramId" + ), + "X-Preview-Mime": mediaType, + "X-Preview-Width": str(img.width), + "X-Preview-Height": str(img.height), + "X-Preview-Depth": str(depth), + "X-Preview-Colormap": usedColormap or "", + "X-Preview-Format": pilFormat, + "X-Preview-TomogramId": str(tomogramId), + } + + return Response(content=buf.getvalue(), media_type=mediaType, headers=headers) + + def createCoords3dOutputFromPointsService( self, projectId: int, diff --git a/app/backend/viewers/postgresql_coords3d_reader.py b/app/backend/viewers/postgresql_coords3d_reader.py index ff11a599..5424390f 100644 --- a/app/backend/viewers/postgresql_coords3d_reader.py +++ b/app/backend/viewers/postgresql_coords3d_reader.py @@ -994,3 +994,68 @@ def _extractBoxSize(self, storedSet: Dict[str, Any]) -> Optional[float]: return value return None + + def getTomogramFile(self, tomogramId: Any) -> Optional[Dict[str, Any]]: + self.lastSkipReason = None + + storedSet = self._getStoredSet() + if storedSet is None: + self.lastSkipReason = "stored_set_not_found" + return None + + requested = self._toTextCandidate(tomogramId) + if not requested: + self.lastSkipReason = "empty_tomogram_id" + return None + + linkedTomograms = self._getLinkedTomogramsFromProperties(storedSet) + for item in linkedTomograms: + normalized = self._normalizeLinkedTomogramItem(item) + if normalized is None: + continue + + matchKeys = self._getTomogramMatchKeys(normalized) + if requested not in matchKeys: + continue + + fileName = normalized.get("fileName") + if not fileName: + self.lastSkipReason = "tomogram_file_not_found_in_linked_metadata tomogramId=%s" % requested + return None + + return { + "id": normalized.get("id"), + "tomoId": normalized.get("tomoId"), + "label": normalized.get("label"), + "name": normalized.get("name"), + "fileName": str(fileName), + "dims": normalized.get("dims"), + "voxelSize": normalized.get("voxelSize"), + } + + payload = self._buildTomogramPayloadFromProjectSets( + coordinateKeys={requested}, + countsByKey={}, + ) + + for item in payload or []: + matchKeys = self._getTomogramMatchKeys(item) + if requested not in matchKeys: + continue + + fileName = item.get("fileName") + if not fileName: + continue + + return { + "id": item.get("id"), + "tomoId": item.get("tomoId"), + "label": item.get("label"), + "name": item.get("name"), + "fileName": str(fileName), + "dims": item.get("dims"), + "voxelSize": item.get("voxelSize"), + } + + self.lastSkipReason = "tomogram_file_not_resolved tomogramId=%s" % requested + return None From 2b8a11ef643cdc67a21c0f0cb3120bef500b02a4 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 20 Jun 2026 23:55:05 +0200 Subject: [PATCH 057/278] store edited coords3d outputs in postgresql --- app/backend/api/routers/project_router.py | 3 ++- app/backend/api/services/project_service.py | 28 ++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 73ee8aac..3181c7c0 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2398,7 +2398,8 @@ def createCoords3dOutputFromPoints(projectId: int, result = service.createCoords3dOutputFromPointsService(projectId=projectId, protocolId=protocolId, outputName=outputName, - payload=payload) + payload=payload, + mapper=mapper,) resp = JSONResponse(result or {"success": True, "outputName": result['outputName']}) resp.headers["X-Debug-Auth"] = "ok" resp.headers["X-Debug-UserId"] = str(getattr(currentUser, "id", currentUser.get("id", ""))) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 0fa5b114..27627bbe 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -8722,13 +8722,13 @@ def _renderCoords3dTomogramSliceFromPath( return Response(content=buf.getvalue(), media_type=mediaType, headers=headers) - def createCoords3dOutputFromPointsService( self, projectId: int, protocolId: int, outputName: str, payload: Any, + mapper=None ) -> Dict[str, Any]: tomograms = payload['tomograms'] @@ -8815,6 +8815,30 @@ def createCoords3dOutputFromPointsService( except Exception as e: raise HTTPException(500, f"Failed to attach new coords3d output: {e}") + postgresqlStored = False + postgresqlError = None + + if mapper is not None: + try: + from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + setMapper = ScipionSetPostgresqlMapper(mapper.db) + setMapper.storeSet( + projectId=projectId, + protocolDbId=protocolId, + outputName=outName, + scipionSet=dstSet, + ) + postgresqlStored = True + except Exception as e: + postgresqlError = str(e) + logger.exception( + "Failed to store new Coordinates3D output in PostgreSQL. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outName, + ) + return { "success": True, "outputName": outName, @@ -8823,6 +8847,8 @@ def createCoords3dOutputFromPointsService( "sourceOutputName": outputName, "replacedPoints": replaced, "copiedPoints": copied, + "postgresqlStored": postgresqlStored, + "postgresqlError": postgresqlError, }, } From b689faf74fa7b245b5cd61a393cc18ff8e98250d Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sun, 21 Jun 2026 00:44:24 +0200 Subject: [PATCH 058/278] add postgresql volume reader --- .../viewers/postgresql_volume_reader.py | 699 ++++++++++++++++++ 1 file changed, 699 insertions(+) create mode 100644 app/backend/viewers/postgresql_volume_reader.py diff --git a/app/backend/viewers/postgresql_volume_reader.py b/app/backend/viewers/postgresql_volume_reader.py new file mode 100644 index 00000000..9d0e2e31 --- /dev/null +++ b/app/backend/viewers/postgresql_volume_reader.py @@ -0,0 +1,699 @@ +import json +import os +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple, Union + +import numpy as np + +from app.backend.utils.volume_utils import readVolumeArray3d + + +class PostgresqlVolumeReader: + """Read Volume, VolumeMask and SetOfVolumes outputs from PostgreSQL.""" + + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = int(projectId) + self.protocolId = int(protocolId) + self.outputName = str(outputName) + self.lastSkipReason = None + self._protocolDbId = None + self._storedSet = None + self._storedObjectTree = None + self._volumes = None + + def hasOutput(self) -> bool: + if self._getStoredSet() is not None: + return True + + return bool(self._getStoredObjectTree()) + + def listVolumes(self) -> Optional[List[Dict[str, Any]]]: + self.lastSkipReason = None + + if self._volumes is not None: + return self._volumes + + storedSet = self._getStoredSet() + if storedSet is not None: + volumes = self._listSetVolumes(storedSet) + if volumes: + self._volumes = volumes + return self._volumes + + objectTree = self._getStoredObjectTree() + if objectTree: + volume = self._buildSingleVolumeFromObjectTree(objectTree) + if volume is not None: + self._volumes = [volume] + return self._volumes + + self.lastSkipReason = "volume_output_not_resolved" + return None + + def getVolumeInfo(self, volumeId: Union[int, str]) -> Optional[Dict[str, Any]]: + self.lastSkipReason = None + + volumes = self.listVolumes() + if not volumes: + self.lastSkipReason = self.lastSkipReason or "volume_list_empty" + return None + + volume = self._findVolume(volumeId, volumes) + if volume is None: + self.lastSkipReason = "volume_not_found volumeId=%s" % str(volumeId) + return None + + info = dict(volume) + self._ensureVolumeInfoFromFile(info) + return info + + def getVolumeFile(self, volumeId: Union[int, str]) -> Optional[Dict[str, Any]]: + info = self.getVolumeInfo(volumeId) + if info is None: + return None + + fileName = info.get("fileName") or info.get("path") + if not fileName: + self.lastSkipReason = "volume_file_not_found volumeId=%s" % str(volumeId) + return None + + resolvedPath = self._resolveExistingPath(fileName) + if resolvedPath is None: + self.lastSkipReason = "volume_file_missing fileName=%s" % str(fileName) + return None + + info["fileName"] = resolvedPath + info["path"] = resolvedPath + return info + + def getVolumeArray( + self, + volumeId: Union[int, str], + ) -> Optional[Tuple[np.ndarray, Dict[str, Any], Dict[str, Any]]]: + info = self.getVolumeFile(volumeId) + if info is None: + return None + + volumePath = info.get("fileName") or info.get("path") + if not volumePath: + self.lastSkipReason = "volume_file_not_found volumeId=%s" % str(volumeId) + return None + + try: + array, props = readVolumeArray3d(str(volumePath)) + except Exception as exc: + self.lastSkipReason = "volume_read_failed volumeId=%s error=%s" % ( + str(volumeId), + str(exc), + ) + return None + + props = props if isinstance(props, dict) else {} + return array, props, info + + def getHistogram( + self, + volumeId: Union[int, str], + bins: int = 128, + ) -> Optional[Dict[str, Any]]: + result = self.getVolumeArray(volumeId) + if result is None: + return None + + array, _props, _info = result + cleanArray = np.asarray(array, dtype=np.float32) + cleanArray = cleanArray[np.isfinite(cleanArray)] + + if cleanArray.size == 0: + return { + "binEdges": [], + "counts": [], + } + + counts, binEdges = np.histogram(cleanArray, bins=max(4, int(bins or 128))) + + return { + "binEdges": [float(value) for value in binEdges.tolist()], + "counts": [int(value) for value in counts.tolist()], + } + + def _listSetVolumes(self, storedSet: Dict[str, Any]) -> List[Dict[str, Any]]: + volumes: List[Dict[str, Any]] = [] + + for index, item in enumerate(storedSet.get("items") or []): + values = self._normalizeJsonObject(item.get("values")) + volume = self._buildVolumeFromSetItem( + item=item, + values=values, + index=index, + ) + + if volume is not None: + volumes.append(volume) + + return volumes + + def _buildVolumeFromSetItem( + self, + item: Dict[str, Any], + values: Dict[str, Any], + index: int, + ) -> Optional[Dict[str, Any]]: + fileName, locationIndex = self._extractVolumeFile(values) + scipionItemId = item.get("scipionItemId") + + label = ( + item.get("label") + or self._firstValueBySuffix(values, ["objLabel", "label", "name", "volName"]) + or self._makeVolumeLabel(fileName, index) + ) + + volume: Dict[str, Any] = { + "id": int(index), + "index": int(index), + "name": str(label), + "label": str(label), + } + + if scipionItemId is not None: + volume["objectId"] = scipionItemId + volume["scipionItemId"] = scipionItemId + + className = item.get("className") or self._firstValueBySuffix(values, ["className"]) + if className: + volume["className"] = str(className) + + if fileName: + volume["fileName"] = fileName + volume["path"] = fileName + + if locationIndex is not None: + volume["locationIndex"] = locationIndex + + dims = self._extractDims(values) + if dims is not None: + volume["dims"] = dims + + samplingRate = self._extractSamplingRate(values) + if samplingRate is not None: + self._attachSamplingRate(volume, samplingRate) + + return volume + + def _buildSingleVolumeFromObjectTree( + self, + rows: List[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + root = None + valuesByPath: Dict[str, Any] = {} + + for row in rows: + path = str(row.get("path") or "") + if path == self.outputName: + root = row + continue + + if path.startswith(self.outputName + "."): + suffix = path[len(self.outputName) + 1:] + valuesByPath[suffix] = row.get("value") + + if root is None: + return None + + fileName, locationIndex = self._extractVolumeFile(valuesByPath) + label = root.get("label") or root.get("name") or self.outputName + + volume: Dict[str, Any] = { + "id": 0, + "index": 0, + "name": str(label), + "label": str(label), + } + + objectId = root.get("scipionObjId") + if objectId is not None: + volume["objectId"] = objectId + volume["scipionItemId"] = objectId + + className = root.get("className") + if className: + volume["className"] = str(className) + + if fileName: + volume["fileName"] = fileName + volume["path"] = fileName + + if locationIndex is not None: + volume["locationIndex"] = locationIndex + + dims = self._extractDims(valuesByPath) + if dims is not None: + volume["dims"] = dims + + samplingRate = self._extractSamplingRate(valuesByPath) + if samplingRate is not None: + self._attachSamplingRate(volume, samplingRate) + + return volume + + def _findVolume( + self, + volumeId: Union[int, str], + volumes: List[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + requested = self._toText(volumeId) + if requested is None: + return None + + for volume in volumes: + if self._toText(volume.get("id")) == requested: + return volume + + for volume in volumes: + if self._toText(volume.get("index")) == requested: + return volume + + requestedInt = self._toOptionalInt(requested) + if requestedInt is not None and 0 <= requestedInt < len(volumes): + return volumes[requestedInt] + + for volume in volumes: + for key in ("objectId", "scipionItemId"): + if self._toText(volume.get(key)) == requested: + return volume + + for volume in volumes: + for key in ("name", "label"): + if self._toText(volume.get(key)) == requested: + return volume + + return None + + def _ensureVolumeInfoFromFile(self, volume: Dict[str, Any]) -> None: + fileName = volume.get("fileName") or volume.get("path") + if not fileName: + return + + resolvedPath = self._resolveExistingPath(fileName) + if resolvedPath is None: + return + + volume["fileName"] = resolvedPath + volume["path"] = resolvedPath + + try: + array, props = readVolumeArray3d(resolvedPath) + except Exception: + return + + if "dims" not in volume and getattr(array, "ndim", None) == 3: + zDim, yDim, xDim = array.shape + volume["dims"] = [int(xDim), int(yDim), int(zDim)] + + if "samplingRate" not in volume: + samplingRate = self._extractSamplingRate(props if isinstance(props, dict) else {}) + if samplingRate is not None: + self._attachSamplingRate(volume, samplingRate) + + try: + finite = np.asarray(array, dtype=np.float32) + finite = finite[np.isfinite(finite)] + if finite.size: + volume["min"] = float(np.min(finite)) + volume["max"] = float(np.max(finite)) + volume["mean"] = float(np.mean(finite)) + except Exception: + pass + + def _getStoredSet(self) -> Optional[Dict[str, Any]]: + if self._storedSet is not None: + return self._storedSet + + protocolDbId = self._resolveProtocolDbId() + if protocolDbId is None: + return None + + storedSet = self.db.fetchOne( + """ + SELECT id, "projectId", "protocolDbId", "objectId", "outputName", + "setClassName", "itemClassName", properties, "createdAt", "updatedAt" + FROM scipion_sets + WHERE "projectId" = %s + AND "protocolDbId" = %s + AND "outputName" = %s + """, + (self.projectId, protocolDbId, self.outputName), + ) + + if storedSet is None: + return None + + items = self.db.fetchAll( + """ + SELECT id, "setId", "scipionItemId", enabled, label, comment, + creation, "values", "createdAt", "updatedAt" + FROM scipion_set_items + WHERE "setId" = %s + ORDER BY "scipionItemId" ASC NULLS LAST, id ASC + """, + (storedSet["id"],), + ) + + storedSet["items"] = items or [] + self._storedSet = storedSet + return self._storedSet + + def _getStoredObjectTree(self) -> List[Dict[str, Any]]: + if self._storedObjectTree is not None: + return self._storedObjectTree + + protocolDbId = self._resolveProtocolDbId() + if protocolDbId is None: + self._storedObjectTree = [] + return self._storedObjectTree + + rootPath = str(self.outputName) + rows = self.db.fetchAll( + """ + SELECT id, "projectId", "protocolDbId", "scipionObjId", "parentObjectId", + name, path, "className", value, label, comment, creation, + metadata, "createdAt", "updatedAt" + FROM scipion_objects + WHERE "projectId" = %s + AND "protocolDbId" = %s + AND (path = %s OR path LIKE %s) + ORDER BY path ASC + """, + (self.projectId, protocolDbId, rootPath, rootPath + ".%"), + ) + + self._storedObjectTree = rows or [] + return self._storedObjectTree + + def _resolveProtocolDbId(self) -> Optional[int]: + if self._protocolDbId is not None: + return self._protocolDbId + + row = self.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE id = %s + AND "projectId" = %s + """, + (self.protocolId, self.projectId), + ) + + if row is not None: + self._protocolDbId = int(row["id"]) + return self._protocolDbId + + row = self.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE "projectId" = %s + AND "protocolId" = %s + """, + (self.projectId, str(self.protocolId)), + ) + + if row is not None: + self._protocolDbId = int(row["id"]) + return self._protocolDbId + + self.lastSkipReason = "protocol_not_found" + return None + + def _extractVolumeFile(self, values: Dict[str, Any]) -> Tuple[Optional[str], Optional[int]]: + raw = self._firstValueBySuffix( + values, + [ + "fileName", + "filename", + "location", + "path", + "stack", + ], + ) + + return self._parseLocation(raw) + + def _parseLocation(self, raw: Any) -> Tuple[Optional[str], Optional[int]]: + parsed = self._parseJsonValue(raw) + + if isinstance(parsed, dict): + pathValue = None + for key in ("fileName", "filename", "path", "location", "stack"): + if key in parsed: + pathValue = parsed.get(key) + break + + indexValue = None + for key in ("index", "locationIndex", "slice", "itemIndex"): + if key in parsed: + indexValue = parsed.get(key) + break + + fileName, embeddedIndex = self._parseLocation(pathValue) + locationIndex = self._toOptionalInt(indexValue) + if locationIndex is None: + locationIndex = embeddedIndex + + return fileName, locationIndex + + if isinstance(parsed, (list, tuple)): + if len(parsed) >= 2: + locationIndex = self._toOptionalInt(parsed[0]) + fileName = self._toText(parsed[1]) + return fileName, locationIndex + + if len(parsed) == 1: + return self._parseLocation(parsed[0]) + + return None, None + + text = self._toText(parsed) + if not text: + return None, None + + locationIndex = None + fileName = text + + if "@" in text: + indexText, pathText = text.split("@", 1) + parsedIndex = self._toOptionalInt(indexText) + if parsedIndex is not None: + locationIndex = parsedIndex + fileName = pathText + + return fileName, locationIndex + + def _extractDims(self, values: Dict[str, Any]) -> Optional[List[int]]: + raw = self._firstValueBySuffix( + values, + [ + "dim", + "dims", + "dimensions", + "volumeDim", + "volumeDims", + ], + ) + + parsed = self._parseJsonValue(raw) + if isinstance(parsed, dict): + parsed = [ + parsed.get("x") or parsed.get("X") or parsed.get("nx"), + parsed.get("y") or parsed.get("Y") or parsed.get("ny"), + parsed.get("z") or parsed.get("Z") or parsed.get("nz"), + ] + + if isinstance(parsed, (list, tuple)) and len(parsed) >= 3: + dims = [] + for value in parsed[:3]: + intValue = self._toOptionalInt(value) + if intValue is None: + return None + dims.append(intValue) + return dims + + return None + + def _extractSamplingRate(self, values: Any) -> Optional[float]: + if isinstance(values, dict): + raw = self._firstValueBySuffix( + values, + [ + "samplingRate", + "pixelSize", + "voxelSize", + "sampling", + "apix", + ], + ) + else: + raw = values + + parsed = self._parseJsonValue(raw) + + if isinstance(parsed, dict): + for key in ("x", "X", "samplingRate", "pixelSize", "voxelSize", "apix"): + value = parsed.get(key) + number = self._toOptionalFloat(value) + if number is not None: + return number + return None + + if isinstance(parsed, (list, tuple)) and parsed: + return self._toOptionalFloat(parsed[0]) + + return self._toOptionalFloat(parsed) + + def _attachSamplingRate(self, volume: Dict[str, Any], samplingRate: float) -> None: + samplingRate = float(samplingRate) + volume["samplingRate"] = samplingRate + volume["pixelSize"] = samplingRate + volume["voxelSize"] = [samplingRate, samplingRate, samplingRate] + + def _resolveExistingPath(self, fileName: Any) -> Optional[str]: + text = self._toText(fileName) + if not text: + return None + + path = Path(text).expanduser() + candidates = [] + + if path.is_absolute(): + candidates.append(path) + else: + candidates.append(path) + candidates.append(Path.cwd() / path) + + for candidate in candidates: + try: + resolved = candidate.resolve() + if resolved.exists(): + return str(resolved) + except Exception: + continue + + return None + + def _makeVolumeLabel(self, fileName: Optional[str], index: int) -> str: + if fileName: + try: + return Path(str(fileName)).name + except Exception: + pass + + return "Volume %s" % (int(index) + 1) + + def _getVolumeApiKeys(self, volume: Dict[str, Any]) -> Set[str]: + return { + text + for text in ( + self._toText(volume.get("id")), + self._toText(volume.get("index")), + ) + if text + } + + def _firstValueBySuffix(self, values: Dict[str, Any], suffixes: List[str]) -> Any: + if not isinstance(values, dict): + return None + + normalizedSuffixes = [ + self._normalizeKey(suffix) + for suffix in suffixes + if suffix is not None + ] + + for suffix in normalizedSuffixes: + for key, value in values.items(): + normalizedKey = self._normalizeKey(key) + if normalizedKey == suffix or normalizedKey.endswith(suffix): + return value + + return None + + def _normalizeJsonObject(self, value: Any) -> Dict[str, Any]: + parsed = self._parseJsonValue(value) + if isinstance(parsed, dict): + return parsed + return {} + + def _parseJsonValue(self, value: Any) -> Any: + if isinstance(value, (dict, list, tuple)): + return value + + if not isinstance(value, str): + return value + + text = value.strip() + if not text: + return value + + if not ( + text.startswith("{") + or text.startswith("[") + or text.startswith('"') + ): + return value + + try: + return json.loads(text) + except Exception: + return value + + def _normalizeKey(self, value: Any) -> str: + return ( + str(value or "") + .replace("_", "") + .replace(".", "") + .replace("-", "") + .lower() + ) + + def _toText(self, value: Any) -> Optional[str]: + if value is None: + return None + + getter = getattr(value, "get", None) + if callable(getter): + try: + value = getter() + except Exception: + return None + + text = str(value).strip() + return text or None + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + + try: + return int(value) + except Exception: + pass + + try: + return int(float(value)) + except Exception: + return None + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + + try: + number = float(value) + except Exception: + return None + + if not np.isfinite(number): + return None + + return number \ No newline at end of file From ba95e916933a825008ee02be2584d8ff29201e46 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sun, 21 Jun 2026 14:23:50 +0200 Subject: [PATCH 059/278] Match volume slice defaults with coords3d --- app/backend/api/routers/project_router.py | 21 +- app/backend/api/services/project_service.py | 435 +++++++++++++++++- .../viewers/postgresql_volume_reader.py | 65 ++- 3 files changed, 488 insertions(+), 33 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 3181c7c0..10222bb9 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1510,7 +1510,10 @@ def listOutputVolumes( if not project: raise HTTPException(status_code=404, detail="Project not found") - items = service.listOutputVolumesService(projectId, protocolId, outputName) + items = service.listOutputVolumesService(projectId, + protocolId, + outputName, + mapper=mapper) from fastapi.responses import JSONResponse resp = JSONResponse(items) @@ -1539,7 +1542,11 @@ def getVolumeInfo( if not project: raise HTTPException(status_code=404, detail="Project not found") - info = service.getVolumeInfoService(projectId, protocolId, outputName, volumeId) + info = service.getVolumeInfoService(projectId, + protocolId, + outputName, + volumeId, + mapper=mapper,) from fastapi.responses import JSONResponse resp = JSONResponse(info) @@ -1582,6 +1589,7 @@ def getVolumeHistogram( outputName=outputName, volumeId=volumeId, bins=bins, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -1608,7 +1616,7 @@ def renderVolumeSlice( colormapParam: Optional[str] = Query(None, alias="colormap"), formatParam: Optional[str] = Query(None, alias="format"), fmtParam: Optional[str] = Query(None, alias="fmt"), - normalize: Optional[str] = Query(None), + normalize: Optional[str] = Query("minmax"), scale: float = Query(1.0, gt=0), inline: bool = Query(True), thumb: Optional[int] = Query(None, ge=32, le=2048), @@ -1622,7 +1630,7 @@ def renderVolumeSlice( if not project: raise HTTPException(status_code=404, detail="Project not found") - cmap = cmapParam or colormapParam or "viridis" + cmap = cmapParam or colormapParam fmt = fmtParam or formatParam or "webp" resp = service.renderVolumeSliceService( @@ -1640,6 +1648,7 @@ def renderVolumeSlice( thumb=thumb, fast=fast, quality=quality, + mapper=mapper, ) resp.headers["X-Debug-Auth"] = "ok" resp.headers["X-Debug-UserId"] = str(getattr(currentUser, "id", currentUser.get("id", ""))) @@ -1673,6 +1682,7 @@ def getVolumeData3d( volumeId=volumeId, maxDim=maxDim, method=method, + mapper=mapper, ) @router.get( @@ -1707,7 +1717,8 @@ def getVolumeSurfaceMesh( maxDim=maxDim, method=method, maxTriangles=maxTriangles, - currentUser=currentUser) + currentUser=currentUser, + mapper=mapper,) except HTTPException: raise diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 27627bbe..3b5a8b79 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6570,12 +6570,247 @@ def _resolveOutputForVolumes(self, protocolId: int, outputName: str): raise HTTPException(status_code=404, detail=f"Output '{outputName}' not found in protocol (tried {candidates})") - def listOutputVolumesService(self, projectId: int, protocolId: int, outputName: str): + def _getPostgresqlVolumeReaderIfAvailable( + self, + mapper, + projectId: int, + protocolId: int, + outputName: str, + ): + if mapper is None: + return None + + try: + from app.backend.viewers.postgresql_volume_reader import PostgresqlVolumeReader + + reader = PostgresqlVolumeReader( + db=mapper.db, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if reader.hasOutput(): + return reader + + except Exception: + logger.exception( + "Failed to initialize PostgreSQL volume reader. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + ) + + return None + + def _renderVolumeSliceFromArray( + self, + volume: np.ndarray, + volumeId: Union[int, str], + volumeName: Optional[str], + sliceIndex: int, + axis: str, + colormap: Optional[str], + normalize: Optional[str], + scale: float, + inline: bool, + fmt: str = "webp", + thumb: Optional[int] = None, + quality: int = 75, + ) -> Response: + from PIL import Image as PILImage + + volume = np.asarray(volume, dtype=np.float32) + + if volume.ndim == 2: + volume = volume[None, :, :] + + if volume.ndim != 3: + raise HTTPException( + status_code=500, + detail=f"Invalid volume shape: {volume.shape}", + ) + + axis = (axis or "z").lower() + if axis not in ("x", "y", "z"): + axis = "z" + + fmtLower = (fmt or "png").lower() + if fmtLower in ("jpg", "jpeg"): + pilFormat = "JPEG" + mediaType = "image/jpeg" + saveKw = {"quality": int(quality or 75)} + elif fmtLower == "webp": + pilFormat = "WEBP" + mediaType = "image/webp" + saveKw = {"quality": int(quality or 75)} + else: + pilFormat = "PNG" + mediaType = "image/png" + saveKw = {} + + zdim, ydim, xdim = int(volume.shape[0]), int(volume.shape[1]), int(volume.shape[2]) + depth = max(zdim, 1) + + try: + requestedIndex = int(sliceIndex or 0) + except Exception: + requestedIndex = 0 + requestedIndex = max(0, requestedIndex) + + if axis == "z": + dim = zdim + elif axis == "y": + dim = ydim + else: + dim = xdim + + if dim <= 0: + raise HTTPException(status_code=500, detail="Empty volume") + + sliceUsed = max(0, min(requestedIndex, dim - 1)) + + if axis == "z": + slice2d = volume[sliceUsed, :, :] + elif axis == "y": + slice2d = volume[:, sliceUsed, :] + else: + slice2d = volume[:, :, sliceUsed] + + gray = self._normalize2dSlice(slice2d, mode=normalize or "minmax") + + if thumb is not None and thumb > 0: + pilTmp = PILImage.fromarray(gray.astype(np.uint8), mode="L") + pilTmp.thumbnail((thumb, thumb)) + gray = np.asarray(pilTmp) + + if gray.dtype != np.uint8: + gray = gray.astype(np.uint8, copy=False) + + usedColormap = colormap + imgArray = gray.astype(np.uint8, copy=False) + pilMode = "L" + + if usedColormap: + try: + import matplotlib.cm as cm + + sliceNorm = imgArray.astype(np.float32) / 255.0 + cmapObj = cm.get_cmap(usedColormap) + rgba = cmapObj(sliceNorm) + rgb = (rgba[..., :3] * 255.0).clip(0, 255).astype(np.uint8) + imgArray = rgb + pilMode = "RGB" + except Exception: + usedColormap = None + imgArray = gray.astype(np.uint8, copy=False) + pilMode = "L" + + if scale is not None and scale != 1.0: + try: + pilScale = PILImage.fromarray(imgArray, mode=pilMode) + newW = max(1, int(round(pilScale.width * float(scale)))) + newH = max(1, int(round(pilScale.height * float(scale)))) + pilScale = pilScale.resize( + (newW, newH), + resample=PILImage.Resampling.BILINEAR, + ) + imgArray = np.asarray(pilScale, copy=False) + except Exception: + pass + + img = PILImage.fromarray(imgArray, mode=pilMode) + + buf = io.BytesIO() + img.save(buf, format=pilFormat, **saveKw) + + disp = "inline" if inline else "attachment" + safeName = str(volumeName or f"volume_{volumeId}").replace("/", "_").replace("\\", "_") + filename = f"{safeName}_axis-{axis}_slice-{sliceUsed}.{fmtLower}" + + headers = { + "Content-Disposition": f'{disp}; filename="{filename}"', + "Access-Control-Expose-Headers": ( + "Content-Disposition, " + "X-Preview-Mime, " + "X-Preview-Width, " + "X-Preview-Height, " + "X-Preview-Depth, " + "X-Preview-Colormap, " + "X-Preview-Format, " + "X-Preview-VolumeId" + ), + "X-Preview-Mime": mediaType, + "X-Preview-Width": str(img.width), + "X-Preview-Height": str(img.height), + "X-Preview-Depth": str(depth), + "X-Preview-Colormap": usedColormap or "", + "X-Preview-Format": pilFormat, + "X-Preview-VolumeId": str(volumeId), + } + + return Response(content=buf.getvalue(), media_type=mediaType, headers=headers) + + def listOutputVolumesService( + self, + projectId: int, + protocolId: int, + outputName: str, + mapper=None, + ): + pgReader = self._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.listVolumes() + if payload is not None: + return payload + + logger.info( + "Skipping PostgreSQL volume list reader. projectId=%s protocolId=%s outputName=%s reason=%s", + projectId, + protocolId, + outputName, + getattr(pgReader, "lastSkipReason", None), + ) + protocol, output = self._resolveOutputForVolumes(protocolId, outputName) op = OutputsPreview(self.currentProject, protocol, output) return op.listOutputVolumes() - def getVolumeInfoService(self, projectId: int, protocolId: int, outputName: str, volumeId: int): + def getVolumeInfoService( + self, + projectId: int, + protocolId: int, + outputName: str, + volumeId: Union[int, str], + mapper=None, + ): + pgReader = self._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.getVolumeInfo(volumeId) + if payload is not None: + return payload + + logger.info( + "Skipping PostgreSQL volume info reader. projectId=%s protocolId=%s outputName=%s volumeId=%s reason=%s", + projectId, + protocolId, + outputName, + volumeId, + getattr(pgReader, "lastSkipReason", None), + ) + protocol, output = self._resolveOutputForVolumes(protocolId, outputName) op = OutputsPreview(self.currentProject, protocol, output) return op.getVolumeInfo(volumeId) @@ -6587,24 +6822,39 @@ def getVolumeHistogramService( outputName: str, volumeId: Union[int, str], bins: int = 128, + mapper=None, ): - """ - Compute or retrieve an intensity histogram for a single volume. + pgReader = self._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.getHistogram(volumeId=volumeId, bins=bins) + if payload is not None: + return { + "binEdges": payload.get("binEdges") or [], + "counts": payload.get("counts") or [], + } + + logger.info( + "Skipping PostgreSQL volume histogram reader. projectId=%s protocolId=%s outputName=%s volumeId=%s reason=%s", + projectId, + protocolId, + outputName, + volumeId, + getattr(pgReader, "lastSkipReason", None), + ) - It delegates to OutputsPreview so all volume handling stays in one place. - The result is normalized to always expose: - { - "binEdges": [...], - "counts": [...] - } - """ protocol, output = self._resolveOutputForVolumes(protocolId, outputName) op = OutputsPreview(self.currentProject, protocol, output) if isinstance(output, SetOfVolumes): - output = output.getItem('_objId', volumeId + 1) - raw = op.getVolumeHistogram(volumePath=output.getFileName(), bins=bins) + output = output.getItem('_objId', int(volumeId) + 1) + raw = op.getVolumeHistogram(volumePath=output.getFileName(), bins=bins) if not raw: return {"binEdges": [], "counts": []} @@ -6615,6 +6865,7 @@ def getVolumeHistogramService( binEdges = list(binEdges) except Exception: binEdges = [] + try: counts = list(counts) except Exception: @@ -6626,11 +6877,59 @@ def getVolumeHistogramService( } def renderVolumeSliceService( - self, projectId: int, protocolId: int, outputName: str, volumeId: int, - sliceIndex: int, axis: str, colormap: Optional[str], normalize: Optional[str], - scale: float, inline: bool, fmt: str = "webp", - thumb: Optional[int] = None, fast: bool = True, quality: int = 75, + self, + projectId: int, + protocolId: int, + outputName: str, + volumeId: Union[int, str], + sliceIndex: int, + axis: str, + colormap: Optional[str], + normalize: Optional[str], + scale: float, + inline: bool, + fmt: str = "webp", + thumb: Optional[int] = None, + fast: bool = True, + quality: int = 75, + mapper=None, ) -> Response: + pgReader = self._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + result = pgReader.getVolumeArray(volumeId) + if result is not None: + volume, _props, info = result + + return self._renderVolumeSliceFromArray( + volume=volume, + volumeId=volumeId, + volumeName=info.get("name") or info.get("label"), + sliceIndex=sliceIndex, + axis=axis, + colormap=colormap, + normalize=normalize or "minmax", + scale=scale, + inline=inline, + fmt=fmt, + thumb=thumb, + quality=quality, + ) + + logger.info( + "Skipping PostgreSQL volume slice reader. projectId=%s protocolId=%s outputName=%s volumeId=%s reason=%s", + projectId, + protocolId, + outputName, + volumeId, + getattr(pgReader, "lastSkipReason", None), + ) + protocol, output = self._resolveOutputForVolumes(protocolId, outputName) op = OutputsPreview(self.currentProject, protocol, output) return op.renderVolumeSlice( @@ -6655,12 +6954,49 @@ def getVolumeData3dService( volumeId: Union[int, str], maxDim: int = 160, method: str = "binning", + mapper=None, ): + pgReader = self._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + result = pgReader.getVolumeArray(volumeId) + if result is not None: + volume, _props, _info = result + + if (method or "").lower() == "none": + volumeSmall = np.asarray(volume, dtype=np.float32) + else: + volumeSmall = self._downsampleVolumePreview( + np.asarray(volume, dtype=np.float32), + maxDim=maxDim, + method=method, + ) + + z, y, x = volumeSmall.shape + return { + "dims": [int(z), int(y), int(x)], + "values": volumeSmall.ravel(order="C").astype(np.float32).tolist(), + } + + logger.info( + "Skipping PostgreSQL volume data3d reader. projectId=%s protocolId=%s outputName=%s volumeId=%s reason=%s", + projectId, + protocolId, + outputName, + volumeId, + getattr(pgReader, "lastSkipReason", None), + ) + protocol, output = self._resolveOutputForVolumes(protocolId, outputName) volumePath = self._getVolumePathFromOutput(output, volumeId) try: - vol, _props = readVolumeArray3d(volumePath) # Z, Y, X + vol, _props = readVolumeArray3d(volumePath) except HTTPException: raise except FileNotFoundError: @@ -6668,7 +7004,10 @@ def getVolumeData3dService( except Exception as e: raise HTTPException(status_code=500, detail=f"Cannot read volume file: {e}") - volSmall = self._downsampleVolumePreview(vol, maxDim=maxDim, method=method) + if (method or "").lower() == "none": + volSmall = np.asarray(vol, dtype=np.float32) + else: + volSmall = self._downsampleVolumePreview(vol, maxDim=maxDim, method=method) z, y, x = volSmall.shape return { @@ -6860,8 +7199,62 @@ def _downsampleVolumeForSurface( maxDim=maxDim, method=methodLower) - def getVolumeSurfaceMesh(self, protocolId, outputName, volumeId, level, - maxDim, method, maxTriangles, currentUser): + def getVolumeSurfaceMesh( + self, + projectId: int, + protocolId: int, + outputName: str, + volumeId: Union[int, str], + level, + maxDim, + method, + maxTriangles, + currentUser, + mapper=None, + ): + pgReader = self._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + result = pgReader.getVolumeArray(volumeId) + if result is not None: + volume, _props, _info = result + volumeSmall = self._downsampleVolumeForSurface( + np.asarray(volume, dtype=np.float32), + maxDim=maxDim, + method=method, + ) + + mesh = buildVolumeSurfaceMesh( + volumeSmall, + level=level, + maxTriangles=maxTriangles, + ) + + mesh["sourceDims"] = [int(volume.shape[0]), int(volume.shape[1]), int(volume.shape[2])] + mesh["maxDim"] = int(maxDim) + mesh["method"] = method + mesh["volumeId"] = str(volumeId) + mesh["outputName"] = outputName + + response = JSONResponse(mesh) + response.headers["X-Debug-Auth"] = "ok" + response.headers["X-Debug-UserId"] = str(getattr(currentUser, "id", currentUser.get("id", ""))) + response.headers["Vary"] = "Authorization" + return response + + logger.info( + "Skipping PostgreSQL volume surface reader. projectId=%s protocolId=%s outputName=%s volumeId=%s reason=%s", + projectId, + protocolId, + outputName, + volumeId, + getattr(pgReader, "lastSkipReason", None), + ) _protocol, output = self._resolveOutputForVolumes(protocolId, outputName) volumePath = self._getVolumePathFromOutput(output, volumeId) diff --git a/app/backend/viewers/postgresql_volume_reader.py b/app/backend/viewers/postgresql_volume_reader.py index 9d0e2e31..d479cb71 100644 --- a/app/backend/viewers/postgresql_volume_reader.py +++ b/app/backend/viewers/postgresql_volume_reader.py @@ -163,10 +163,15 @@ def _buildVolumeFromSetItem( fileName, locationIndex = self._extractVolumeFile(values) scipionItemId = item.get("scipionItemId") - label = ( - item.get("label") - or self._firstValueBySuffix(values, ["objLabel", "label", "name", "volName"]) - or self._makeVolumeLabel(fileName, index) + rawLabel = ( + item.get("label") + or self._firstValueBySuffix(values, ["objLabel", "label", "name", "volName"]) + ) + + label = self._normalizeVolumeDisplayName( + label=rawLabel, + fileName=fileName, + index=index, ) volume: Dict[str, Any] = { @@ -174,6 +179,7 @@ def _buildVolumeFromSetItem( "index": int(index), "name": str(label), "label": str(label), + "relPath": str(label), } if scipionItemId is not None: @@ -222,13 +228,19 @@ def _buildSingleVolumeFromObjectTree( return None fileName, locationIndex = self._extractVolumeFile(valuesByPath) - label = root.get("label") or root.get("name") or self.outputName + rawLabel = root.get("label") or root.get("name") or self.outputName + label = self._normalizeVolumeDisplayName( + label=rawLabel, + fileName=fileName, + index=0, + ) volume: Dict[str, Any] = { "id": 0, "index": 0, "name": str(label), "label": str(label), + "relPath": str(label), } objectId = root.get("scipionObjId") @@ -307,9 +319,10 @@ def _ensureVolumeInfoFromFile(self, volume: Dict[str, Any]) -> None: except Exception: return - if "dims" not in volume and getattr(array, "ndim", None) == 3: + if getattr(array, "ndim", None) == 3: zDim, yDim, xDim = array.shape - volume["dims"] = [int(xDim), int(yDim), int(zDim)] + volume["dims"] = [int(zDim), int(yDim), int(xDim)] + volume["xyzDims"] = [int(xDim), int(yDim), int(zDim)] if "samplingRate" not in volume: samplingRate = self._extractSamplingRate(props if isinstance(props, dict) else {}) @@ -581,6 +594,44 @@ def _resolveExistingPath(self, fileName: Any) -> Optional[str]: return None + def _normalizeVolumeDisplayName( + self, + label: Any, + fileName: Optional[str], + index: int, + ) -> str: + labelText = self._toText(label) + + if labelText: + if self._looksLikePath(labelText): + return Path(labelText).name + return labelText + + if fileName: + return Path(str(fileName)).name + + return "Volume %s" % (int(index) + 1) + + def _looksLikePath(self, value: Any) -> bool: + text = str(value or "").strip() + if not text: + return False + + if "/" in text or "\\" in text: + return True + + suffix = Path(text).suffix.lower() + return suffix in { + ".mrc", + ".map", + ".mrcs", + ".rec", + ".ali", + ".vol", + ".spi", + ".stk", + } + def _makeVolumeLabel(self, fileName: Optional[str], index: int) -> str: if fileName: try: From bbb242efa03b5a1800e30f00d308d297d304bcc0 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sun, 21 Jun 2026 17:50:33 +0200 Subject: [PATCH 060/278] Reuse image registry for postgresql volume slices --- app/backend/api/routers/project_router.py | 3 +- app/backend/api/services/project_service.py | 188 +++----------------- 2 files changed, 22 insertions(+), 169 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 10222bb9..5af30d62 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1710,7 +1710,8 @@ def getVolumeSurfaceMesh( try: - return service.getVolumeSurfaceMesh(protocolId=protocolId, + return service.getVolumeSurfaceMesh(projectId=projectId, + protocolId=protocolId, outputName=outputName, volumeId=volumeId, level=level, diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 3b5a8b79..ae4bdfce 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6603,154 +6603,6 @@ def _getPostgresqlVolumeReaderIfAvailable( return None - def _renderVolumeSliceFromArray( - self, - volume: np.ndarray, - volumeId: Union[int, str], - volumeName: Optional[str], - sliceIndex: int, - axis: str, - colormap: Optional[str], - normalize: Optional[str], - scale: float, - inline: bool, - fmt: str = "webp", - thumb: Optional[int] = None, - quality: int = 75, - ) -> Response: - from PIL import Image as PILImage - - volume = np.asarray(volume, dtype=np.float32) - - if volume.ndim == 2: - volume = volume[None, :, :] - - if volume.ndim != 3: - raise HTTPException( - status_code=500, - detail=f"Invalid volume shape: {volume.shape}", - ) - - axis = (axis or "z").lower() - if axis not in ("x", "y", "z"): - axis = "z" - - fmtLower = (fmt or "png").lower() - if fmtLower in ("jpg", "jpeg"): - pilFormat = "JPEG" - mediaType = "image/jpeg" - saveKw = {"quality": int(quality or 75)} - elif fmtLower == "webp": - pilFormat = "WEBP" - mediaType = "image/webp" - saveKw = {"quality": int(quality or 75)} - else: - pilFormat = "PNG" - mediaType = "image/png" - saveKw = {} - - zdim, ydim, xdim = int(volume.shape[0]), int(volume.shape[1]), int(volume.shape[2]) - depth = max(zdim, 1) - - try: - requestedIndex = int(sliceIndex or 0) - except Exception: - requestedIndex = 0 - requestedIndex = max(0, requestedIndex) - - if axis == "z": - dim = zdim - elif axis == "y": - dim = ydim - else: - dim = xdim - - if dim <= 0: - raise HTTPException(status_code=500, detail="Empty volume") - - sliceUsed = max(0, min(requestedIndex, dim - 1)) - - if axis == "z": - slice2d = volume[sliceUsed, :, :] - elif axis == "y": - slice2d = volume[:, sliceUsed, :] - else: - slice2d = volume[:, :, sliceUsed] - - gray = self._normalize2dSlice(slice2d, mode=normalize or "minmax") - - if thumb is not None and thumb > 0: - pilTmp = PILImage.fromarray(gray.astype(np.uint8), mode="L") - pilTmp.thumbnail((thumb, thumb)) - gray = np.asarray(pilTmp) - - if gray.dtype != np.uint8: - gray = gray.astype(np.uint8, copy=False) - - usedColormap = colormap - imgArray = gray.astype(np.uint8, copy=False) - pilMode = "L" - - if usedColormap: - try: - import matplotlib.cm as cm - - sliceNorm = imgArray.astype(np.float32) / 255.0 - cmapObj = cm.get_cmap(usedColormap) - rgba = cmapObj(sliceNorm) - rgb = (rgba[..., :3] * 255.0).clip(0, 255).astype(np.uint8) - imgArray = rgb - pilMode = "RGB" - except Exception: - usedColormap = None - imgArray = gray.astype(np.uint8, copy=False) - pilMode = "L" - - if scale is not None and scale != 1.0: - try: - pilScale = PILImage.fromarray(imgArray, mode=pilMode) - newW = max(1, int(round(pilScale.width * float(scale)))) - newH = max(1, int(round(pilScale.height * float(scale)))) - pilScale = pilScale.resize( - (newW, newH), - resample=PILImage.Resampling.BILINEAR, - ) - imgArray = np.asarray(pilScale, copy=False) - except Exception: - pass - - img = PILImage.fromarray(imgArray, mode=pilMode) - - buf = io.BytesIO() - img.save(buf, format=pilFormat, **saveKw) - - disp = "inline" if inline else "attachment" - safeName = str(volumeName or f"volume_{volumeId}").replace("/", "_").replace("\\", "_") - filename = f"{safeName}_axis-{axis}_slice-{sliceUsed}.{fmtLower}" - - headers = { - "Content-Disposition": f'{disp}; filename="{filename}"', - "Access-Control-Expose-Headers": ( - "Content-Disposition, " - "X-Preview-Mime, " - "X-Preview-Width, " - "X-Preview-Height, " - "X-Preview-Depth, " - "X-Preview-Colormap, " - "X-Preview-Format, " - "X-Preview-VolumeId" - ), - "X-Preview-Mime": mediaType, - "X-Preview-Width": str(img.width), - "X-Preview-Height": str(img.height), - "X-Preview-Depth": str(depth), - "X-Preview-Colormap": usedColormap or "", - "X-Preview-Format": pilFormat, - "X-Preview-VolumeId": str(volumeId), - } - - return Response(content=buf.getvalue(), media_type=mediaType, headers=headers) - def listOutputVolumesService( self, projectId: int, @@ -6902,24 +6754,24 @@ def renderVolumeSliceService( ) if pgReader is not None: - result = pgReader.getVolumeArray(volumeId) - if result is not None: - volume, _props, info = result - - return self._renderVolumeSliceFromArray( - volume=volume, - volumeId=volumeId, - volumeName=info.get("name") or info.get("label"), - sliceIndex=sliceIndex, - axis=axis, - colormap=colormap, - normalize=normalize or "minmax", - scale=scale, - inline=inline, - fmt=fmt, - thumb=thumb, - quality=quality, - ) + info = pgReader.getVolumeFile(volumeId) + if info is not None: + volumePath = info.get("fileName") or info.get("path") + if volumePath and os.path.exists(str(volumePath)): + return self._renderTomogramSliceFromPath( + volumePath=str(volumePath), + tomogramId=volumeId, + sliceIndex=sliceIndex, + axis=axis, + colormap=colormap, + normalize=normalize or "minmax", + scale=scale, + inline=inline, + fmt=fmt, + thumb=thumb, + fast=fast, + quality=quality, + ) logger.info( "Skipping PostgreSQL volume slice reader. projectId=%s protocolId=%s outputName=%s volumeId=%s reason=%s", @@ -8655,7 +8507,7 @@ def renderCoords3dTomogramSliceService( if tomogramInfo is not None: volumePath = tomogramInfo.get("fileName") if volumePath and os.path.exists(volumePath): - return self._renderCoords3dTomogramSliceFromPath( + return self._renderTomogramSliceFromPath( volumePath=volumePath, tomogramId=tomogramId, sliceIndex=sliceIndex, @@ -8908,7 +8760,7 @@ def renderCoords3dTomogramSliceService( return Response(content=buf.getvalue(), media_type=mediaType, headers=headers) - def _renderCoords3dTomogramSliceFromPath( + def _renderTomogramSliceFromPath( self, volumePath: str, tomogramId: Union[int, str], From 9a2a5b67f865a475d325eba7f8a1aa68a9f1b6b1 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sun, 21 Jun 2026 22:41:32 +0200 Subject: [PATCH 061/278] store class size in postgresql metadata --- app/backend/mapper/scipion_set_mapper.py | 50 +++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index c263bc23..af35c4ae 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,7 +41,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 7 +NESTED_LOGICAL_TABLES_VERSION = 8 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): @@ -1015,6 +1015,10 @@ def _getItemValues(self, item: Any, scipionSet: Optional[Any] = None) -> Dict[st if str(label) != SELF_LABEL } + classSize = self._getClassItemSize(item) + if classSize is not None and "_size" not in values: + values["_size"] = classSize + self._addCoordinate3dBottomLeftCoordinates( item=item, values=values, @@ -1023,6 +1027,30 @@ def _getItemValues(self, item: Any, scipionSet: Optional[Any] = None) -> Dict[st return values + def _getClassItemSize(self, item: Any) -> Optional[int]: + className = self._getClassName(item) or item.__class__.__name__ + if not str(className or "").startswith("Class"): + return None + + for methodName in ("getSize", "getObjSize", "count"): + getter = getattr(item, methodName, None) + if not callable(getter): + continue + + try: + value = getter() + except Exception: + continue + + sizeValue = self._toOptionalInt(value) + if sizeValue is not None: + return sizeValue + + try: + return int(len(item)) + except Exception: + return None + def _addCoordinate3dBottomLeftCoordinates( self, item: Any, @@ -1275,6 +1303,21 @@ def _getSetColumns(self, itemSchema: Dict[str, Any]) -> List[Dict[str, Any]]: ) position += 1 + if self._schemaIsClassItem(itemSchema) and not any( + column.get("labelProperty") == "_size" + for column in columns + ): + columns.append( + { + "labelProperty": "_size", + "columnName": "c%02d" % position, + "className": "Integer", + "valueType": "integer", + "position": position, + "indexed": True, + } + ) + return columns def _getItemClassName(self, item: Any, itemSchema: Dict[str, Any]) -> str: @@ -1284,6 +1327,11 @@ def _getItemClassName(self, item: Any, itemSchema: Dict[str, Any]) -> str: return schemaClassName return self._getClassName(item) or item.__class__.__name__ if item is not None else "Unknown" + def _schemaIsClassItem(self, itemSchema: Dict[str, Any]) -> bool: + selfSchema = itemSchema.get(SELF_LABEL) + className = self._getSchemaClassName(selfSchema) + return str(className or "").startswith("Class") + def _getSchemaClassName(self, schemaValue: Any) -> Optional[str]: if isinstance(schemaValue, (tuple, list)) and schemaValue: return str(schemaValue[0]) if schemaValue[0] else None From 5dec466e9f5e814faa4e2bb07866b6e9cc81bdeb Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sun, 21 Jun 2026 23:06:36 +0200 Subject: [PATCH 062/278] expose metadata additional info columns --- app/backend/api/services/project_service.py | 33 +++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index ae4bdfce..09cda88e 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -9617,11 +9617,44 @@ def getMetadataTableSchemaService(self, projectId: int, hasColumnId = True columns = list(table.getColumns()) + columnNames = {str(col.getName()) for col in columns} + + additionalInfoColumns = [] + try: + dao = objMgr.getDAO() + getAdditionalInfoFn = getattr(dao, "getTableWithAdditionalInfo", None) + + if callable(getAdditionalInfoFn): + additionalTable, additionalColumns = getAdditionalInfoFn() + + if hasattr(additionalTable, "getName"): + additionalTableName = str(additionalTable.getName() or "") + else: + additionalTableName = "" if additionalTable is None else str(additionalTable) + + currentTableNames = { + str(tableName or ""), + str(table.getName() or ""), + str(table.getAlias() or ""), + } + + appliesToTable = not additionalTableName or additionalTableName in currentTableNames + + if appliesToTable: + additionalInfoColumns = [ + str(columnName) + for columnName in (additionalColumns or []) + if str(columnName) in columnNames + ] + except Exception: + additionalInfoColumns = [] + schema = { "name": tableName, "alias": table.getAlias(), "hasColumnId": bool(hasColumnId), "actions": actions, + "additionalInfoColumns": additionalInfoColumns, # "renderLabels": renderLabels, # "orderLabels": orderLabels, "columns": [], From 58077ac808fdeb6f9f326ea007b00a21e7d94330 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 08:52:45 +0200 Subject: [PATCH 063/278] Add postgresql integrated analyze context --- app/backend/api/routers/project_router.py | 1 + app/backend/api/services/project_service.py | 382 ++++++++++++++++++++ 2 files changed, 383 insertions(+) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 5af30d62..fa0a3e57 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2444,6 +2444,7 @@ def getIntegratedAnalyzeContext( projectId=projectId, protocolId=protocolId, outputName=outputName, + mapper=mapper, ) resp = JSONResponse(payload) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 09cda88e..8e093a43 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2341,11 +2341,383 @@ def buildProtocolOutputThumbnailUrl(projectId: int, protocolId: int, outputName: # buildProtocolOutputThumbnailUrl return f"/projects/{projectId}/protocols/{protocolId}/outputs/{outputName}/thumbnail" + def _getPostgresqlStoredSetForIntegratedContext( + self, + mapper, + projectId: int, + protocolId: int, + outputName: str, + ) -> Optional[Dict[str, Any]]: + if mapper is None: + return None + + try: + from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + setMapper = ScipionSetPostgresqlMapper(mapper.db) + return setMapper.getStoredSet( + projectId=projectId, + protocolDbId=protocolId, + outputName=outputName, + limit=None, + offset=0, + ) + except Exception: + logger.debug( + "Could not load PostgreSQL stored set for integrated context. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + exc_info=True, + ) + return None + + def _getPostgresqlIntegratedKind(self, storedSet: Dict[str, Any]) -> Optional[str]: + classText = "%s %s" % ( + storedSet.get("setClassName") or "", + storedSet.get("itemClassName") or "", + ) + classText = classText.replace(" ", "").lower() + + if "setofcoordinates3d" in classText or "coordinate3d" in classText: + return "coordinates3d" + + if "setofctftomoseries" in classText or "ctftomoseries" in classText: + return "ctf" + + if "setoftiltseries" in classText or "tiltseries" in classText: + return "tiltSeries" + + if "setoftomograms" in classText or "tomogram" in classText: + return "tomogram" + + return None + + def _getPostgresqlStoredSetProperty( + self, + storedSet: Dict[str, Any], + key: str, + default=None, + ): + properties = storedSet.get("properties") or {} + + if isinstance(properties, str): + try: + properties = json.loads(properties) + except Exception: + properties = {} + + if isinstance(properties, dict) and key in properties: + return properties.get(key) + + for item in storedSet.get("setProperties") or []: + if str(item.get("key")) == str(key): + return item.get("value") + + return default + + def _buildPostgresqlIntegratedLink( + self, + projectId: int, + protocolId: int, + outputName: str, + storedSet: Dict[str, Any], + label: Optional[str] = None, + statusValue: str = "available", + ) -> Dict[str, Any]: + return { + "protocolId": protocolId, + "outputName": outputName, + "itemId": storedSet.get("objectId") or storedSet.get("id"), + "label": label or outputName, + "status": statusValue, + } + + def _buildPostgresqlIntegratedSummary( + self, + storedSet: Dict[str, Any], + extra: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + items = storedSet.get("items") or [] + itemsCount = self._getPostgresqlStoredSetProperty(storedSet, "itemsCount", None) + + if itemsCount is None: + itemsCount = len(items) + + summary = { + "objectClass": storedSet.get("setClassName") or storedSet.get("itemClassName"), + "objectId": storedSet.get("objectId") or storedSet.get("id"), + "size": itemsCount, + "fileName": self._getPostgresqlStoredSetProperty(storedSet, "fileName", None), + "samplingRate": self._getPostgresqlStoredSetProperty(storedSet, "samplingRate", None), + } + + if extra: + summary.update(extra) + + return self._safeScipionValue(summary) + + def _firstPostgresqlValueByName( + self, + values: Dict[str, Any], + names: List[str], + ): + normalizedNames = { + str(name).replace("_", "").replace(".", "").replace("-", "").lower() + for name in names + } + + for key, value in (values or {}).items(): + normalizedKey = str(key).replace("_", "").replace(".", "").replace("-", "").lower() + if normalizedKey in normalizedNames: + return value + + return None + + def _addPostgresqlIntegratedRelation( + self, + relationsByKey: Dict[str, Dict[str, Any]], + keyValue: Any, + **values: Any, + ) -> None: + key = str(keyValue) if keyValue is not None else "" + if not key: + return + + relation = relationsByKey.setdefault( + key, + { + "key": key, + "label": key, + }, + ) + + for name, value in values.items(): + if value is not None: + relation[name] = value + + def _addPostgresqlTiltSeriesRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tiltSeriesId = item.get("tiltSeriesId") or item.get("id") or index + label = item.get("label") or str(tiltSeriesId) + + self._addPostgresqlIntegratedRelation( + relationsByKey, + tiltSeriesId, + tiltSeriesId=tiltSeriesId, + label=label, + ) + + def _addPostgresqlCtftomoRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tiltSeriesId = item.get("tiltSeriesId") or item.get("id") or index + label = item.get("label") or str(tiltSeriesId) + + self._addPostgresqlIntegratedRelation( + relationsByKey, + tiltSeriesId, + ctfSeriesId=tiltSeriesId, + tiltSeriesId=tiltSeriesId, + label=label, + ) + + def _addPostgresqlTomogramRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tomogramId = ( + item.get("tomoId") + or item.get("tomogramId") + or item.get("id") + or item.get("label") + or index + ) + + label = item.get("label") or item.get("name") or str(tomogramId) + volumeId = item.get("tomogramVolumeId") or item.get("volumeId") or index + + self._addPostgresqlIntegratedRelation( + relationsByKey, + tomogramId, + tomogramId=tomogramId, + tomogramVolumeId=volumeId, + label=label, + ) + + def _addPostgresqlCoordinates3dRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tomogramId = ( + item.get("tomoId") + or item.get("tomogramId") + or item.get("id") + or item.get("label") + or index + ) + + label = item.get("label") or item.get("name") or str(tomogramId) + volumeId = item.get("tomogramVolumeId") or item.get("volumeId") or index + + self._addPostgresqlIntegratedRelation( + relationsByKey, + tomogramId, + coordinatesTomogramId=tomogramId, + tomogramId=tomogramId, + tomogramVolumeId=volumeId, + label=label, + ) + + def _getPostgresqlIntegratedAnalyzeContextIfAvailable( + self, + mapper, + projectId: int, + protocolId: int, + outputName: str, + ) -> Optional[Dict[str, Any]]: + storedSet = self._getPostgresqlStoredSetForIntegratedContext( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if storedSet is None: + return None + + rootKind = self._getPostgresqlIntegratedKind(storedSet) + if rootKind is None: + return None + + links = { + "tiltSeries": None, + "ctf": None, + "tomogram": None, + "coordinates3d": None, + } + summaries = { + "tiltSeries": None, + "ctf": None, + "tomogram": None, + "coordinates3d": None, + } + relationsByKey: Dict[str, Dict[str, Any]] = {} + + rootLink = self._buildPostgresqlIntegratedLink( + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + storedSet=storedSet, + ) + + links[rootKind] = rootLink + summaries[rootKind] = self._buildPostgresqlIntegratedSummary(storedSet) + + if rootKind == "tiltSeries": + pgReader = self._getPostgresqlTiltSeriesReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + items = pgReader.listTiltSeries() if pgReader is not None else [] + self._addPostgresqlTiltSeriesRelations(relationsByKey, items) + + elif rootKind == "ctf": + pgReader = self._getPostgresqlCtftomoReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + items = pgReader.listCtftomoSeries() if pgReader is not None else [] + self._addPostgresqlCtftomoRelations(relationsByKey, items) + + elif rootKind == "coordinates3d": + pgReader = self._getPostgresqlCoords3dReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + tomograms = pgReader.listTomograms() if pgReader is not None else [] + self._addPostgresqlCoordinates3dRelations(relationsByKey, tomograms) + + if tomograms: + links["tomogram"] = { + "protocolId": None, + "outputName": None, + "itemId": None, + "label": "Tomograms", + "status": "inferred", + } + summaries["tomogram"] = self._safeScipionValue( + { + "objectClass": "SetOfTomograms", + "size": len(tomograms), + } + ) + + elif rootKind == "tomogram": + items = [] + for index, item in enumerate(storedSet.get("items") or []): + values = item.get("values") or {} + + tomogramId = self._firstPostgresqlValueByName( + values, + ["_tomoId", "tomoId", "tomogramId", "_tsId", "tsId"], + ) + + label = self._firstPostgresqlValueByName( + values, + ["_objLabel", "label", "_tsId", "tsId", "tomoId"], + ) + + items.append( + { + "id": tomogramId or item.get("scipionItemId") or index, + "tomoId": tomogramId or item.get("scipionItemId") or index, + "label": label or tomogramId or item.get("scipionItemId") or index, + "volumeId": index, + } + ) + + self._addPostgresqlTomogramRelations(relationsByKey, items) + + return { + "root": { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "outputClass": storedSet.get("setClassName") or storedSet.get("itemClassName"), + }, + "links": links, + "summaries": summaries, + "relations": self._safeScipionValue( + { + "items": list(relationsByKey.values()), + } + ), + } + def getIntegratedAnalyzeContextService( self, projectId: int, protocolId: int, outputName: str, + mapper=None, ) -> Dict[str, Any]: if self.currentProject is None: raise HTTPException( @@ -2353,6 +2725,16 @@ def getIntegratedAnalyzeContextService( detail="No current project loaded", ) + pgContext = self._getPostgresqlIntegratedAnalyzeContextIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgContext is not None: + return pgContext + try: protocol = self.currentProject.getProtocol(int(protocolId)) except Exception as e: From 7d0d7b8786de12ae273de0d8afad9cb164b20843 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 09:19:30 +0200 Subject: [PATCH 064/278] move postgresql integrated context to viewer reader --- app/backend/api/services/project_service.py | 130 +--- .../postgresql_integrated_context_reader.py | 681 ++++++++++++++++++ 2 files changed, 695 insertions(+), 116 deletions(-) create mode 100644 app/backend/viewers/postgresql_integrated_context_reader.py diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 8e093a43..b5414d6c 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2587,130 +2587,28 @@ def _getPostgresqlIntegratedAnalyzeContextIfAvailable( protocolId: int, outputName: str, ) -> Optional[Dict[str, Any]]: - storedSet = self._getPostgresqlStoredSetForIntegratedContext( - mapper=mapper, - projectId=projectId, - protocolId=protocolId, - outputName=outputName, - ) - - if storedSet is None: - return None - - rootKind = self._getPostgresqlIntegratedKind(storedSet) - if rootKind is None: + if mapper is None: return None - links = { - "tiltSeries": None, - "ctf": None, - "tomogram": None, - "coordinates3d": None, - } - summaries = { - "tiltSeries": None, - "ctf": None, - "tomogram": None, - "coordinates3d": None, - } - relationsByKey: Dict[str, Dict[str, Any]] = {} - - rootLink = self._buildPostgresqlIntegratedLink( - projectId=projectId, - protocolId=protocolId, - outputName=outputName, - storedSet=storedSet, - ) - - links[rootKind] = rootLink - summaries[rootKind] = self._buildPostgresqlIntegratedSummary(storedSet) - - if rootKind == "tiltSeries": - pgReader = self._getPostgresqlTiltSeriesReaderIfAvailable( - mapper=mapper, - projectId=projectId, - protocolId=protocolId, - outputName=outputName, - ) - items = pgReader.listTiltSeries() if pgReader is not None else [] - self._addPostgresqlTiltSeriesRelations(relationsByKey, items) + try: + from app.backend.viewers.postgresql_integrated_context_reader import PostgresqlIntegratedContextReader - elif rootKind == "ctf": - pgReader = self._getPostgresqlCtftomoReaderIfAvailable( - mapper=mapper, + reader = PostgresqlIntegratedContextReader( + db=mapper.db, projectId=projectId, protocolId=protocolId, outputName=outputName, ) - items = pgReader.listCtftomoSeries() if pgReader is not None else [] - self._addPostgresqlCtftomoRelations(relationsByKey, items) - - elif rootKind == "coordinates3d": - pgReader = self._getPostgresqlCoords3dReaderIfAvailable( - mapper=mapper, - projectId=projectId, - protocolId=protocolId, - outputName=outputName, + return reader.getContext() + except Exception: + logger.debug( + "Could not build PostgreSQL integrated analyze context. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + exc_info=True, ) - tomograms = pgReader.listTomograms() if pgReader is not None else [] - self._addPostgresqlCoordinates3dRelations(relationsByKey, tomograms) - - if tomograms: - links["tomogram"] = { - "protocolId": None, - "outputName": None, - "itemId": None, - "label": "Tomograms", - "status": "inferred", - } - summaries["tomogram"] = self._safeScipionValue( - { - "objectClass": "SetOfTomograms", - "size": len(tomograms), - } - ) - - elif rootKind == "tomogram": - items = [] - for index, item in enumerate(storedSet.get("items") or []): - values = item.get("values") or {} - - tomogramId = self._firstPostgresqlValueByName( - values, - ["_tomoId", "tomoId", "tomogramId", "_tsId", "tsId"], - ) - - label = self._firstPostgresqlValueByName( - values, - ["_objLabel", "label", "_tsId", "tsId", "tomoId"], - ) - - items.append( - { - "id": tomogramId or item.get("scipionItemId") or index, - "tomoId": tomogramId or item.get("scipionItemId") or index, - "label": label or tomogramId or item.get("scipionItemId") or index, - "volumeId": index, - } - ) - - self._addPostgresqlTomogramRelations(relationsByKey, items) - - return { - "root": { - "projectId": projectId, - "protocolId": protocolId, - "outputName": outputName, - "outputClass": storedSet.get("setClassName") or storedSet.get("itemClassName"), - }, - "links": links, - "summaries": summaries, - "relations": self._safeScipionValue( - { - "items": list(relationsByKey.values()), - } - ), - } + return None def getIntegratedAnalyzeContextService( self, diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py new file mode 100644 index 00000000..fd76f74a --- /dev/null +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -0,0 +1,681 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +import json +import logging +from datetime import date, datetime +from typing import Any, Dict, List, Optional + +from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper +from app.backend.viewers.postgresql_coords3d_reader import PostgresqlCoords3dReader +from app.backend.viewers.postgresql_ctftomo_reader import PostgresqlCtftomoReader +from app.backend.viewers.postgresql_tiltseries_reader import PostgresqlTiltSeriesReader + +logger = logging.getLogger(__name__) + + +class PostgresqlIntegratedContextReader: + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + self.setMapper = ScipionSetPostgresqlMapper(db) + self._rootStoredSet = None + self.lastSkipReason = None + + def hasOutput(self) -> bool: + storedSet = self._getRootStoredSet() + return storedSet is not None and self._getIntegratedKind(storedSet) is not None + + def getContext(self) -> Optional[Dict[str, Any]]: + self.lastSkipReason = None + + storedSet = self._getRootStoredSet() + if storedSet is None: + self.lastSkipReason = "stored_set_not_found" + return None + + rootKind = self._getIntegratedKind(storedSet) + if rootKind is None: + self.lastSkipReason = "unsupported_integrated_context_kind" + return None + + links = { + "tiltSeries": None, + "ctf": None, + "tomogram": None, + "coordinates3d": None, + } + summaries = { + "tiltSeries": None, + "ctf": None, + "tomogram": None, + "coordinates3d": None, + } + relationsByKey: Dict[str, Dict[str, Any]] = {} + + links[rootKind] = self._buildLink( + protocolId=self.protocolId, + outputName=self.outputName, + storedSet=storedSet, + statusValue="available", + ) + summaries[rootKind] = self._buildSummary(storedSet) + + self._mergeRootRelations( + rootKind=rootKind, + storedSet=storedSet, + relationsByKey=relationsByKey, + links=links, + summaries=summaries, + ) + + self._mergeRelatedStoredSets( + rootStoredSet=storedSet, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + return { + "root": { + "projectId": self.projectId, + "protocolId": self.protocolId, + "outputName": self.outputName, + "outputClass": storedSet.get("setClassName") or storedSet.get("itemClassName"), + }, + "links": self._safeValue(links), + "summaries": self._safeValue(summaries), + "relations": self._safeValue( + { + "items": list(relationsByKey.values()), + } + ), + } + + def _getRootStoredSet(self) -> Optional[Dict[str, Any]]: + if self._rootStoredSet is None: + try: + self._rootStoredSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=self.protocolId, + outputName=self.outputName, + limit=None, + offset=0, + ) + except Exception: + logger.debug( + "Could not load PostgreSQL root stored set for integrated context. " + "projectId=%s protocolId=%s outputName=%s", + self.projectId, + self.protocolId, + self.outputName, + exc_info=True, + ) + self._rootStoredSet = None + + return self._rootStoredSet + + def _getIntegratedKind(self, storedSet: Dict[str, Any]) -> Optional[str]: + classText = "%s %s" % ( + storedSet.get("setClassName") or "", + storedSet.get("itemClassName") or "", + ) + classText = classText.replace(" ", "").lower() + + if "setofcoordinates3d" in classText or "coordinate3d" in classText: + return "coordinates3d" + + if "setofctftomoseries" in classText or "ctftomoseries" in classText: + return "ctf" + + if "setoftiltseries" in classText or "tiltseries" in classText: + return "tiltSeries" + + if "setoftomograms" in classText or "tomogram" in classText: + return "tomogram" + + return None + + def _getStoredSetProperty( + self, + storedSet: Dict[str, Any], + key: str, + default=None, + ): + properties = storedSet.get("properties") or {} + + if isinstance(properties, str): + try: + properties = json.loads(properties) + except Exception: + properties = {} + + if isinstance(properties, dict) and key in properties: + return properties.get(key) + + for item in storedSet.get("setProperties") or []: + if str(item.get("key")) == str(key): + return item.get("value") + + return default + + def _buildLink( + self, + protocolId: Any, + outputName: Any, + storedSet: Dict[str, Any], + label: Optional[str] = None, + statusValue: str = "available", + ) -> Dict[str, Any]: + return { + "protocolId": protocolId, + "outputName": outputName, + "itemId": storedSet.get("objectId") or storedSet.get("id"), + "label": label or outputName, + "status": statusValue, + } + + def _buildSummary( + self, + storedSet: Dict[str, Any], + extra: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + items = storedSet.get("items") or [] + itemsCount = self._getStoredSetProperty(storedSet, "itemsCount", None) + + if itemsCount is None: + itemsCount = len(items) + + summary = { + "objectClass": storedSet.get("setClassName") or storedSet.get("itemClassName"), + "objectId": storedSet.get("objectId") or storedSet.get("id"), + "size": itemsCount, + "fileName": self._getStoredSetProperty(storedSet, "fileName", None), + "samplingRate": self._getStoredSetProperty(storedSet, "samplingRate", None), + } + + if extra: + summary.update(extra) + + return summary + + def _firstValueByName( + self, + values: Dict[str, Any], + names: List[str], + ): + normalizedNames = { + self._normalizeName(name) + for name in names + } + + for key, value in (values or {}).items(): + if self._normalizeName(key) in normalizedNames: + return value + + return None + + def _normalizeName(self, value: Any) -> str: + return str(value).replace("_", "").replace(".", "").replace("-", "").lower() + + def _addRelation( + self, + relationsByKey: Dict[str, Dict[str, Any]], + keyValue: Any, + **values: Any, + ) -> None: + key = str(keyValue) if keyValue is not None else "" + if not key: + return + + relation = relationsByKey.setdefault( + key, + { + "key": key, + "label": key, + }, + ) + + for name, value in values.items(): + if value is not None: + relation[name] = value + + def _addTiltSeriesRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tiltSeriesId = item.get("tiltSeriesId") or item.get("id") or index + label = item.get("label") or str(tiltSeriesId) + + self._addRelation( + relationsByKey, + tiltSeriesId, + tiltSeriesId=tiltSeriesId, + label=label, + ) + + def _addCtftomoRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tiltSeriesId = item.get("tiltSeriesId") or item.get("id") or index + label = item.get("label") or str(tiltSeriesId) + + self._addRelation( + relationsByKey, + tiltSeriesId, + ctfSeriesId=tiltSeriesId, + tiltSeriesId=tiltSeriesId, + label=label, + ) + + def _addTomogramRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tomogramId = ( + item.get("tomoId") + or item.get("tomogramId") + or item.get("id") + or item.get("label") + or index + ) + + label = item.get("label") or item.get("name") or str(tomogramId) + volumeId = item.get("tomogramVolumeId") or item.get("volumeId") or index + + self._addRelation( + relationsByKey, + tomogramId, + tomogramId=tomogramId, + tomogramVolumeId=volumeId, + label=label, + ) + + def _addCoordinates3dRelations( + self, + relationsByKey: Dict[str, Dict[str, Any]], + items: List[Dict[str, Any]], + ) -> None: + for index, item in enumerate(items or []): + tomogramId = ( + item.get("tomoId") + or item.get("tomogramId") + or item.get("id") + or item.get("label") + or index + ) + + label = item.get("label") or item.get("name") or str(tomogramId) + volumeId = item.get("tomogramVolumeId") or item.get("volumeId") or index + + self._addRelation( + relationsByKey, + tomogramId, + coordinatesTomogramId=tomogramId, + tomogramId=tomogramId, + tomogramVolumeId=volumeId, + label=label, + ) + + def _mergeRootRelations( + self, + rootKind: str, + storedSet: Dict[str, Any], + relationsByKey: Dict[str, Dict[str, Any]], + links: Dict[str, Optional[Dict[str, Any]]], + summaries: Dict[str, Optional[Dict[str, Any]]], + ) -> None: + if rootKind == "tiltSeries": + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=self.protocolId, + outputName=self.outputName, + ) + self._addTiltSeriesRelations(relationsByKey, reader.listTiltSeries()) + return + + if rootKind == "ctf": + reader = PostgresqlCtftomoReader( + db=self.db, + projectId=self.projectId, + protocolId=self.protocolId, + outputName=self.outputName, + ) + self._addCtftomoRelations(relationsByKey, reader.listCtftomoSeries()) + return + + if rootKind == "coordinates3d": + reader = PostgresqlCoords3dReader( + db=self.db, + projectId=self.projectId, + protocolId=self.protocolId, + outputName=self.outputName, + ) + tomograms = reader.listTomograms() or [] + self._addCoordinates3dRelations(relationsByKey, tomograms) + + if tomograms: + links["tomogram"] = { + "protocolId": None, + "outputName": None, + "itemId": None, + "label": "Tomograms", + "status": "inferred", + } + summaries["tomogram"] = { + "objectClass": "SetOfTomograms", + "size": len(tomograms), + } + return + + if rootKind == "tomogram": + items = self._buildTomogramItemsFromStoredSet(storedSet) + self._addTomogramRelations(relationsByKey, items) + + def _buildTomogramItemsFromStoredSet( + self, + storedSet: Dict[str, Any], + ) -> List[Dict[str, Any]]: + items = [] + + for index, item in enumerate(storedSet.get("items") or []): + values = item.get("values") or {} + + tomogramId = self._firstValueByName( + values, + ["_tomoId", "tomoId", "tomogramId", "_tsId", "tsId"], + ) + + label = self._firstValueByName( + values, + ["_objLabel", "label", "_tsId", "tsId", "tomoId"], + ) + + publicId = tomogramId or item.get("scipionItemId") or index + + items.append( + { + "id": publicId, + "tomoId": publicId, + "label": label or publicId, + "volumeId": index, + } + ) + + return items + + def _listRelatedStoredSets(self) -> List[Dict[str, Any]]: + try: + rows = self.db.fetchAll( + """ + WITH root_protocol AS ( + SELECT id, "projectId", "protocolId", "protocolClassName" + FROM protocols + WHERE "projectId" = %s + AND (id = %s OR "protocolId" = %s) + LIMIT 1 + ), + related_protocols AS ( + SELECT + id, + "projectId", + "protocolId", + "protocolClassName", + 'root' AS "relationRole", + 0 AS distance + FROM root_protocol + + UNION ALL + + SELECT + parent.id, + parent."projectId", + parent."protocolId", + parent."protocolClassName", + 'parent' AS "relationRole", + 1 AS distance + FROM protocol_dependencies d + JOIN root_protocol root + ON root.id = d."childProtocolDbId" + AND root."projectId" = d."projectId" + JOIN protocols parent + ON parent.id = d."parentProtocolDbId" + AND parent."projectId" = d."projectId" + + UNION ALL + + SELECT + child.id, + child."projectId", + child."protocolId", + child."protocolClassName", + 'child' AS "relationRole", + 1 AS distance + FROM protocol_dependencies d + JOIN root_protocol root + ON root.id = d."parentProtocolDbId" + AND root."projectId" = d."projectId" + JOIN protocols child + ON child.id = d."childProtocolDbId" + AND child."projectId" = d."projectId" + ) + SELECT DISTINCT ON (s.id) + s.id, + s."projectId", + s."protocolDbId", + s."objectId", + s."outputName", + s."setClassName", + s."itemClassName", + s.properties, + s."createdAt", + s."updatedAt", + rp."protocolId" AS "publicProtocolId", + rp."protocolClassName", + rp."relationRole", + rp.distance + FROM related_protocols rp + JOIN scipion_sets s + ON s."projectId" = rp."projectId" + AND s."protocolDbId" = rp.id + ORDER BY s.id, rp.distance ASC, rp."relationRole" ASC + """, + ( + self.projectId, + int(self.protocolId), + str(self.protocolId), + ), + ) + except Exception: + logger.debug( + "Could not list related PostgreSQL stored sets for integrated context. " + "projectId=%s protocolId=%s outputName=%s", + self.projectId, + self.protocolId, + self.outputName, + exc_info=True, + ) + return [] + + result = [dict(row) for row in rows or []] + result.sort( + key=lambda item: ( + int(item.get("distance") or 0), + str(item.get("relationRole") or ""), + int(item.get("protocolDbId") or 0), + str(item.get("outputName") or ""), + ) + ) + return result + + def _mergeRelatedStoredSets( + self, + rootStoredSet: Dict[str, Any], + links: Dict[str, Optional[Dict[str, Any]]], + summaries: Dict[str, Optional[Dict[str, Any]]], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> None: + for candidate in self._listRelatedStoredSets(): + if self._isSameStoredSet(candidate, rootStoredSet): + continue + + candidateKind = self._getIntegratedKind(candidate) + if candidateKind is None or candidateKind not in links: + continue + + if self._shouldReplaceLink(links.get(candidateKind)): + links[candidateKind] = self._buildLink( + protocolId=self._getCandidateProtocolId(candidate), + outputName=candidate.get("outputName"), + storedSet=candidate, + statusValue="related", + ) + summaries[candidateKind] = self._buildSummary(candidate) + + self._mergeRelationsForCandidate( + candidate=candidate, + candidateKind=candidateKind, + relationsByKey=relationsByKey, + ) + + def _mergeRelationsForCandidate( + self, + candidate: Dict[str, Any], + candidateKind: str, + relationsByKey: Dict[str, Dict[str, Any]], + ) -> None: + protocolDbId = candidate.get("protocolDbId") + outputName = candidate.get("outputName") + + if protocolDbId is None or not outputName: + return + + if candidateKind == "tiltSeries": + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + self._addTiltSeriesRelations(relationsByKey, reader.listTiltSeries()) + return + + if candidateKind == "ctf": + reader = PostgresqlCtftomoReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + self._addCtftomoRelations(relationsByKey, reader.listCtftomoSeries()) + return + + if candidateKind == "coordinates3d": + reader = PostgresqlCoords3dReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + self._addCoordinates3dRelations(relationsByKey, reader.listTomograms() or []) + return + + if candidateKind == "tomogram": + storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=protocolDbId, + outputName=outputName, + limit=None, + offset=0, + ) + + if storedSet is None: + return + + items = self._buildTomogramItemsFromStoredSet(storedSet) + self._addTomogramRelations(relationsByKey, items) + + def _isSameStoredSet( + self, + left: Dict[str, Any], + right: Dict[str, Any], + ) -> bool: + return ( + str(left.get("protocolDbId")) == str(right.get("protocolDbId")) + and str(left.get("outputName")) == str(right.get("outputName")) + ) + + def _shouldReplaceLink( + self, + existingLink: Optional[Dict[str, Any]], + ) -> bool: + if existingLink is None: + return True + + protocolId = existingLink.get("protocolId") + outputName = existingLink.get("outputName") + statusValue = str(existingLink.get("status") or "") + + if protocolId is None and outputName is None: + return True + + return statusValue == "inferred" and protocolId is None + + def _getCandidateProtocolId(self, candidate: Dict[str, Any]) -> Any: + return candidate.get("publicProtocolId") or candidate.get("protocolDbId") + + def _safeValue(self, value: Any) -> Any: + if isinstance(value, dict): + return { + str(key): self._safeValue(item) + for key, item in value.items() + } + + if isinstance(value, (list, tuple, set)): + return [ + self._safeValue(item) + for item in value + ] + + if isinstance(value, (datetime, date)): + return value.isoformat() + + if hasattr(value, "item"): + try: + return value.item() + except Exception: + pass + + return value \ No newline at end of file From 9cacaab2d3fd062b71607b1d591ff1f5a7a99423 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 09:46:34 +0200 Subject: [PATCH 065/278] filter related integrated context relations --- .../postgresql_integrated_context_reader.py | 57 ++++++++++++++++++- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index fd76f74a..ce599ae1 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -564,17 +564,52 @@ def _mergeRelatedStoredSets( ) summaries[candidateKind] = self._buildSummary(candidate) + allowedRelationKeys = None + rootKind = self._getIntegratedKind(rootStoredSet) + + if rootKind == "coordinates3d": + allowedRelationKeys = set(relationsByKey.keys()) + self._mergeRelationsForCandidate( candidate=candidate, candidateKind=candidateKind, relationsByKey=relationsByKey, + allowedRelationKeys=allowedRelationKeys, ) + def _filterIntegratedItemsByAllowedKeys( + self, + items: List[Dict[str, Any]], + allowedRelationKeys: Optional[set], + ) -> List[Dict[str, Any]]: + if not allowedRelationKeys: + return items or [] + + filteredItems = [] + + for item in items or []: + candidates = [ + item.get("key"), + item.get("label"), + item.get("id"), + item.get("tomoId"), + item.get("tomogramId"), + item.get("tiltSeriesId"), + item.get("ctfSeriesId"), + item.get("coordinatesTomogramId"), + ] + + if any(str(value) in allowedRelationKeys for value in candidates if value is not None): + filteredItems.append(item) + + return filteredItems + def _mergeRelationsForCandidate( self, candidate: Dict[str, Any], candidateKind: str, relationsByKey: Dict[str, Dict[str, Any]], + allowedRelationKeys: Optional[set] = None, ) -> None: protocolDbId = candidate.get("protocolDbId") outputName = candidate.get("outputName") @@ -589,6 +624,11 @@ def _mergeRelationsForCandidate( protocolId=protocolDbId, outputName=outputName, ) + items = self._filterIntegratedItemsByAllowedKeys( + reader.listTiltSeries(), + allowedRelationKeys, + ) + self._addTiltSeriesRelations(relationsByKey, items) self._addTiltSeriesRelations(relationsByKey, reader.listTiltSeries()) return @@ -599,7 +639,11 @@ def _mergeRelationsForCandidate( protocolId=protocolDbId, outputName=outputName, ) - self._addCtftomoRelations(relationsByKey, reader.listCtftomoSeries()) + items = self._filterIntegratedItemsByAllowedKeys( + reader.listCtftomoSeries(), + allowedRelationKeys, + ) + self._addCtftomoRelations(relationsByKey, items) return if candidateKind == "coordinates3d": @@ -609,7 +653,11 @@ def _mergeRelationsForCandidate( protocolId=protocolDbId, outputName=outputName, ) - self._addCoordinates3dRelations(relationsByKey, reader.listTomograms() or []) + items = self._filterIntegratedItemsByAllowedKeys( + reader.listTomograms() or [], + allowedRelationKeys, + ) + self._addCoordinates3dRelations(relationsByKey, items) return if candidateKind == "tomogram": @@ -624,7 +672,10 @@ def _mergeRelationsForCandidate( if storedSet is None: return - items = self._buildTomogramItemsFromStoredSet(storedSet) + items = self._filterIntegratedItemsByAllowedKeys( + self._buildTomogramItemsFromStoredSet(storedSet), + allowedRelationKeys, + ) self._addTomogramRelations(relationsByKey, items) def _isSameStoredSet( From e28d706d6939e39ef4363d068cb089391e0b3841 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 10:00:24 +0200 Subject: [PATCH 066/278] prefer volume data files over tomostar metadata --- .../viewers/postgresql_volume_reader.py | 131 +++++++++++++++++- 1 file changed, 128 insertions(+), 3 deletions(-) diff --git a/app/backend/viewers/postgresql_volume_reader.py b/app/backend/viewers/postgresql_volume_reader.py index d479cb71..eb5c3f08 100644 --- a/app/backend/viewers/postgresql_volume_reader.py +++ b/app/backend/viewers/postgresql_volume_reader.py @@ -7,6 +7,30 @@ from app.backend.utils.volume_utils import readVolumeArray3d +VOLUME_DATA_EXTENSIONS = ( + ".mrc", + ".mrcs", + ".map", + ".rec", + ".em", + ".spi", + ".vol", + ".hdf", + ".h5", +) + +METADATA_LIKE_VOLUME_EXTENSIONS = ( + ".tomostar", + ".star", + ".xmd", + ".sqlite", + ".db", + ".json", + ".xml", + ".txt", + ".csv", + ".tsv", +) class PostgresqlVolumeReader: """Read Volume, VolumeMask and SetOfVolumes outputs from PostgreSQL.""" @@ -163,6 +187,9 @@ def _buildVolumeFromSetItem( fileName, locationIndex = self._extractVolumeFile(values) scipionItemId = item.get("scipionItemId") + if not fileName: + return None + rawLabel = ( item.get("label") or self._firstValueBySuffix(values, ["objLabel", "label", "name", "volName"]) @@ -440,18 +467,116 @@ def _resolveProtocolDbId(self) -> Optional[int]: return None def _extractVolumeFile(self, values: Dict[str, Any]) -> Tuple[Optional[str], Optional[int]]: - raw = self._firstValueBySuffix( - values, + candidates: List[Tuple[str, Optional[int]]] = [] + + priorityGroups = [ + [ + "tomogramFileName", + "tomogramFilename", + "tomogramFile", + "tomogramFilePath", + "tomogramPath", + "tomoFileName", + "tomoFilename", + "tomoFile", + "tomoPath", + ], + [ + "volumeFileName", + "volumeFilename", + "volumeFile", + "volumeFilePath", + "volumePath", + "mapFileName", + "mapFilename", + "mapFile", + "mrcFileName", + "mrcFilename", + "mrcFile", + "recFileName", + "recFilename", + "recFile", + ], [ "fileName", "filename", + "filePath", + "filepath", "location", "path", "stack", ], + [ + "volName", + "volumeName", + "tomoName", + "tomogramName", + ], + ] + + for suffixes in priorityGroups: + for raw in self._valuesBySuffix(values, suffixes): + fileName, locationIndex = self._parseLocation(raw) + if not fileName: + continue + + candidates.append((str(fileName), locationIndex)) + + for fileName, locationIndex in candidates: + if self._isVolumeDataPath(fileName): + return fileName, locationIndex + + for fileName, locationIndex in candidates: + if not self._isMetadataLikeVolumePath(fileName): + return fileName, locationIndex + + return None, None + + def _valuesBySuffix(self, values: Dict[str, Any], suffixes: List[str]) -> List[Any]: + normalizedSuffixes = [ + self._normalizeKey(suffix) + for suffix in suffixes + ] + + result = [] + seen = set() + + for key, value in values.items(): + if value is None: + continue + + normalizedKey = self._normalizeKey(key) + if not any(normalizedKey.endswith(suffix) for suffix in normalizedSuffixes): + continue + + marker = repr(value) + if marker in seen: + continue + + seen.add(marker) + result.append(value) + + return result + + def _isVolumeDataPath(self, fileName: Any) -> bool: + text = str(fileName or "").strip().lower() + if not text: + return False + + return any( + text.endswith(extension) or ("%s:" % extension) in text + for extension in VOLUME_DATA_EXTENSIONS ) - return self._parseLocation(raw) + def _isMetadataLikeVolumePath(self, fileName: Any) -> bool: + text = str(fileName or "").strip().lower() + if not text: + return False + + return any( + text.endswith(extension) + for extension in METADATA_LIKE_VOLUME_EXTENSIONS + ) def _parseLocation(self, raw: Any) -> Tuple[Optional[str], Optional[int]]: parsed = self._parseJsonValue(raw) From 310e00bcf7290561fc23cb6350fef765e6bf8832 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 10:16:25 +0200 Subject: [PATCH 067/278] serve integrated tomograms from coords3d context --- app/backend/api/services/project_service.py | 23 + ...tgresql_coords3d_tomogram_volume_reader.py | 470 ++++++++++++++++++ .../postgresql_integrated_context_reader.py | 13 +- 3 files changed, 503 insertions(+), 3 deletions(-) create mode 100644 app/backend/viewers/postgresql_coords3d_tomogram_volume_reader.py diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index b5414d6c..03dce8dc 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6860,6 +6860,29 @@ def _getPostgresqlVolumeReaderIfAvailable( if mapper is None: return None + try: + from app.backend.viewers.postgresql_coords3d_tomogram_volume_reader import \ + PostgresqlCoords3dTomogramVolumeReader + + reader = PostgresqlCoords3dTomogramVolumeReader( + db=mapper.db, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if reader.hasOutput(): + return reader + + except Exception: + logger.debug( + "PostgreSQL Coords3D-derived tomogram volume reader is not available. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + exc_info=True, + ) + try: from app.backend.viewers.postgresql_volume_reader import PostgresqlVolumeReader diff --git a/app/backend/viewers/postgresql_coords3d_tomogram_volume_reader.py b/app/backend/viewers/postgresql_coords3d_tomogram_volume_reader.py new file mode 100644 index 00000000..c909fd26 --- /dev/null +++ b/app/backend/viewers/postgresql_coords3d_tomogram_volume_reader.py @@ -0,0 +1,470 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +import os +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np + +from app.backend.utils.volume_utils import readVolumeArray3d +from app.backend.viewers.postgresql_coords3d_reader import PostgresqlCoords3dReader + + +class PostgresqlCoords3dTomogramVolumeReader: + """Expose tomograms resolved from a SetOfCoordinates3D as volume-like items.""" + + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = int(projectId) + self.protocolId = int(protocolId) + self.outputName = str(outputName) + self.coordsReader = PostgresqlCoords3dReader( + db=db, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + self.lastSkipReason = None + self._volumes = None + + def hasOutput(self) -> bool: + return bool(self.listVolumes()) + + def listVolumes(self) -> Optional[List[Dict[str, Any]]]: + self.lastSkipReason = None + + if self._volumes is not None: + return self._volumes + + tomograms = self.coordsReader.listTomograms() + if not tomograms: + self.lastSkipReason = getattr(self.coordsReader, "lastSkipReason", None) or "coords3d_tomograms_not_found" + return None + + volumes = [] + + for index, tomogram in enumerate(tomograms): + volume = self._buildVolumeFromTomogram(tomogram, index) + if volume is not None: + volumes.append(volume) + + if not volumes: + self.lastSkipReason = "coords3d_tomograms_without_volume_files" + return None + + self._volumes = volumes + return self._volumes + + def getVolumeInfo(self, volumeId: Union[int, str]) -> Optional[Dict[str, Any]]: + self.lastSkipReason = None + + volumes = self.listVolumes() + if not volumes: + self.lastSkipReason = self.lastSkipReason or "volume_list_empty" + return None + + volume = self._findVolume(volumeId, volumes) + if volume is None: + self.lastSkipReason = "volume_not_found volumeId=%s" % str(volumeId) + return None + + info = dict(volume) + volumeFile = self.getVolumeFile(volume.get("id")) + if volumeFile is None: + return info + + info.update(volumeFile) + self._ensureVolumeInfoFromFile(info) + return info + + def getVolumeFile(self, volumeId: Union[int, str]) -> Optional[Dict[str, Any]]: + volumes = self.listVolumes() + if not volumes: + self.lastSkipReason = self.lastSkipReason or "volume_list_empty" + return None + + volume = self._findVolume(volumeId, volumes) + if volume is None: + self.lastSkipReason = "volume_not_found volumeId=%s" % str(volumeId) + return None + + info = dict(volume) + fileName = info.get("fileName") or info.get("path") + if not fileName: + self.lastSkipReason = "volume_file_not_found volumeId=%s" % str(volumeId) + return None + + resolvedPath = self._resolveExistingPath(fileName) + if resolvedPath is None: + self.lastSkipReason = "volume_file_missing fileName=%s" % str(fileName) + return None + + info["fileName"] = resolvedPath + info["path"] = resolvedPath + return info + + def getVolumeArray( + self, + volumeId: Union[int, str], + ) -> Optional[Tuple[np.ndarray, Dict[str, Any], Dict[str, Any]]]: + info = self.getVolumeFile(volumeId) + if info is None: + return None + + volumePath = info.get("fileName") or info.get("path") + if not volumePath: + self.lastSkipReason = "volume_file_not_found volumeId=%s" % str(volumeId) + return None + + try: + array, props = readVolumeArray3d(str(volumePath)) + except Exception as exc: + self.lastSkipReason = "volume_read_failed volumeId=%s error=%s" % ( + str(volumeId), + str(exc), + ) + return None + + props = props if isinstance(props, dict) else {} + return array, props, info + + def getHistogram( + self, + volumeId: Union[int, str], + bins: int = 128, + ) -> Optional[Dict[str, Any]]: + result = self.getVolumeArray(volumeId) + if result is None: + return None + + array, _props, _info = result + cleanArray = np.asarray(array, dtype=np.float32) + cleanArray = cleanArray[np.isfinite(cleanArray)] + + if cleanArray.size == 0: + return { + "binEdges": [], + "counts": [], + } + + counts, binEdges = np.histogram(cleanArray, bins=max(4, int(bins or 128))) + + return { + "binEdges": [float(value) for value in binEdges.tolist()], + "counts": [int(value) for value in counts.tolist()], + } + + def _buildVolumeFromTomogram( + self, + tomogram: Dict[str, Any], + index: int, + ) -> Optional[Dict[str, Any]]: + fileName, locationIndex = self._extractTomogramFile(tomogram) + if not fileName: + return None + + volumeId = ( + tomogram.get("volumeId") + or tomogram.get("objectId") + or tomogram.get("id") + or tomogram.get("tomoId") + or tomogram.get("label") + or index + ) + + tomoId = tomogram.get("tomoId") or tomogram.get("id") or volumeId + label = tomogram.get("label") or tomogram.get("name") or tomoId or Path(str(fileName)).name + + volume: Dict[str, Any] = { + "id": str(volumeId), + "index": int(index), + "name": str(label), + "label": str(label), + "relPath": str(label), + "tomoId": str(tomoId), + "fileName": str(fileName), + "path": str(fileName), + "source": "coordinates3d", + } + + if locationIndex is not None: + volume["locationIndex"] = locationIndex + + for key in ( + "objectId", + "scipionItemId", + "tsId", + "tiltSeriesId", + "nCoords", + "count", + "sourceTomoId", + ): + value = tomogram.get(key) + if value is not None: + volume[key] = value + + dims = self._normalizeDims(tomogram.get("dims")) + if dims is not None: + volume["dims"] = dims + + voxelSize = self._normalizeVoxelSize(tomogram.get("voxelSize")) + if voxelSize is not None: + volume["voxelSize"] = voxelSize + volume["pixelSize"] = voxelSize[0] + volume["samplingRate"] = voxelSize[0] + + return volume + + def _extractTomogramFile(self, tomogram: Dict[str, Any]) -> Tuple[Optional[str], Optional[int]]: + for key in ( + "fileName", + "filename", + "path", + "tomogramFile", + "tomogramFileName", + "tomoFile", + "tomoFileName", + "volumeFile", + "volumeFileName", + "location", + ): + value = tomogram.get(key) + if value is None: + continue + + fileName, locationIndex = self._parseLocation(value) + if fileName: + return fileName, locationIndex + + return None, None + + def _parseLocation(self, raw: Any) -> Tuple[Optional[str], Optional[int]]: + if isinstance(raw, dict): + pathValue = None + for key in ("fileName", "filename", "path", "location", "stack"): + if key in raw: + pathValue = raw.get(key) + break + + indexValue = None + for key in ("index", "locationIndex", "slice", "itemIndex"): + if key in raw: + indexValue = raw.get(key) + break + + fileName, embeddedIndex = self._parseLocation(pathValue) + locationIndex = self._toOptionalInt(indexValue) + if locationIndex is None: + locationIndex = embeddedIndex + + return fileName, locationIndex + + if isinstance(raw, (list, tuple)): + if len(raw) >= 2: + return self._toText(raw[1]), self._toOptionalInt(raw[0]) + if len(raw) == 1: + return self._parseLocation(raw[0]) + return None, None + + text = self._toText(raw) + if not text: + return None, None + + locationIndex = None + fileName = text + + if "@" in text: + indexText, pathText = text.split("@", 1) + parsedIndex = self._toOptionalInt(indexText) + if parsedIndex is not None: + locationIndex = parsedIndex + fileName = pathText + + return fileName, locationIndex + + def _findVolume( + self, + volumeId: Union[int, str], + volumes: List[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + requested = self._toText(volumeId) + if requested is None: + return None + + for volume in volumes: + for key in ("id", "index", "tomoId", "volumeId", "objectId", "scipionItemId", "name", "label"): + if self._toText(volume.get(key)) == requested: + return volume + + requestedInt = self._toOptionalInt(requested) + if requestedInt is not None and 0 <= requestedInt < len(volumes): + return volumes[requestedInt] + + return None + + def _ensureVolumeInfoFromFile(self, volume: Dict[str, Any]) -> None: + fileName = volume.get("fileName") or volume.get("path") + resolvedPath = self._resolveExistingPath(fileName) + if resolvedPath is None: + return + + volume["fileName"] = resolvedPath + volume["path"] = resolvedPath + + try: + array, props = readVolumeArray3d(resolvedPath) + except Exception: + return + + if getattr(array, "ndim", None) == 3: + zDim, yDim, xDim = array.shape + volume["dims"] = [int(zDim), int(yDim), int(xDim)] + volume["xyzDims"] = [int(xDim), int(yDim), int(zDim)] + + props = props if isinstance(props, dict) else {} + + if "samplingRate" not in volume: + samplingRate = self._extractSamplingRate(props) + if samplingRate is not None: + volume["samplingRate"] = samplingRate + volume["pixelSize"] = samplingRate + volume["voxelSize"] = [samplingRate, samplingRate, samplingRate] + + try: + finite = np.asarray(array, dtype=np.float32) + finite = finite[np.isfinite(finite)] + if finite.size: + volume["min"] = float(np.min(finite)) + volume["max"] = float(np.max(finite)) + volume["mean"] = float(np.mean(finite)) + except Exception: + pass + + def _resolveExistingPath(self, fileName: Any) -> Optional[str]: + text = self._toText(fileName) + if not text: + return None + + path = Path(text).expanduser() + candidates = [] + + if path.is_absolute(): + candidates.append(path) + else: + candidates.append(path) + candidates.append(Path.cwd() / path) + + for candidate in candidates: + try: + resolved = candidate.resolve() + if resolved.exists(): + return str(resolved) + except Exception: + continue + + return None + + def _normalizeDims(self, value: Any) -> Optional[List[int]]: + if isinstance(value, dict): + value = [ + value.get("x") or value.get("X") or value.get("nx"), + value.get("y") or value.get("Y") or value.get("ny"), + value.get("z") or value.get("Z") or value.get("nz"), + ] + + if isinstance(value, (list, tuple)) and len(value) >= 3: + dims = [] + for item in value[:3]: + intValue = self._toOptionalInt(item) + if intValue is None or intValue <= 0: + return None + dims.append(intValue) + return dims + + return None + + def _normalizeVoxelSize(self, value: Any) -> Optional[List[float]]: + if value is None: + return None + + if isinstance(value, dict): + for key in ("x", "X", "samplingRate", "pixelSize", "voxelSize", "apix"): + number = self._toOptionalFloat(value.get(key)) + if number is not None: + return [number, number, number] + return None + + if isinstance(value, (list, tuple)): + if len(value) == 1: + number = self._toOptionalFloat(value[0]) + return [number, number, number] if number is not None else None + + if len(value) >= 3: + voxelSize = [] + for item in value[:3]: + number = self._toOptionalFloat(item) + if number is None: + return None + voxelSize.append(number) + return voxelSize + + number = self._toOptionalFloat(value) + return [number, number, number] if number is not None else None + + def _extractSamplingRate(self, values: Dict[str, Any]) -> Optional[float]: + for key in ("samplingRate", "pixelSize", "voxelSize", "apix"): + number = self._toOptionalFloat(values.get(key)) + if number is not None: + return number + return None + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + + try: + return int(value) + except Exception: + try: + return int(float(value)) + except Exception: + return None + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + + try: + return float(value) + except Exception: + return None + + def _toText(self, value: Any) -> Optional[str]: + if value is None: + return None + + text = str(value).strip() + return text or None \ No newline at end of file diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index ce599ae1..0f967ee1 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -388,15 +388,17 @@ def _mergeRootRelations( if tomograms: links["tomogram"] = { - "protocolId": None, - "outputName": None, + "protocolId": self.protocolId, + "outputName": self.outputName, "itemId": None, "label": "Tomograms", - "status": "inferred", + "status": "derived", + "source": "coordinates3d", } summaries["tomogram"] = { "objectClass": "SetOfTomograms", "size": len(tomograms), + "source": "coordinates3d", } return @@ -555,6 +557,11 @@ def _mergeRelatedStoredSets( if candidateKind is None or candidateKind not in links: continue + rootKind = self._getIntegratedKind(rootStoredSet) + + if rootKind == "coordinates3d" and candidateKind == "tomogram": + continue + if self._shouldReplaceLink(links.get(candidateKind)): links[candidateKind] = self._buildLink( protocolId=self._getCandidateProtocolId(candidate), From 787032a17b39e1e35d189d23db4adee6d1413288 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 10:30:12 +0200 Subject: [PATCH 068/278] clean integrated tomogram context workarounds --- ...tgresql_coords3d_tomogram_volume_reader.py | 1 - .../postgresql_integrated_context_reader.py | 7 +- .../viewers/postgresql_volume_reader.py | 131 +----------------- 3 files changed, 5 insertions(+), 134 deletions(-) diff --git a/app/backend/viewers/postgresql_coords3d_tomogram_volume_reader.py b/app/backend/viewers/postgresql_coords3d_tomogram_volume_reader.py index c909fd26..ef5bc2d0 100644 --- a/app/backend/viewers/postgresql_coords3d_tomogram_volume_reader.py +++ b/app/backend/viewers/postgresql_coords3d_tomogram_volume_reader.py @@ -23,7 +23,6 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ****************************************************************************** -import os from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 0f967ee1..64591a83 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -549,6 +549,8 @@ def _mergeRelatedStoredSets( summaries: Dict[str, Optional[Dict[str, Any]]], relationsByKey: Dict[str, Dict[str, Any]], ) -> None: + rootKind = self._getIntegratedKind(rootStoredSet) + for candidate in self._listRelatedStoredSets(): if self._isSameStoredSet(candidate, rootStoredSet): continue @@ -557,8 +559,6 @@ def _mergeRelatedStoredSets( if candidateKind is None or candidateKind not in links: continue - rootKind = self._getIntegratedKind(rootStoredSet) - if rootKind == "coordinates3d" and candidateKind == "tomogram": continue @@ -572,8 +572,6 @@ def _mergeRelatedStoredSets( summaries[candidateKind] = self._buildSummary(candidate) allowedRelationKeys = None - rootKind = self._getIntegratedKind(rootStoredSet) - if rootKind == "coordinates3d": allowedRelationKeys = set(relationsByKey.keys()) @@ -636,7 +634,6 @@ def _mergeRelationsForCandidate( allowedRelationKeys, ) self._addTiltSeriesRelations(relationsByKey, items) - self._addTiltSeriesRelations(relationsByKey, reader.listTiltSeries()) return if candidateKind == "ctf": diff --git a/app/backend/viewers/postgresql_volume_reader.py b/app/backend/viewers/postgresql_volume_reader.py index eb5c3f08..d479cb71 100644 --- a/app/backend/viewers/postgresql_volume_reader.py +++ b/app/backend/viewers/postgresql_volume_reader.py @@ -7,30 +7,6 @@ from app.backend.utils.volume_utils import readVolumeArray3d -VOLUME_DATA_EXTENSIONS = ( - ".mrc", - ".mrcs", - ".map", - ".rec", - ".em", - ".spi", - ".vol", - ".hdf", - ".h5", -) - -METADATA_LIKE_VOLUME_EXTENSIONS = ( - ".tomostar", - ".star", - ".xmd", - ".sqlite", - ".db", - ".json", - ".xml", - ".txt", - ".csv", - ".tsv", -) class PostgresqlVolumeReader: """Read Volume, VolumeMask and SetOfVolumes outputs from PostgreSQL.""" @@ -187,9 +163,6 @@ def _buildVolumeFromSetItem( fileName, locationIndex = self._extractVolumeFile(values) scipionItemId = item.get("scipionItemId") - if not fileName: - return None - rawLabel = ( item.get("label") or self._firstValueBySuffix(values, ["objLabel", "label", "name", "volName"]) @@ -467,116 +440,18 @@ def _resolveProtocolDbId(self) -> Optional[int]: return None def _extractVolumeFile(self, values: Dict[str, Any]) -> Tuple[Optional[str], Optional[int]]: - candidates: List[Tuple[str, Optional[int]]] = [] - - priorityGroups = [ - [ - "tomogramFileName", - "tomogramFilename", - "tomogramFile", - "tomogramFilePath", - "tomogramPath", - "tomoFileName", - "tomoFilename", - "tomoFile", - "tomoPath", - ], - [ - "volumeFileName", - "volumeFilename", - "volumeFile", - "volumeFilePath", - "volumePath", - "mapFileName", - "mapFilename", - "mapFile", - "mrcFileName", - "mrcFilename", - "mrcFile", - "recFileName", - "recFilename", - "recFile", - ], + raw = self._firstValueBySuffix( + values, [ "fileName", "filename", - "filePath", - "filepath", "location", "path", "stack", ], - [ - "volName", - "volumeName", - "tomoName", - "tomogramName", - ], - ] - - for suffixes in priorityGroups: - for raw in self._valuesBySuffix(values, suffixes): - fileName, locationIndex = self._parseLocation(raw) - if not fileName: - continue - - candidates.append((str(fileName), locationIndex)) - - for fileName, locationIndex in candidates: - if self._isVolumeDataPath(fileName): - return fileName, locationIndex - - for fileName, locationIndex in candidates: - if not self._isMetadataLikeVolumePath(fileName): - return fileName, locationIndex - - return None, None - - def _valuesBySuffix(self, values: Dict[str, Any], suffixes: List[str]) -> List[Any]: - normalizedSuffixes = [ - self._normalizeKey(suffix) - for suffix in suffixes - ] - - result = [] - seen = set() - - for key, value in values.items(): - if value is None: - continue - - normalizedKey = self._normalizeKey(key) - if not any(normalizedKey.endswith(suffix) for suffix in normalizedSuffixes): - continue - - marker = repr(value) - if marker in seen: - continue - - seen.add(marker) - result.append(value) - - return result - - def _isVolumeDataPath(self, fileName: Any) -> bool: - text = str(fileName or "").strip().lower() - if not text: - return False - - return any( - text.endswith(extension) or ("%s:" % extension) in text - for extension in VOLUME_DATA_EXTENSIONS ) - def _isMetadataLikeVolumePath(self, fileName: Any) -> bool: - text = str(fileName or "").strip().lower() - if not text: - return False - - return any( - text.endswith(extension) - for extension in METADATA_LIKE_VOLUME_EXTENSIONS - ) + return self._parseLocation(raw) def _parseLocation(self, raw: Any) -> Tuple[Optional[str], Optional[int]]: parsed = self._parseJsonValue(raw) From 278ce2017532bf55f8e488d25557e7c1323beacb Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 10:40:24 +0200 Subject: [PATCH 069/278] match integrated tilt series by tomogram ids" --- .../postgresql_integrated_context_reader.py | 152 ++++++++++++++---- 1 file changed, 121 insertions(+), 31 deletions(-) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 64591a83..4324955b 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -415,26 +415,38 @@ def _buildTomogramItemsFromStoredSet( for index, item in enumerate(storedSet.get("items") or []): values = item.get("values") or {} - tomogramId = self._firstValueByName( + tsId = self._firstValueByName( values, - ["_tomoId", "tomoId", "tomogramId", "_tsId", "tsId"], + ["_tsId", "tsId", "tiltSeriesId", "tilt_series_id"], + ) + + tomoId = self._firstValueByName( + values, + ["_tomoId", "tomoId", "tomogramId", "tomo_id", "tomogram_id"], ) label = self._firstValueByName( values, - ["_objLabel", "label", "_tsId", "tsId", "tomoId"], + ["_objLabel", "label", "name", "_tsId", "tsId", "tomoId"], ) - publicId = tomogramId or item.get("scipionItemId") or index + publicId = tsId or tomoId or item.get("scipionItemId") or index - items.append( - { - "id": publicId, - "tomoId": publicId, - "label": label or publicId, - "volumeId": index, - } - ) + row = { + "id": publicId, + "tomoId": publicId, + "label": label or publicId, + "volumeId": index, + } + + if tsId is not None: + row["tsId"] = tsId + row["tiltSeriesId"] = tsId + + if tomoId is not None: + row["sourceTomoId"] = tomoId + + items.append(row) return items @@ -535,13 +547,51 @@ def _listRelatedStoredSets(self) -> List[Dict[str, Any]]: result.sort( key=lambda item: ( int(item.get("distance") or 0), - str(item.get("relationRole") or ""), + self._relationRoleRank(item.get("relationRole")), int(item.get("protocolDbId") or 0), str(item.get("outputName") or ""), ) ) return result + def _relationRoleRank(self, relationRole: Any) -> int: + role = str(relationRole or "") + return { + "root": 0, + "parent": 1, + "child": 2, + }.get(role, 9) + + def _getAllowedRelationKeysForRoot( + self, + rootKind: Optional[str], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> Optional[set]: + if rootKind not in {"coordinates3d", "tomogram", "ctf"}: + return None + + keys = set() + + for key, relation in relationsByKey.items(): + candidates = [ + key, + relation.get("key"), + relation.get("label"), + relation.get("tiltSeriesId"), + relation.get("tsId"), + relation.get("tomogramId"), + relation.get("coordinatesTomogramId"), + relation.get("ctfSeriesId"), + relation.get("sourceTomoId"), + ] + + for value in candidates: + text = str(value).strip() if value is not None else "" + if text: + keys.add(text) + + return keys or None + def _mergeRelatedStoredSets( self, rootStoredSet: Dict[str, Any], @@ -559,9 +609,29 @@ def _mergeRelatedStoredSets( if candidateKind is None or candidateKind not in links: continue + relationRole = str(candidate.get("relationRole") or "") + + if rootKind in {"coordinates3d", "tomogram", "ctf"} and relationRole == "child": + continue + if rootKind == "coordinates3d" and candidateKind == "tomogram": continue + allowedRelationKeys = self._getAllowedRelationKeysForRoot( + rootKind=rootKind, + relationsByKey=relationsByKey, + ) + + mergedRelations = self._mergeRelationsForCandidate( + candidate=candidate, + candidateKind=candidateKind, + relationsByKey=relationsByKey, + allowedRelationKeys=allowedRelationKeys, + ) + + if not mergedRelations: + continue + if self._shouldReplaceLink(links.get(candidateKind)): links[candidateKind] = self._buildLink( protocolId=self._getCandidateProtocolId(candidate), @@ -571,17 +641,6 @@ def _mergeRelatedStoredSets( ) summaries[candidateKind] = self._buildSummary(candidate) - allowedRelationKeys = None - if rootKind == "coordinates3d": - allowedRelationKeys = set(relationsByKey.keys()) - - self._mergeRelationsForCandidate( - candidate=candidate, - candidateKind=candidateKind, - relationsByKey=relationsByKey, - allowedRelationKeys=allowedRelationKeys, - ) - def _filterIntegratedItemsByAllowedKeys( self, items: List[Dict[str, Any]], @@ -590,6 +649,12 @@ def _filterIntegratedItemsByAllowedKeys( if not allowedRelationKeys: return items or [] + allowedTextKeys = { + str(value).strip() + for value in allowedRelationKeys + if value is not None and str(value).strip() + } + filteredItems = [] for item in items or []: @@ -599,12 +664,18 @@ def _filterIntegratedItemsByAllowedKeys( item.get("id"), item.get("tomoId"), item.get("tomogramId"), + item.get("sourceTomoId"), + item.get("tsId"), item.get("tiltSeriesId"), item.get("ctfSeriesId"), item.get("coordinatesTomogramId"), ] - if any(str(value) in allowedRelationKeys for value in candidates if value is not None): + if any( + str(value).strip() in allowedTextKeys + for value in candidates + if value is not None and str(value).strip() + ): filteredItems.append(item) return filteredItems @@ -615,12 +686,12 @@ def _mergeRelationsForCandidate( candidateKind: str, relationsByKey: Dict[str, Dict[str, Any]], allowedRelationKeys: Optional[set] = None, - ) -> None: + ) -> bool: protocolDbId = candidate.get("protocolDbId") outputName = candidate.get("outputName") if protocolDbId is None or not outputName: - return + return False if candidateKind == "tiltSeries": reader = PostgresqlTiltSeriesReader( @@ -633,8 +704,12 @@ def _mergeRelationsForCandidate( reader.listTiltSeries(), allowedRelationKeys, ) + + if not items: + return False + self._addTiltSeriesRelations(relationsByKey, items) - return + return True if candidateKind == "ctf": reader = PostgresqlCtftomoReader( @@ -647,8 +722,12 @@ def _mergeRelationsForCandidate( reader.listCtftomoSeries(), allowedRelationKeys, ) + + if not items: + return False + self._addCtftomoRelations(relationsByKey, items) - return + return True if candidateKind == "coordinates3d": reader = PostgresqlCoords3dReader( @@ -661,8 +740,12 @@ def _mergeRelationsForCandidate( reader.listTomograms() or [], allowedRelationKeys, ) + + if not items: + return False + self._addCoordinates3dRelations(relationsByKey, items) - return + return True if candidateKind == "tomogram": storedSet = self.setMapper.getStoredSet( @@ -674,13 +757,20 @@ def _mergeRelationsForCandidate( ) if storedSet is None: - return + return False items = self._filterIntegratedItemsByAllowedKeys( self._buildTomogramItemsFromStoredSet(storedSet), allowedRelationKeys, ) + + if not items: + return False + self._addTomogramRelations(relationsByKey, items) + return True + + return False def _isSameStoredSet( self, From a1ec5250e05e504c436bf921cbef26e567beff55 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 10:53:32 +0200 Subject: [PATCH 070/278] derive integrated tilt series from tomograms --- app/backend/mapper/scipion_set_mapper.py | 125 +++++++++- .../postgresql_integrated_context_reader.py | 224 ++++++++++++++++++ 2 files changed, 348 insertions(+), 1 deletion(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index af35c4ae..cec9a953 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,7 +41,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 8 +NESTED_LOGICAL_TABLES_VERSION = 9 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): @@ -1369,6 +1369,10 @@ def _getSetProperties(self, scipionSet: Any) -> Dict[str, Any]: if linkedTomograms: properties["linkedTomograms"] = linkedTomograms + linkedTiltSeries = self._getLinkedTiltSeriesSummary(scipionSet) + if linkedTiltSeries: + properties["linkedTiltSeries"] = linkedTiltSeries + for attrName, attrValue in self._getAttributesToStore(scipionSet): if self._getAttributesToStore(attrValue): continue @@ -1386,6 +1390,125 @@ def _getLinkedTomogramsSummary(self, scipionSet: Any) -> List[Dict[str, Any]]: return tomograms + def _getLinkedTiltSeriesSummary(self, scipionSet: Any) -> Optional[Dict[str, Any]]: + tiltSeriesSet = self._callOptionalGetter(scipionSet, "getSetOfTiltSeries") + if tiltSeriesSet is None: + return None + + className = self._getClassName(tiltSeriesSet) + objectId = self._callOptionalGetter(tiltSeriesSet, "getObjId") + fileName = self._callOptionalGetter(tiltSeriesSet, "getFileName") + size = self._callOptionalGetter(tiltSeriesSet, "getSize") + + itemSummaries = [] + tsIds = [] + + for index, tiltSeries in enumerate(self._iterTiltSeriesItems(tiltSeriesSet)): + item = self._buildLinkedTiltSeriesItemSummary(tiltSeries, index) + if item is None: + continue + + itemSummaries.append(item) + + tiltSeriesId = item.get("tiltSeriesId") or item.get("tsId") or item.get("id") + if tiltSeriesId is not None: + tsIds.append(str(tiltSeriesId)) + + summary: Dict[str, Any] = { + "className": className, + "objectId": str(objectId) if objectId is not None else None, + "fileName": str(fileName) if fileName else None, + "size": self._toOptionalInt(size), + "tsIds": self._uniqueTextValues(tsIds), + "items": itemSummaries, + } + + return { + key: value + for key, value in summary.items() + if value not in (None, [], "") + } + + def _iterTiltSeriesItems(self, tiltSeriesSet: Any) -> Iterable[Any]: + iterItems = getattr(tiltSeriesSet, "iterItems", None) + if callable(iterItems): + try: + return iterItems(iterate=False) + except TypeError: + return iterItems() + except Exception: + pass + + try: + return iter(tiltSeriesSet) + except Exception: + return iter(()) + + def _buildLinkedTiltSeriesItemSummary(self, tiltSeries: Any, index: int) -> Optional[Dict[str, Any]]: + objectId = self._callOptionalGetter(tiltSeries, "getObjId") + tsId = ( + self._callOptionalGetter(tiltSeries, "getTsId") + or self._callOptionalGetter(tiltSeries, "getTSId") + or objectId + or index + ) + + if tsId is None: + return None + + label = ( + self._callOptionalGetter(tiltSeries, "getObjLabel") + or self._callOptionalGetter(tiltSeries, "getNameId") + or tsId + ) + + item: Dict[str, Any] = { + "id": str(tsId), + "tsId": str(tsId), + "tiltSeriesId": str(tsId), + "label": str(label), + } + + if objectId is not None: + item["objectId"] = str(objectId) + + dims = self._normalizeLinkedTomogramDims( + self._callOptionalGetter(tiltSeries, "getDim") + ) + if dims is not None: + item["dims"] = dims + + samplingRate = self._toOptionalFloat( + self._callOptionalGetter(tiltSeries, "getSamplingRate") + ) + if samplingRate is not None: + item["pixelSize"] = samplingRate + + tiltAxisAngle = self._toOptionalFloat( + self._callOptionalGetter( + self._callOptionalGetter(tiltSeries, "getAcquisition"), + "getTiltAxisAngle", + ) + ) + if tiltAxisAngle is not None: + item["tiltAxisAngle"] = tiltAxisAngle + + return item + + def _uniqueTextValues(self, values: Iterable[Any]) -> List[str]: + result = [] + seen = set() + + for value in values or []: + text = str(value).strip() if value is not None else "" + if not text or text in seen: + continue + + seen.add(text) + result.append(text) + + return result + def _iterLinkedTomograms(self, scipionSet: Any) -> Iterable[Any]: for methodName in ("iterTomograms", "iterVolumes"): iteratorGetter = getattr(scipionSet, methodName, None) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 4324955b..ec023721 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -348,6 +348,220 @@ def _addCoordinates3dRelations( label=label, ) + def _getLinkedTiltSeriesFromProperties(self, storedSet: Dict[str, Any]) -> Optional[Dict[str, Any]]: + properties = storedSet.get("properties") or {} + + if isinstance(properties, str): + try: + properties = json.loads(properties) + except Exception: + properties = {} + + linkedTiltSeries = properties.get("linkedTiltSeries") if isinstance(properties, dict) else None + if isinstance(linkedTiltSeries, dict): + return linkedTiltSeries + + for item in storedSet.get("setProperties") or []: + if str(item.get("key")) != "linkedTiltSeries": + continue + + value = item.get("value") + if isinstance(value, dict): + return value + + if isinstance(value, str): + try: + parsed = json.loads(value) + except Exception: + parsed = None + + if isinstance(parsed, dict): + return parsed + + return None + + def _listTiltSeriesStoredSetCandidates(self) -> List[Dict[str, Any]]: + try: + rows = self.db.fetchAll( + """ + SELECT + s.id, + s."projectId", + s."protocolDbId", + s."objectId", + s."outputName", + s."setClassName", + s."itemClassName", + s.properties, + s."createdAt", + s."updatedAt", + p."protocolId" AS "publicProtocolId", + p."protocolClassName" + FROM scipion_sets s + JOIN protocols p + ON p.id = s."protocolDbId" + AND p."projectId" = s."projectId" + WHERE s."projectId" = %s + AND ( + LOWER(COALESCE(s."setClassName", '')) LIKE '%%setoftiltseries%%' + OR LOWER(COALESCE(s."itemClassName", '')) LIKE '%%tiltseries%%' + ) + ORDER BY s."protocolDbId" ASC, s."outputName" ASC + """, + (self.projectId,), + ) + except Exception: + logger.debug( + "Could not list PostgreSQL tilt-series candidates for integrated context. projectId=%s", + self.projectId, + exc_info=True, + ) + return [] + + return [dict(row) for row in rows or []] + + def _getStoredSetPropertiesDict(self, storedSet: Dict[str, Any]) -> Dict[str, Any]: + properties = storedSet.get("properties") or {} + + if isinstance(properties, str): + try: + properties = json.loads(properties) + except Exception: + properties = {} + + return properties if isinstance(properties, dict) else {} + + def _scoreLinkedTiltSeriesCandidate( + self, + candidate: Dict[str, Any], + linkedTiltSeries: Dict[str, Any], + relationKeys: Optional[set], + ) -> int: + score = 0 + candidateProperties = self._getStoredSetPropertiesDict(candidate) + + linkedObjectId = linkedTiltSeries.get("objectId") + candidateObjectIds = [ + candidate.get("objectId"), + candidateProperties.get("scipionObjId"), + candidateProperties.get("objectId"), + ] + + if linkedObjectId is not None and any(str(value) == str(linkedObjectId) for value in candidateObjectIds if value is not None): + score += 1000 + + linkedFileName = linkedTiltSeries.get("fileName") + candidateFileName = candidateProperties.get("fileName") + if linkedFileName and candidateFileName and str(linkedFileName) == str(candidateFileName): + score += 300 + + linkedTsIds = { + str(value).strip() + for value in linkedTiltSeries.get("tsIds") or [] + if value is not None and str(value).strip() + } + + if relationKeys: + linkedTsIds.update( + str(value).strip() + for value in relationKeys + if value is not None and str(value).strip() + ) + + if linkedTsIds: + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=candidate.get("protocolDbId"), + outputName=candidate.get("outputName"), + ) + candidateItems = reader.listTiltSeries() + candidateTsIds = { + str(item.get("tiltSeriesId") or item.get("tsId") or item.get("id")).strip() + for item in candidateItems or [] + if str(item.get("tiltSeriesId") or item.get("tsId") or item.get("id") or "").strip() + } + + intersection = candidateTsIds.intersection(linkedTsIds) + if intersection: + score += 10 * len(intersection) + + return score + + def _resolveLinkedTiltSeriesStoredSet( + self, + linkedTiltSeries: Optional[Dict[str, Any]], + relationKeys: Optional[set], + ) -> Optional[Dict[str, Any]]: + if not linkedTiltSeries: + return None + + bestCandidate = None + bestScore = 0 + + for candidate in self._listTiltSeriesStoredSetCandidates(): + score = self._scoreLinkedTiltSeriesCandidate( + candidate=candidate, + linkedTiltSeries=linkedTiltSeries, + relationKeys=relationKeys, + ) + + if score > bestScore: + bestScore = score + bestCandidate = candidate + + return bestCandidate + + def _mergeTomogramLinkedTiltSeries( + self, + storedSet: Dict[str, Any], + links: Dict[str, Optional[Dict[str, Any]]], + summaries: Dict[str, Optional[Dict[str, Any]]], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> None: + linkedTiltSeries = self._getLinkedTiltSeriesFromProperties(storedSet) + relationKeys = set(relationsByKey.keys()) + + candidate = self._resolveLinkedTiltSeriesStoredSet( + linkedTiltSeries=linkedTiltSeries, + relationKeys=relationKeys, + ) + + if candidate is None: + return + + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=candidate.get("protocolDbId"), + outputName=candidate.get("outputName"), + ) + + items = self._filterIntegratedItemsByAllowedKeys( + reader.listTiltSeries(), + relationKeys, + ) + + if not items: + return + + self._addTiltSeriesRelations(relationsByKey, items) + + links["tiltSeries"] = self._buildLink( + protocolId=self._getCandidateProtocolId(candidate), + outputName=candidate.get("outputName"), + storedSet=candidate, + statusValue="derived", + ) + links["tiltSeries"]["source"] = "tomogram" + + summaries["tiltSeries"] = self._buildSummary( + candidate, + extra={ + "source": "tomogram", + }, + ) + def _mergeRootRelations( self, rootKind: str, @@ -406,6 +620,13 @@ def _mergeRootRelations( items = self._buildTomogramItemsFromStoredSet(storedSet) self._addTomogramRelations(relationsByKey, items) + self._mergeTomogramLinkedTiltSeries( + storedSet=storedSet, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + def _buildTomogramItemsFromStoredSet( self, storedSet: Dict[str, Any], @@ -617,6 +838,9 @@ def _mergeRelatedStoredSets( if rootKind == "coordinates3d" and candidateKind == "tomogram": continue + if rootKind == "tomogram" and candidateKind == "tiltSeries" and links.get("tiltSeries") is not None: + continue + allowedRelationKeys = self._getAllowedRelationKeysForRoot( rootKind=rootKind, relationsByKey=relationsByKey, From 8045c893c2b6f34c42805398d789ed5f841c6b8b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 11:14:11 +0200 Subject: [PATCH 071/278] serve integrated tilt series from tomogram context --- app/backend/api/services/project_service.py | 22 ++ app/backend/mapper/scipion_set_mapper.py | 120 +++++++++- .../postgresql_integrated_context_reader.py | 213 ++++------------- .../postgresql_tomogram_tiltseries_reader.py | 219 ++++++++++++++++++ 4 files changed, 403 insertions(+), 171 deletions(-) create mode 100644 app/backend/viewers/postgresql_tomogram_tiltseries_reader.py diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 03dce8dc..ce4228cd 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6337,6 +6337,28 @@ def _getPostgresqlTiltSeriesReaderIfAvailable( if mapper is None: return None + try: + from app.backend.viewers.postgresql_tomogram_tiltseries_reader import PostgresqlTomogramTiltSeriesReader + + reader = PostgresqlTomogramTiltSeriesReader( + db=mapper.db, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if reader.hasOutput(): + return reader + + except Exception: + logger.debug( + "PostgreSQL Tomogram-derived tilt-series reader is not available. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + exc_info=True, + ) + try: from app.backend.viewers.postgresql_tiltseries_reader import PostgresqlTiltSeriesReader diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index cec9a953..f0056dcd 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,7 +41,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 9 +NESTED_LOGICAL_TABLES_VERSION = 10 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): @@ -1484,17 +1484,127 @@ def _buildLinkedTiltSeriesItemSummary(self, tiltSeries: Any, index: int) -> Opti if samplingRate is not None: item["pixelSize"] = samplingRate + acquisition = self._callOptionalGetter(tiltSeries, "getAcquisition") tiltAxisAngle = self._toOptionalFloat( - self._callOptionalGetter( - self._callOptionalGetter(tiltSeries, "getAcquisition"), - "getTiltAxisAngle", - ) + self._callOptionalGetter(acquisition, "getTiltAxisAngle") ) if tiltAxisAngle is not None: item["tiltAxisAngle"] = tiltAxisAngle + frames = self._buildLinkedTiltSeriesFramesSummary(tiltSeries) + if frames: + item["frames"] = frames + item["nViews"] = len(frames) + else: + size = self._toOptionalInt(self._callOptionalGetter(tiltSeries, "getSize")) + if size is not None: + item["nViews"] = size + return item + def _buildLinkedTiltSeriesFramesSummary(self, tiltSeries: Any) -> List[Dict[str, Any]]: + frames = [] + + for position, tiltImage in enumerate(self._iterTiltSeriesImages(tiltSeries)): + frame = self._buildLinkedTiltImageFrameSummary(tiltImage, position) + if frame is not None: + frames.append(frame) + + return frames + + def _iterTiltSeriesImages(self, tiltSeries: Any) -> Iterable[Any]: + iterItems = getattr(tiltSeries, "iterItems", None) + if callable(iterItems): + try: + return iterItems(iterate=False) + except TypeError: + return iterItems() + except Exception: + pass + + try: + return iter(tiltSeries) + except Exception: + return iter(()) + + def _buildLinkedTiltImageFrameSummary(self, tiltImage: Any, position: int) -> Optional[Dict[str, Any]]: + viewId = self._callOptionalGetter(tiltImage, "getObjId") or position + imageIndex = self._toOptionalInt(self._callOptionalGetter(tiltImage, "getIndex")) + + if imageIndex is None: + imageIndex = position + + frame: Dict[str, Any] = { + "viewId": viewId, + "index": imageIndex, + } + + order = self._toOptionalInt(self._callOptionalGetter(tiltImage, "getAcquisitionOrder")) + if order is not None: + frame["order"] = order + + tiltAngle = self._toOptionalFloat(self._callOptionalGetter(tiltImage, "getTiltAngle")) + if tiltAngle is not None: + frame["tiltAngle"] = tiltAngle + + excluded = self._getLinkedTiltImageExcluded(tiltImage) + frame["excluded"] = excluded + + acquisition = self._callOptionalGetter(tiltImage, "getAcquisition") + dose = self._toOptionalFloat(self._callOptionalGetter(acquisition, "getAccumDose")) + if dose is not None: + frame["dose"] = dose + + fileName = self._callOptionalGetter(tiltImage, "getFileName") + if fileName: + frame["path"] = "%s@%s" % (str(imageIndex), str(fileName)) + + transform = self._getLinkedTiltImageTransform(tiltImage) + frame.update(transform) + + return frame + + def _getLinkedTiltImageExcluded(self, tiltImage: Any) -> bool: + for methodName in ("isExcluded", "getIsExcluded"): + value = self._callOptionalGetter(tiltImage, methodName) + if value is not None: + return bool(value) + + for methodName in ("isEnabled", "getEnabled"): + value = self._callOptionalGetter(tiltImage, methodName) + if value is not None: + return not bool(value) + + return False + + def _getLinkedTiltImageTransform(self, tiltImage: Any) -> Dict[str, Any]: + output = {} + + hasTransform = self._callOptionalGetter(tiltImage, "hasTransform") + if hasTransform is False: + return output + + transform = self._callOptionalGetter(tiltImage, "getTransform") + if transform is None: + return output + + try: + import math + + _alpha, _beta, gamma = transform.getEulerAngles() + output["rot"] = math.degrees(-gamma) + except Exception: + pass + + try: + matrix = transform.getMatrixAsList() + output["shiftX"] = matrix[2] + output["shiftY"] = matrix[5] + except Exception: + pass + + return output + def _uniqueTextValues(self, values: Iterable[Any]) -> List[str]: result = [] seen = set() diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index ec023721..c277c565 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -380,138 +380,6 @@ def _getLinkedTiltSeriesFromProperties(self, storedSet: Dict[str, Any]) -> Optio return None - def _listTiltSeriesStoredSetCandidates(self) -> List[Dict[str, Any]]: - try: - rows = self.db.fetchAll( - """ - SELECT - s.id, - s."projectId", - s."protocolDbId", - s."objectId", - s."outputName", - s."setClassName", - s."itemClassName", - s.properties, - s."createdAt", - s."updatedAt", - p."protocolId" AS "publicProtocolId", - p."protocolClassName" - FROM scipion_sets s - JOIN protocols p - ON p.id = s."protocolDbId" - AND p."projectId" = s."projectId" - WHERE s."projectId" = %s - AND ( - LOWER(COALESCE(s."setClassName", '')) LIKE '%%setoftiltseries%%' - OR LOWER(COALESCE(s."itemClassName", '')) LIKE '%%tiltseries%%' - ) - ORDER BY s."protocolDbId" ASC, s."outputName" ASC - """, - (self.projectId,), - ) - except Exception: - logger.debug( - "Could not list PostgreSQL tilt-series candidates for integrated context. projectId=%s", - self.projectId, - exc_info=True, - ) - return [] - - return [dict(row) for row in rows or []] - - def _getStoredSetPropertiesDict(self, storedSet: Dict[str, Any]) -> Dict[str, Any]: - properties = storedSet.get("properties") or {} - - if isinstance(properties, str): - try: - properties = json.loads(properties) - except Exception: - properties = {} - - return properties if isinstance(properties, dict) else {} - - def _scoreLinkedTiltSeriesCandidate( - self, - candidate: Dict[str, Any], - linkedTiltSeries: Dict[str, Any], - relationKeys: Optional[set], - ) -> int: - score = 0 - candidateProperties = self._getStoredSetPropertiesDict(candidate) - - linkedObjectId = linkedTiltSeries.get("objectId") - candidateObjectIds = [ - candidate.get("objectId"), - candidateProperties.get("scipionObjId"), - candidateProperties.get("objectId"), - ] - - if linkedObjectId is not None and any(str(value) == str(linkedObjectId) for value in candidateObjectIds if value is not None): - score += 1000 - - linkedFileName = linkedTiltSeries.get("fileName") - candidateFileName = candidateProperties.get("fileName") - if linkedFileName and candidateFileName and str(linkedFileName) == str(candidateFileName): - score += 300 - - linkedTsIds = { - str(value).strip() - for value in linkedTiltSeries.get("tsIds") or [] - if value is not None and str(value).strip() - } - - if relationKeys: - linkedTsIds.update( - str(value).strip() - for value in relationKeys - if value is not None and str(value).strip() - ) - - if linkedTsIds: - reader = PostgresqlTiltSeriesReader( - db=self.db, - projectId=self.projectId, - protocolId=candidate.get("protocolDbId"), - outputName=candidate.get("outputName"), - ) - candidateItems = reader.listTiltSeries() - candidateTsIds = { - str(item.get("tiltSeriesId") or item.get("tsId") or item.get("id")).strip() - for item in candidateItems or [] - if str(item.get("tiltSeriesId") or item.get("tsId") or item.get("id") or "").strip() - } - - intersection = candidateTsIds.intersection(linkedTsIds) - if intersection: - score += 10 * len(intersection) - - return score - - def _resolveLinkedTiltSeriesStoredSet( - self, - linkedTiltSeries: Optional[Dict[str, Any]], - relationKeys: Optional[set], - ) -> Optional[Dict[str, Any]]: - if not linkedTiltSeries: - return None - - bestCandidate = None - bestScore = 0 - - for candidate in self._listTiltSeriesStoredSetCandidates(): - score = self._scoreLinkedTiltSeriesCandidate( - candidate=candidate, - linkedTiltSeries=linkedTiltSeries, - relationKeys=relationKeys, - ) - - if score > bestScore: - bestScore = score - bestCandidate = candidate - - return bestCandidate - def _mergeTomogramLinkedTiltSeries( self, storedSet: Dict[str, Any], @@ -520,47 +388,60 @@ def _mergeTomogramLinkedTiltSeries( relationsByKey: Dict[str, Dict[str, Any]], ) -> None: linkedTiltSeries = self._getLinkedTiltSeriesFromProperties(storedSet) - relationKeys = set(relationsByKey.keys()) - - candidate = self._resolveLinkedTiltSeriesStoredSet( - linkedTiltSeries=linkedTiltSeries, - relationKeys=relationKeys, - ) - - if candidate is None: + if not linkedTiltSeries: return - reader = PostgresqlTiltSeriesReader( - db=self.db, - projectId=self.projectId, - protocolId=candidate.get("protocolDbId"), - outputName=candidate.get("outputName"), - ) - - items = self._filterIntegratedItemsByAllowedKeys( - reader.listTiltSeries(), - relationKeys, - ) - + items = self._buildLinkedTiltSeriesItemsFromProperties(linkedTiltSeries) if not items: return self._addTiltSeriesRelations(relationsByKey, items) - links["tiltSeries"] = self._buildLink( - protocolId=self._getCandidateProtocolId(candidate), - outputName=candidate.get("outputName"), - storedSet=candidate, - statusValue="derived", - ) - links["tiltSeries"]["source"] = "tomogram" + links["tiltSeries"] = { + "protocolId": self.protocolId, + "outputName": self.outputName, + "itemId": linkedTiltSeries.get("objectId"), + "label": "Tilt series", + "status": "derived", + "source": "tomogram", + } - summaries["tiltSeries"] = self._buildSummary( - candidate, - extra={ - "source": "tomogram", - }, - ) + summaries["tiltSeries"] = { + "objectClass": linkedTiltSeries.get("className") or "SetOfTiltSeries", + "objectId": linkedTiltSeries.get("objectId"), + "size": linkedTiltSeries.get("size") or len(items), + "tsIds": linkedTiltSeries.get("tsIds") or [], + "fileName": linkedTiltSeries.get("fileName"), + "source": "tomogram", + } + + + def _buildLinkedTiltSeriesItemsFromProperties( + self, + linkedTiltSeries: Dict[str, Any], + ) -> List[Dict[str, Any]]: + items = [] + + for index, item in enumerate(linkedTiltSeries.get("items") or []): + if not isinstance(item, dict): + continue + + tiltSeriesId = ( + item.get("tiltSeriesId") + or item.get("tsId") + or item.get("id") + or index + ) + + row = dict(item) + row["id"] = str(tiltSeriesId) + row["tsId"] = str(tiltSeriesId) + row["tiltSeriesId"] = str(tiltSeriesId) + row["label"] = str(item.get("label") or "TiltSeries %s" % str(tiltSeriesId)) + + items.append(row) + + return items def _mergeRootRelations( self, @@ -838,7 +719,7 @@ def _mergeRelatedStoredSets( if rootKind == "coordinates3d" and candidateKind == "tomogram": continue - if rootKind == "tomogram" and candidateKind == "tiltSeries" and links.get("tiltSeries") is not None: + if rootKind == "tomogram" and candidateKind == "tiltSeries": continue allowedRelationKeys = self._getAllowedRelationKeysForRoot( diff --git a/app/backend/viewers/postgresql_tomogram_tiltseries_reader.py b/app/backend/viewers/postgresql_tomogram_tiltseries_reader.py new file mode 100644 index 00000000..6eac0db8 --- /dev/null +++ b/app/backend/viewers/postgresql_tomogram_tiltseries_reader.py @@ -0,0 +1,219 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +import json +from typing import Any, Dict, List, Optional, Union + +from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + +class PostgresqlTomogramTiltSeriesReader: + """Expose tilt series linked from a SetOfTomograms as tilt-series-like items.""" + + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + self.setMapper = ScipionSetPostgresqlMapper(db) + self._storedSet = None + self._linkedTiltSeries = None + self.lastSkipReason = None + + def hasOutput(self) -> bool: + return bool(self._getLinkedTiltSeriesItems()) + + def listTiltSeries(self) -> List[Dict[str, Any]]: + items = self._getLinkedTiltSeriesItems() + return [ + self._buildTiltSeriesSummary(item, index) + for index, item in enumerate(items) + ] + + def getTiltSeriesFrames(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: + item = self._findTiltSeriesItem(tiltSeriesId) + if item is None: + self.lastSkipReason = "tilt_series_not_found tiltSeriesId=%s" % str(tiltSeriesId) + return None + + summary = self._buildTiltSeriesSummary(item, 0) + frames = item.get("frames") or [] + + payload = dict(summary) + payload["frames"] = frames + payload["nViews"] = len(frames) + + return payload + + def getTiltImageFrame(self, tiltSeriesId: Any, index: Any) -> Optional[Dict[str, Any]]: + payload = self.getTiltSeriesFrames(tiltSeriesId) + if not payload: + return None + + frames = payload.get("frames") or [] + targetIndex = self._toOptionalInt(index) + + if targetIndex is not None: + for frame in frames: + frameIndex = self._toOptionalInt(frame.get("index")) + if frameIndex == targetIndex: + return frame + + if 0 <= targetIndex < len(frames): + return frames[targetIndex] + + return None + + def _getStoredSet(self) -> Optional[Dict[str, Any]]: + if self._storedSet is None: + try: + self._storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=self.protocolId, + outputName=self.outputName, + limit=None, + offset=0, + ) + except Exception: + self._storedSet = None + + return self._storedSet + + def _getLinkedTiltSeries(self) -> Optional[Dict[str, Any]]: + if self._linkedTiltSeries is not None: + return self._linkedTiltSeries + + storedSet = self._getStoredSet() + if storedSet is None: + self.lastSkipReason = "stored_set_not_found" + return None + + properties = storedSet.get("properties") or {} + if isinstance(properties, str): + try: + properties = json.loads(properties) + except Exception: + properties = {} + + linkedTiltSeries = properties.get("linkedTiltSeries") if isinstance(properties, dict) else None + if isinstance(linkedTiltSeries, dict): + self._linkedTiltSeries = linkedTiltSeries + return self._linkedTiltSeries + + for item in storedSet.get("setProperties") or []: + if str(item.get("key")) != "linkedTiltSeries": + continue + + value = item.get("value") + if isinstance(value, dict): + self._linkedTiltSeries = value + return self._linkedTiltSeries + + if isinstance(value, str): + try: + parsed = json.loads(value) + except Exception: + parsed = None + + if isinstance(parsed, dict): + self._linkedTiltSeries = parsed + return self._linkedTiltSeries + + self.lastSkipReason = "linked_tilt_series_not_found" + return None + + def _getLinkedTiltSeriesItems(self) -> List[Dict[str, Any]]: + linkedTiltSeries = self._getLinkedTiltSeries() + if not linkedTiltSeries: + return [] + + items = linkedTiltSeries.get("items") or [] + return [ + item + for item in items + if isinstance(item, dict) + ] + + def _buildTiltSeriesSummary(self, item: Dict[str, Any], index: int) -> Dict[str, Any]: + tiltSeriesId = ( + item.get("tiltSeriesId") + or item.get("tsId") + or item.get("id") + or index + ) + + label = item.get("label") or "TiltSeries %s" % str(tiltSeriesId) + frames = item.get("frames") or [] + + summary: Dict[str, Any] = { + "tiltSeriesId": str(tiltSeriesId), + "label": str(label), + "source": "tomogram", + } + + if item.get("dims") is not None: + summary["dims"] = item.get("dims") + + if item.get("pixelSize") is not None: + summary["pixelSize"] = item.get("pixelSize") + + if item.get("tiltAxisAngle") is not None: + summary["tiltAxisAngle"] = item.get("tiltAxisAngle") + + if frames: + summary["nViews"] = len(frames) + elif item.get("nViews") is not None: + summary["nViews"] = item.get("nViews") + + return summary + + def _findTiltSeriesItem(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: + target = str(tiltSeriesId) + + for index, item in enumerate(self._getLinkedTiltSeriesItems()): + candidates = [ + item.get("tiltSeriesId"), + item.get("tsId"), + item.get("id"), + item.get("label"), + index, + ] + + if any(str(value) == target for value in candidates if value is not None): + return item + + return None + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + + try: + return int(value) + except Exception: + try: + return int(float(value)) + except Exception: + return None \ No newline at end of file From 38816c20c7b92756835fbef9e5ded4cf22174764 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 11:19:19 +0200 Subject: [PATCH 072/278] Revert "serve integrated tilt series from tomogram context" This reverts commit 8045c893c2b6f34c42805398d789ed5f841c6b8b. --- app/backend/api/services/project_service.py | 22 -- app/backend/mapper/scipion_set_mapper.py | 120 +--------- .../postgresql_integrated_context_reader.py | 213 +++++++++++++---- .../postgresql_tomogram_tiltseries_reader.py | 219 ------------------ 4 files changed, 171 insertions(+), 403 deletions(-) delete mode 100644 app/backend/viewers/postgresql_tomogram_tiltseries_reader.py diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index ce4228cd..03dce8dc 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6337,28 +6337,6 @@ def _getPostgresqlTiltSeriesReaderIfAvailable( if mapper is None: return None - try: - from app.backend.viewers.postgresql_tomogram_tiltseries_reader import PostgresqlTomogramTiltSeriesReader - - reader = PostgresqlTomogramTiltSeriesReader( - db=mapper.db, - projectId=projectId, - protocolId=protocolId, - outputName=outputName, - ) - - if reader.hasOutput(): - return reader - - except Exception: - logger.debug( - "PostgreSQL Tomogram-derived tilt-series reader is not available. projectId=%s protocolId=%s outputName=%s", - projectId, - protocolId, - outputName, - exc_info=True, - ) - try: from app.backend.viewers.postgresql_tiltseries_reader import PostgresqlTiltSeriesReader diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index f0056dcd..cec9a953 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,7 +41,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 10 +NESTED_LOGICAL_TABLES_VERSION = 9 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): @@ -1484,127 +1484,17 @@ def _buildLinkedTiltSeriesItemSummary(self, tiltSeries: Any, index: int) -> Opti if samplingRate is not None: item["pixelSize"] = samplingRate - acquisition = self._callOptionalGetter(tiltSeries, "getAcquisition") tiltAxisAngle = self._toOptionalFloat( - self._callOptionalGetter(acquisition, "getTiltAxisAngle") + self._callOptionalGetter( + self._callOptionalGetter(tiltSeries, "getAcquisition"), + "getTiltAxisAngle", + ) ) if tiltAxisAngle is not None: item["tiltAxisAngle"] = tiltAxisAngle - frames = self._buildLinkedTiltSeriesFramesSummary(tiltSeries) - if frames: - item["frames"] = frames - item["nViews"] = len(frames) - else: - size = self._toOptionalInt(self._callOptionalGetter(tiltSeries, "getSize")) - if size is not None: - item["nViews"] = size - return item - def _buildLinkedTiltSeriesFramesSummary(self, tiltSeries: Any) -> List[Dict[str, Any]]: - frames = [] - - for position, tiltImage in enumerate(self._iterTiltSeriesImages(tiltSeries)): - frame = self._buildLinkedTiltImageFrameSummary(tiltImage, position) - if frame is not None: - frames.append(frame) - - return frames - - def _iterTiltSeriesImages(self, tiltSeries: Any) -> Iterable[Any]: - iterItems = getattr(tiltSeries, "iterItems", None) - if callable(iterItems): - try: - return iterItems(iterate=False) - except TypeError: - return iterItems() - except Exception: - pass - - try: - return iter(tiltSeries) - except Exception: - return iter(()) - - def _buildLinkedTiltImageFrameSummary(self, tiltImage: Any, position: int) -> Optional[Dict[str, Any]]: - viewId = self._callOptionalGetter(tiltImage, "getObjId") or position - imageIndex = self._toOptionalInt(self._callOptionalGetter(tiltImage, "getIndex")) - - if imageIndex is None: - imageIndex = position - - frame: Dict[str, Any] = { - "viewId": viewId, - "index": imageIndex, - } - - order = self._toOptionalInt(self._callOptionalGetter(tiltImage, "getAcquisitionOrder")) - if order is not None: - frame["order"] = order - - tiltAngle = self._toOptionalFloat(self._callOptionalGetter(tiltImage, "getTiltAngle")) - if tiltAngle is not None: - frame["tiltAngle"] = tiltAngle - - excluded = self._getLinkedTiltImageExcluded(tiltImage) - frame["excluded"] = excluded - - acquisition = self._callOptionalGetter(tiltImage, "getAcquisition") - dose = self._toOptionalFloat(self._callOptionalGetter(acquisition, "getAccumDose")) - if dose is not None: - frame["dose"] = dose - - fileName = self._callOptionalGetter(tiltImage, "getFileName") - if fileName: - frame["path"] = "%s@%s" % (str(imageIndex), str(fileName)) - - transform = self._getLinkedTiltImageTransform(tiltImage) - frame.update(transform) - - return frame - - def _getLinkedTiltImageExcluded(self, tiltImage: Any) -> bool: - for methodName in ("isExcluded", "getIsExcluded"): - value = self._callOptionalGetter(tiltImage, methodName) - if value is not None: - return bool(value) - - for methodName in ("isEnabled", "getEnabled"): - value = self._callOptionalGetter(tiltImage, methodName) - if value is not None: - return not bool(value) - - return False - - def _getLinkedTiltImageTransform(self, tiltImage: Any) -> Dict[str, Any]: - output = {} - - hasTransform = self._callOptionalGetter(tiltImage, "hasTransform") - if hasTransform is False: - return output - - transform = self._callOptionalGetter(tiltImage, "getTransform") - if transform is None: - return output - - try: - import math - - _alpha, _beta, gamma = transform.getEulerAngles() - output["rot"] = math.degrees(-gamma) - except Exception: - pass - - try: - matrix = transform.getMatrixAsList() - output["shiftX"] = matrix[2] - output["shiftY"] = matrix[5] - except Exception: - pass - - return output - def _uniqueTextValues(self, values: Iterable[Any]) -> List[str]: result = [] seen = set() diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index c277c565..ec023721 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -380,6 +380,138 @@ def _getLinkedTiltSeriesFromProperties(self, storedSet: Dict[str, Any]) -> Optio return None + def _listTiltSeriesStoredSetCandidates(self) -> List[Dict[str, Any]]: + try: + rows = self.db.fetchAll( + """ + SELECT + s.id, + s."projectId", + s."protocolDbId", + s."objectId", + s."outputName", + s."setClassName", + s."itemClassName", + s.properties, + s."createdAt", + s."updatedAt", + p."protocolId" AS "publicProtocolId", + p."protocolClassName" + FROM scipion_sets s + JOIN protocols p + ON p.id = s."protocolDbId" + AND p."projectId" = s."projectId" + WHERE s."projectId" = %s + AND ( + LOWER(COALESCE(s."setClassName", '')) LIKE '%%setoftiltseries%%' + OR LOWER(COALESCE(s."itemClassName", '')) LIKE '%%tiltseries%%' + ) + ORDER BY s."protocolDbId" ASC, s."outputName" ASC + """, + (self.projectId,), + ) + except Exception: + logger.debug( + "Could not list PostgreSQL tilt-series candidates for integrated context. projectId=%s", + self.projectId, + exc_info=True, + ) + return [] + + return [dict(row) for row in rows or []] + + def _getStoredSetPropertiesDict(self, storedSet: Dict[str, Any]) -> Dict[str, Any]: + properties = storedSet.get("properties") or {} + + if isinstance(properties, str): + try: + properties = json.loads(properties) + except Exception: + properties = {} + + return properties if isinstance(properties, dict) else {} + + def _scoreLinkedTiltSeriesCandidate( + self, + candidate: Dict[str, Any], + linkedTiltSeries: Dict[str, Any], + relationKeys: Optional[set], + ) -> int: + score = 0 + candidateProperties = self._getStoredSetPropertiesDict(candidate) + + linkedObjectId = linkedTiltSeries.get("objectId") + candidateObjectIds = [ + candidate.get("objectId"), + candidateProperties.get("scipionObjId"), + candidateProperties.get("objectId"), + ] + + if linkedObjectId is not None and any(str(value) == str(linkedObjectId) for value in candidateObjectIds if value is not None): + score += 1000 + + linkedFileName = linkedTiltSeries.get("fileName") + candidateFileName = candidateProperties.get("fileName") + if linkedFileName and candidateFileName and str(linkedFileName) == str(candidateFileName): + score += 300 + + linkedTsIds = { + str(value).strip() + for value in linkedTiltSeries.get("tsIds") or [] + if value is not None and str(value).strip() + } + + if relationKeys: + linkedTsIds.update( + str(value).strip() + for value in relationKeys + if value is not None and str(value).strip() + ) + + if linkedTsIds: + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=candidate.get("protocolDbId"), + outputName=candidate.get("outputName"), + ) + candidateItems = reader.listTiltSeries() + candidateTsIds = { + str(item.get("tiltSeriesId") or item.get("tsId") or item.get("id")).strip() + for item in candidateItems or [] + if str(item.get("tiltSeriesId") or item.get("tsId") or item.get("id") or "").strip() + } + + intersection = candidateTsIds.intersection(linkedTsIds) + if intersection: + score += 10 * len(intersection) + + return score + + def _resolveLinkedTiltSeriesStoredSet( + self, + linkedTiltSeries: Optional[Dict[str, Any]], + relationKeys: Optional[set], + ) -> Optional[Dict[str, Any]]: + if not linkedTiltSeries: + return None + + bestCandidate = None + bestScore = 0 + + for candidate in self._listTiltSeriesStoredSetCandidates(): + score = self._scoreLinkedTiltSeriesCandidate( + candidate=candidate, + linkedTiltSeries=linkedTiltSeries, + relationKeys=relationKeys, + ) + + if score > bestScore: + bestScore = score + bestCandidate = candidate + + return bestCandidate + def _mergeTomogramLinkedTiltSeries( self, storedSet: Dict[str, Any], @@ -388,60 +520,47 @@ def _mergeTomogramLinkedTiltSeries( relationsByKey: Dict[str, Dict[str, Any]], ) -> None: linkedTiltSeries = self._getLinkedTiltSeriesFromProperties(storedSet) - if not linkedTiltSeries: - return - - items = self._buildLinkedTiltSeriesItemsFromProperties(linkedTiltSeries) - if not items: - return - - self._addTiltSeriesRelations(relationsByKey, items) - - links["tiltSeries"] = { - "protocolId": self.protocolId, - "outputName": self.outputName, - "itemId": linkedTiltSeries.get("objectId"), - "label": "Tilt series", - "status": "derived", - "source": "tomogram", - } + relationKeys = set(relationsByKey.keys()) - summaries["tiltSeries"] = { - "objectClass": linkedTiltSeries.get("className") or "SetOfTiltSeries", - "objectId": linkedTiltSeries.get("objectId"), - "size": linkedTiltSeries.get("size") or len(items), - "tsIds": linkedTiltSeries.get("tsIds") or [], - "fileName": linkedTiltSeries.get("fileName"), - "source": "tomogram", - } + candidate = self._resolveLinkedTiltSeriesStoredSet( + linkedTiltSeries=linkedTiltSeries, + relationKeys=relationKeys, + ) + if candidate is None: + return - def _buildLinkedTiltSeriesItemsFromProperties( - self, - linkedTiltSeries: Dict[str, Any], - ) -> List[Dict[str, Any]]: - items = [] + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=candidate.get("protocolDbId"), + outputName=candidate.get("outputName"), + ) - for index, item in enumerate(linkedTiltSeries.get("items") or []): - if not isinstance(item, dict): - continue + items = self._filterIntegratedItemsByAllowedKeys( + reader.listTiltSeries(), + relationKeys, + ) - tiltSeriesId = ( - item.get("tiltSeriesId") - or item.get("tsId") - or item.get("id") - or index - ) + if not items: + return - row = dict(item) - row["id"] = str(tiltSeriesId) - row["tsId"] = str(tiltSeriesId) - row["tiltSeriesId"] = str(tiltSeriesId) - row["label"] = str(item.get("label") or "TiltSeries %s" % str(tiltSeriesId)) + self._addTiltSeriesRelations(relationsByKey, items) - items.append(row) + links["tiltSeries"] = self._buildLink( + protocolId=self._getCandidateProtocolId(candidate), + outputName=candidate.get("outputName"), + storedSet=candidate, + statusValue="derived", + ) + links["tiltSeries"]["source"] = "tomogram" - return items + summaries["tiltSeries"] = self._buildSummary( + candidate, + extra={ + "source": "tomogram", + }, + ) def _mergeRootRelations( self, @@ -719,7 +838,7 @@ def _mergeRelatedStoredSets( if rootKind == "coordinates3d" and candidateKind == "tomogram": continue - if rootKind == "tomogram" and candidateKind == "tiltSeries": + if rootKind == "tomogram" and candidateKind == "tiltSeries" and links.get("tiltSeries") is not None: continue allowedRelationKeys = self._getAllowedRelationKeysForRoot( diff --git a/app/backend/viewers/postgresql_tomogram_tiltseries_reader.py b/app/backend/viewers/postgresql_tomogram_tiltseries_reader.py deleted file mode 100644 index 6eac0db8..00000000 --- a/app/backend/viewers/postgresql_tomogram_tiltseries_reader.py +++ /dev/null @@ -1,219 +0,0 @@ -# ****************************************************************************** -# * -# * Authors: Yunior C. Fonseca Reyna -# * -# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 3 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# * -# ****************************************************************************** -import json -from typing import Any, Dict, List, Optional, Union - -from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper - - -class PostgresqlTomogramTiltSeriesReader: - """Expose tilt series linked from a SetOfTomograms as tilt-series-like items.""" - - def __init__(self, db, projectId: int, protocolId: int, outputName: str): - self.db = db - self.projectId = projectId - self.protocolId = protocolId - self.outputName = outputName - self.setMapper = ScipionSetPostgresqlMapper(db) - self._storedSet = None - self._linkedTiltSeries = None - self.lastSkipReason = None - - def hasOutput(self) -> bool: - return bool(self._getLinkedTiltSeriesItems()) - - def listTiltSeries(self) -> List[Dict[str, Any]]: - items = self._getLinkedTiltSeriesItems() - return [ - self._buildTiltSeriesSummary(item, index) - for index, item in enumerate(items) - ] - - def getTiltSeriesFrames(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: - item = self._findTiltSeriesItem(tiltSeriesId) - if item is None: - self.lastSkipReason = "tilt_series_not_found tiltSeriesId=%s" % str(tiltSeriesId) - return None - - summary = self._buildTiltSeriesSummary(item, 0) - frames = item.get("frames") or [] - - payload = dict(summary) - payload["frames"] = frames - payload["nViews"] = len(frames) - - return payload - - def getTiltImageFrame(self, tiltSeriesId: Any, index: Any) -> Optional[Dict[str, Any]]: - payload = self.getTiltSeriesFrames(tiltSeriesId) - if not payload: - return None - - frames = payload.get("frames") or [] - targetIndex = self._toOptionalInt(index) - - if targetIndex is not None: - for frame in frames: - frameIndex = self._toOptionalInt(frame.get("index")) - if frameIndex == targetIndex: - return frame - - if 0 <= targetIndex < len(frames): - return frames[targetIndex] - - return None - - def _getStoredSet(self) -> Optional[Dict[str, Any]]: - if self._storedSet is None: - try: - self._storedSet = self.setMapper.getStoredSet( - projectId=self.projectId, - protocolDbId=self.protocolId, - outputName=self.outputName, - limit=None, - offset=0, - ) - except Exception: - self._storedSet = None - - return self._storedSet - - def _getLinkedTiltSeries(self) -> Optional[Dict[str, Any]]: - if self._linkedTiltSeries is not None: - return self._linkedTiltSeries - - storedSet = self._getStoredSet() - if storedSet is None: - self.lastSkipReason = "stored_set_not_found" - return None - - properties = storedSet.get("properties") or {} - if isinstance(properties, str): - try: - properties = json.loads(properties) - except Exception: - properties = {} - - linkedTiltSeries = properties.get("linkedTiltSeries") if isinstance(properties, dict) else None - if isinstance(linkedTiltSeries, dict): - self._linkedTiltSeries = linkedTiltSeries - return self._linkedTiltSeries - - for item in storedSet.get("setProperties") or []: - if str(item.get("key")) != "linkedTiltSeries": - continue - - value = item.get("value") - if isinstance(value, dict): - self._linkedTiltSeries = value - return self._linkedTiltSeries - - if isinstance(value, str): - try: - parsed = json.loads(value) - except Exception: - parsed = None - - if isinstance(parsed, dict): - self._linkedTiltSeries = parsed - return self._linkedTiltSeries - - self.lastSkipReason = "linked_tilt_series_not_found" - return None - - def _getLinkedTiltSeriesItems(self) -> List[Dict[str, Any]]: - linkedTiltSeries = self._getLinkedTiltSeries() - if not linkedTiltSeries: - return [] - - items = linkedTiltSeries.get("items") or [] - return [ - item - for item in items - if isinstance(item, dict) - ] - - def _buildTiltSeriesSummary(self, item: Dict[str, Any], index: int) -> Dict[str, Any]: - tiltSeriesId = ( - item.get("tiltSeriesId") - or item.get("tsId") - or item.get("id") - or index - ) - - label = item.get("label") or "TiltSeries %s" % str(tiltSeriesId) - frames = item.get("frames") or [] - - summary: Dict[str, Any] = { - "tiltSeriesId": str(tiltSeriesId), - "label": str(label), - "source": "tomogram", - } - - if item.get("dims") is not None: - summary["dims"] = item.get("dims") - - if item.get("pixelSize") is not None: - summary["pixelSize"] = item.get("pixelSize") - - if item.get("tiltAxisAngle") is not None: - summary["tiltAxisAngle"] = item.get("tiltAxisAngle") - - if frames: - summary["nViews"] = len(frames) - elif item.get("nViews") is not None: - summary["nViews"] = item.get("nViews") - - return summary - - def _findTiltSeriesItem(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: - target = str(tiltSeriesId) - - for index, item in enumerate(self._getLinkedTiltSeriesItems()): - candidates = [ - item.get("tiltSeriesId"), - item.get("tsId"), - item.get("id"), - item.get("label"), - index, - ] - - if any(str(value) == target for value in candidates if value is not None): - return item - - return None - - def _toOptionalInt(self, value: Any) -> Optional[int]: - if value is None or value == "": - return None - - try: - return int(value) - except Exception: - try: - return int(float(value)) - except Exception: - return None \ No newline at end of file From 9b739e3c8b5cc8ba6aa87fe8be26becd3d7f43d9 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 11:19:24 +0200 Subject: [PATCH 073/278] Revert "derive integrated tilt series from tomograms" This reverts commit a1ec5250e05e504c436bf921cbef26e567beff55. --- app/backend/mapper/scipion_set_mapper.py | 125 +--------- .../postgresql_integrated_context_reader.py | 224 ------------------ 2 files changed, 1 insertion(+), 348 deletions(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index cec9a953..af35c4ae 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,7 +41,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 9 +NESTED_LOGICAL_TABLES_VERSION = 8 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): @@ -1369,10 +1369,6 @@ def _getSetProperties(self, scipionSet: Any) -> Dict[str, Any]: if linkedTomograms: properties["linkedTomograms"] = linkedTomograms - linkedTiltSeries = self._getLinkedTiltSeriesSummary(scipionSet) - if linkedTiltSeries: - properties["linkedTiltSeries"] = linkedTiltSeries - for attrName, attrValue in self._getAttributesToStore(scipionSet): if self._getAttributesToStore(attrValue): continue @@ -1390,125 +1386,6 @@ def _getLinkedTomogramsSummary(self, scipionSet: Any) -> List[Dict[str, Any]]: return tomograms - def _getLinkedTiltSeriesSummary(self, scipionSet: Any) -> Optional[Dict[str, Any]]: - tiltSeriesSet = self._callOptionalGetter(scipionSet, "getSetOfTiltSeries") - if tiltSeriesSet is None: - return None - - className = self._getClassName(tiltSeriesSet) - objectId = self._callOptionalGetter(tiltSeriesSet, "getObjId") - fileName = self._callOptionalGetter(tiltSeriesSet, "getFileName") - size = self._callOptionalGetter(tiltSeriesSet, "getSize") - - itemSummaries = [] - tsIds = [] - - for index, tiltSeries in enumerate(self._iterTiltSeriesItems(tiltSeriesSet)): - item = self._buildLinkedTiltSeriesItemSummary(tiltSeries, index) - if item is None: - continue - - itemSummaries.append(item) - - tiltSeriesId = item.get("tiltSeriesId") or item.get("tsId") or item.get("id") - if tiltSeriesId is not None: - tsIds.append(str(tiltSeriesId)) - - summary: Dict[str, Any] = { - "className": className, - "objectId": str(objectId) if objectId is not None else None, - "fileName": str(fileName) if fileName else None, - "size": self._toOptionalInt(size), - "tsIds": self._uniqueTextValues(tsIds), - "items": itemSummaries, - } - - return { - key: value - for key, value in summary.items() - if value not in (None, [], "") - } - - def _iterTiltSeriesItems(self, tiltSeriesSet: Any) -> Iterable[Any]: - iterItems = getattr(tiltSeriesSet, "iterItems", None) - if callable(iterItems): - try: - return iterItems(iterate=False) - except TypeError: - return iterItems() - except Exception: - pass - - try: - return iter(tiltSeriesSet) - except Exception: - return iter(()) - - def _buildLinkedTiltSeriesItemSummary(self, tiltSeries: Any, index: int) -> Optional[Dict[str, Any]]: - objectId = self._callOptionalGetter(tiltSeries, "getObjId") - tsId = ( - self._callOptionalGetter(tiltSeries, "getTsId") - or self._callOptionalGetter(tiltSeries, "getTSId") - or objectId - or index - ) - - if tsId is None: - return None - - label = ( - self._callOptionalGetter(tiltSeries, "getObjLabel") - or self._callOptionalGetter(tiltSeries, "getNameId") - or tsId - ) - - item: Dict[str, Any] = { - "id": str(tsId), - "tsId": str(tsId), - "tiltSeriesId": str(tsId), - "label": str(label), - } - - if objectId is not None: - item["objectId"] = str(objectId) - - dims = self._normalizeLinkedTomogramDims( - self._callOptionalGetter(tiltSeries, "getDim") - ) - if dims is not None: - item["dims"] = dims - - samplingRate = self._toOptionalFloat( - self._callOptionalGetter(tiltSeries, "getSamplingRate") - ) - if samplingRate is not None: - item["pixelSize"] = samplingRate - - tiltAxisAngle = self._toOptionalFloat( - self._callOptionalGetter( - self._callOptionalGetter(tiltSeries, "getAcquisition"), - "getTiltAxisAngle", - ) - ) - if tiltAxisAngle is not None: - item["tiltAxisAngle"] = tiltAxisAngle - - return item - - def _uniqueTextValues(self, values: Iterable[Any]) -> List[str]: - result = [] - seen = set() - - for value in values or []: - text = str(value).strip() if value is not None else "" - if not text or text in seen: - continue - - seen.add(text) - result.append(text) - - return result - def _iterLinkedTomograms(self, scipionSet: Any) -> Iterable[Any]: for methodName in ("iterTomograms", "iterVolumes"): iteratorGetter = getattr(scipionSet, methodName, None) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index ec023721..4324955b 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -348,220 +348,6 @@ def _addCoordinates3dRelations( label=label, ) - def _getLinkedTiltSeriesFromProperties(self, storedSet: Dict[str, Any]) -> Optional[Dict[str, Any]]: - properties = storedSet.get("properties") or {} - - if isinstance(properties, str): - try: - properties = json.loads(properties) - except Exception: - properties = {} - - linkedTiltSeries = properties.get("linkedTiltSeries") if isinstance(properties, dict) else None - if isinstance(linkedTiltSeries, dict): - return linkedTiltSeries - - for item in storedSet.get("setProperties") or []: - if str(item.get("key")) != "linkedTiltSeries": - continue - - value = item.get("value") - if isinstance(value, dict): - return value - - if isinstance(value, str): - try: - parsed = json.loads(value) - except Exception: - parsed = None - - if isinstance(parsed, dict): - return parsed - - return None - - def _listTiltSeriesStoredSetCandidates(self) -> List[Dict[str, Any]]: - try: - rows = self.db.fetchAll( - """ - SELECT - s.id, - s."projectId", - s."protocolDbId", - s."objectId", - s."outputName", - s."setClassName", - s."itemClassName", - s.properties, - s."createdAt", - s."updatedAt", - p."protocolId" AS "publicProtocolId", - p."protocolClassName" - FROM scipion_sets s - JOIN protocols p - ON p.id = s."protocolDbId" - AND p."projectId" = s."projectId" - WHERE s."projectId" = %s - AND ( - LOWER(COALESCE(s."setClassName", '')) LIKE '%%setoftiltseries%%' - OR LOWER(COALESCE(s."itemClassName", '')) LIKE '%%tiltseries%%' - ) - ORDER BY s."protocolDbId" ASC, s."outputName" ASC - """, - (self.projectId,), - ) - except Exception: - logger.debug( - "Could not list PostgreSQL tilt-series candidates for integrated context. projectId=%s", - self.projectId, - exc_info=True, - ) - return [] - - return [dict(row) for row in rows or []] - - def _getStoredSetPropertiesDict(self, storedSet: Dict[str, Any]) -> Dict[str, Any]: - properties = storedSet.get("properties") or {} - - if isinstance(properties, str): - try: - properties = json.loads(properties) - except Exception: - properties = {} - - return properties if isinstance(properties, dict) else {} - - def _scoreLinkedTiltSeriesCandidate( - self, - candidate: Dict[str, Any], - linkedTiltSeries: Dict[str, Any], - relationKeys: Optional[set], - ) -> int: - score = 0 - candidateProperties = self._getStoredSetPropertiesDict(candidate) - - linkedObjectId = linkedTiltSeries.get("objectId") - candidateObjectIds = [ - candidate.get("objectId"), - candidateProperties.get("scipionObjId"), - candidateProperties.get("objectId"), - ] - - if linkedObjectId is not None and any(str(value) == str(linkedObjectId) for value in candidateObjectIds if value is not None): - score += 1000 - - linkedFileName = linkedTiltSeries.get("fileName") - candidateFileName = candidateProperties.get("fileName") - if linkedFileName and candidateFileName and str(linkedFileName) == str(candidateFileName): - score += 300 - - linkedTsIds = { - str(value).strip() - for value in linkedTiltSeries.get("tsIds") or [] - if value is not None and str(value).strip() - } - - if relationKeys: - linkedTsIds.update( - str(value).strip() - for value in relationKeys - if value is not None and str(value).strip() - ) - - if linkedTsIds: - reader = PostgresqlTiltSeriesReader( - db=self.db, - projectId=self.projectId, - protocolId=candidate.get("protocolDbId"), - outputName=candidate.get("outputName"), - ) - candidateItems = reader.listTiltSeries() - candidateTsIds = { - str(item.get("tiltSeriesId") or item.get("tsId") or item.get("id")).strip() - for item in candidateItems or [] - if str(item.get("tiltSeriesId") or item.get("tsId") or item.get("id") or "").strip() - } - - intersection = candidateTsIds.intersection(linkedTsIds) - if intersection: - score += 10 * len(intersection) - - return score - - def _resolveLinkedTiltSeriesStoredSet( - self, - linkedTiltSeries: Optional[Dict[str, Any]], - relationKeys: Optional[set], - ) -> Optional[Dict[str, Any]]: - if not linkedTiltSeries: - return None - - bestCandidate = None - bestScore = 0 - - for candidate in self._listTiltSeriesStoredSetCandidates(): - score = self._scoreLinkedTiltSeriesCandidate( - candidate=candidate, - linkedTiltSeries=linkedTiltSeries, - relationKeys=relationKeys, - ) - - if score > bestScore: - bestScore = score - bestCandidate = candidate - - return bestCandidate - - def _mergeTomogramLinkedTiltSeries( - self, - storedSet: Dict[str, Any], - links: Dict[str, Optional[Dict[str, Any]]], - summaries: Dict[str, Optional[Dict[str, Any]]], - relationsByKey: Dict[str, Dict[str, Any]], - ) -> None: - linkedTiltSeries = self._getLinkedTiltSeriesFromProperties(storedSet) - relationKeys = set(relationsByKey.keys()) - - candidate = self._resolveLinkedTiltSeriesStoredSet( - linkedTiltSeries=linkedTiltSeries, - relationKeys=relationKeys, - ) - - if candidate is None: - return - - reader = PostgresqlTiltSeriesReader( - db=self.db, - projectId=self.projectId, - protocolId=candidate.get("protocolDbId"), - outputName=candidate.get("outputName"), - ) - - items = self._filterIntegratedItemsByAllowedKeys( - reader.listTiltSeries(), - relationKeys, - ) - - if not items: - return - - self._addTiltSeriesRelations(relationsByKey, items) - - links["tiltSeries"] = self._buildLink( - protocolId=self._getCandidateProtocolId(candidate), - outputName=candidate.get("outputName"), - storedSet=candidate, - statusValue="derived", - ) - links["tiltSeries"]["source"] = "tomogram" - - summaries["tiltSeries"] = self._buildSummary( - candidate, - extra={ - "source": "tomogram", - }, - ) - def _mergeRootRelations( self, rootKind: str, @@ -620,13 +406,6 @@ def _mergeRootRelations( items = self._buildTomogramItemsFromStoredSet(storedSet) self._addTomogramRelations(relationsByKey, items) - self._mergeTomogramLinkedTiltSeries( - storedSet=storedSet, - links=links, - summaries=summaries, - relationsByKey=relationsByKey, - ) - def _buildTomogramItemsFromStoredSet( self, storedSet: Dict[str, Any], @@ -838,9 +617,6 @@ def _mergeRelatedStoredSets( if rootKind == "coordinates3d" and candidateKind == "tomogram": continue - if rootKind == "tomogram" and candidateKind == "tiltSeries" and links.get("tiltSeries") is not None: - continue - allowedRelationKeys = self._getAllowedRelationKeysForRoot( rootKind=rootKind, relationsByKey=relationsByKey, From 700fea77320270ba1bf2682551ab1593608661a7 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 11:19:29 +0200 Subject: [PATCH 074/278] Revert "match integrated tilt series by tomogram ids"" This reverts commit 278ce2017532bf55f8e488d25557e7c1323beacb. --- .../postgresql_integrated_context_reader.py | 152 ++++-------------- 1 file changed, 31 insertions(+), 121 deletions(-) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 4324955b..64591a83 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -415,38 +415,26 @@ def _buildTomogramItemsFromStoredSet( for index, item in enumerate(storedSet.get("items") or []): values = item.get("values") or {} - tsId = self._firstValueByName( + tomogramId = self._firstValueByName( values, - ["_tsId", "tsId", "tiltSeriesId", "tilt_series_id"], - ) - - tomoId = self._firstValueByName( - values, - ["_tomoId", "tomoId", "tomogramId", "tomo_id", "tomogram_id"], + ["_tomoId", "tomoId", "tomogramId", "_tsId", "tsId"], ) label = self._firstValueByName( values, - ["_objLabel", "label", "name", "_tsId", "tsId", "tomoId"], + ["_objLabel", "label", "_tsId", "tsId", "tomoId"], ) - publicId = tsId or tomoId or item.get("scipionItemId") or index - - row = { - "id": publicId, - "tomoId": publicId, - "label": label or publicId, - "volumeId": index, - } - - if tsId is not None: - row["tsId"] = tsId - row["tiltSeriesId"] = tsId + publicId = tomogramId or item.get("scipionItemId") or index - if tomoId is not None: - row["sourceTomoId"] = tomoId - - items.append(row) + items.append( + { + "id": publicId, + "tomoId": publicId, + "label": label or publicId, + "volumeId": index, + } + ) return items @@ -547,51 +535,13 @@ def _listRelatedStoredSets(self) -> List[Dict[str, Any]]: result.sort( key=lambda item: ( int(item.get("distance") or 0), - self._relationRoleRank(item.get("relationRole")), + str(item.get("relationRole") or ""), int(item.get("protocolDbId") or 0), str(item.get("outputName") or ""), ) ) return result - def _relationRoleRank(self, relationRole: Any) -> int: - role = str(relationRole or "") - return { - "root": 0, - "parent": 1, - "child": 2, - }.get(role, 9) - - def _getAllowedRelationKeysForRoot( - self, - rootKind: Optional[str], - relationsByKey: Dict[str, Dict[str, Any]], - ) -> Optional[set]: - if rootKind not in {"coordinates3d", "tomogram", "ctf"}: - return None - - keys = set() - - for key, relation in relationsByKey.items(): - candidates = [ - key, - relation.get("key"), - relation.get("label"), - relation.get("tiltSeriesId"), - relation.get("tsId"), - relation.get("tomogramId"), - relation.get("coordinatesTomogramId"), - relation.get("ctfSeriesId"), - relation.get("sourceTomoId"), - ] - - for value in candidates: - text = str(value).strip() if value is not None else "" - if text: - keys.add(text) - - return keys or None - def _mergeRelatedStoredSets( self, rootStoredSet: Dict[str, Any], @@ -609,29 +559,9 @@ def _mergeRelatedStoredSets( if candidateKind is None or candidateKind not in links: continue - relationRole = str(candidate.get("relationRole") or "") - - if rootKind in {"coordinates3d", "tomogram", "ctf"} and relationRole == "child": - continue - if rootKind == "coordinates3d" and candidateKind == "tomogram": continue - allowedRelationKeys = self._getAllowedRelationKeysForRoot( - rootKind=rootKind, - relationsByKey=relationsByKey, - ) - - mergedRelations = self._mergeRelationsForCandidate( - candidate=candidate, - candidateKind=candidateKind, - relationsByKey=relationsByKey, - allowedRelationKeys=allowedRelationKeys, - ) - - if not mergedRelations: - continue - if self._shouldReplaceLink(links.get(candidateKind)): links[candidateKind] = self._buildLink( protocolId=self._getCandidateProtocolId(candidate), @@ -641,6 +571,17 @@ def _mergeRelatedStoredSets( ) summaries[candidateKind] = self._buildSummary(candidate) + allowedRelationKeys = None + if rootKind == "coordinates3d": + allowedRelationKeys = set(relationsByKey.keys()) + + self._mergeRelationsForCandidate( + candidate=candidate, + candidateKind=candidateKind, + relationsByKey=relationsByKey, + allowedRelationKeys=allowedRelationKeys, + ) + def _filterIntegratedItemsByAllowedKeys( self, items: List[Dict[str, Any]], @@ -649,12 +590,6 @@ def _filterIntegratedItemsByAllowedKeys( if not allowedRelationKeys: return items or [] - allowedTextKeys = { - str(value).strip() - for value in allowedRelationKeys - if value is not None and str(value).strip() - } - filteredItems = [] for item in items or []: @@ -664,18 +599,12 @@ def _filterIntegratedItemsByAllowedKeys( item.get("id"), item.get("tomoId"), item.get("tomogramId"), - item.get("sourceTomoId"), - item.get("tsId"), item.get("tiltSeriesId"), item.get("ctfSeriesId"), item.get("coordinatesTomogramId"), ] - if any( - str(value).strip() in allowedTextKeys - for value in candidates - if value is not None and str(value).strip() - ): + if any(str(value) in allowedRelationKeys for value in candidates if value is not None): filteredItems.append(item) return filteredItems @@ -686,12 +615,12 @@ def _mergeRelationsForCandidate( candidateKind: str, relationsByKey: Dict[str, Dict[str, Any]], allowedRelationKeys: Optional[set] = None, - ) -> bool: + ) -> None: protocolDbId = candidate.get("protocolDbId") outputName = candidate.get("outputName") if protocolDbId is None or not outputName: - return False + return if candidateKind == "tiltSeries": reader = PostgresqlTiltSeriesReader( @@ -704,12 +633,8 @@ def _mergeRelationsForCandidate( reader.listTiltSeries(), allowedRelationKeys, ) - - if not items: - return False - self._addTiltSeriesRelations(relationsByKey, items) - return True + return if candidateKind == "ctf": reader = PostgresqlCtftomoReader( @@ -722,12 +647,8 @@ def _mergeRelationsForCandidate( reader.listCtftomoSeries(), allowedRelationKeys, ) - - if not items: - return False - self._addCtftomoRelations(relationsByKey, items) - return True + return if candidateKind == "coordinates3d": reader = PostgresqlCoords3dReader( @@ -740,12 +661,8 @@ def _mergeRelationsForCandidate( reader.listTomograms() or [], allowedRelationKeys, ) - - if not items: - return False - self._addCoordinates3dRelations(relationsByKey, items) - return True + return if candidateKind == "tomogram": storedSet = self.setMapper.getStoredSet( @@ -757,20 +674,13 @@ def _mergeRelationsForCandidate( ) if storedSet is None: - return False + return items = self._filterIntegratedItemsByAllowedKeys( self._buildTomogramItemsFromStoredSet(storedSet), allowedRelationKeys, ) - - if not items: - return False - self._addTomogramRelations(relationsByKey, items) - return True - - return False def _isSameStoredSet( self, From fdd9f49f4d638c70f1786e409c6e16825c7718cf Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 11:23:33 +0200 Subject: [PATCH 075/278] solving conflicts --- .backend_reload_marker | 1 + app/backend/mapper/scipion_set_mapper.py | 126 ++++++++++ .../postgresql_integrated_context_reader.py | 232 ++++++++++++++++++ dump.rdb | Bin 0 -> 771 bytes scipion_home/.plugins_revision | 1 + scipion_home/.plugins_revision.lock | 0 .../config/backup/scipion.conf.20260307213812 | 25 ++ .../config/backup/scipion.conf.20260307214508 | 25 ++ .../config/backup/scipion.conf.20260307215448 | 11 + .../config/backup/scipion.conf.20260307215946 | 12 + .../config/backup/scipion.conf.20260307225850 | 12 + .../config/backup/scipion.conf.20260320071727 | 13 + .../config/backup/scipion.conf.20260320071826 | 13 + .../config/backup/scipion.conf.20260415111435 | 13 + .../config/backup/scipion.conf.20260415111706 | 14 ++ .../config/backup/scipion.conf.20260416211619 | 13 + .../config/backup/scipion.conf.20260421114030 | 13 + .../config/backup/scipion.conf.20260421114143 | 16 ++ .../config/backup/scipion.conf.20260421114423 | 9 + .../config/backup/scipion.conf.20260421114636 | 10 + .../config/backup/scipion.conf.20260421115104 | 9 + .../config/backup/scipion.conf.20260421115119 | 10 + .../config/backup/scipion.conf.20260421122630 | 10 + .../config/backup/scipion.conf.20260421122638 | 12 + .../config/backup/scipion.conf.20260421124315 | 25 ++ .../config/backup/scipion.conf.20260421124338 | 17 ++ .../config/backup/scipion.conf.20260422190617 | 17 ++ .../config/backup/scipion.conf.20260513164709 | 18 ++ .../config/backup/scipion.conf.20260513165026 | 18 ++ .../config/backup/scipion.conf.20260513165138 | 18 ++ .../config/backup/scipion.conf.20260514101059 | 18 ++ .../config/backup/scipion.conf.20260519133809 | 19 ++ .../config/backup/scipion.conf.20260520151542 | 18 ++ .../config/backup/scipion.conf.20260520154051 | 19 ++ .../config/backup/scipion.conf.20260520154226 | 20 ++ .../config/backup/scipion.conf.20260520154544 | 20 ++ .../config/backup/scipion.conf.20260520161814 | 20 ++ .../config/backup/scipion.conf.20260603210905 | 19 ++ .../config/backup/scipion.conf.20260604132329 | 20 ++ .../config/backup/scipion.conf.20260604133119 | 21 ++ scipion_home/config/hosts (Copy).conf | 46 ++++ scipion_home/config/hosts.conf | 30 +++ scipion_home/config/scipion.conf | 21 ++ .../config/scipionweb_environment.json | 4 + scipion_home/software | 1 + scipion_home/web/devel_plugins.json | 106 ++++++++ 46 files changed, 1115 insertions(+) create mode 100644 .backend_reload_marker create mode 100644 dump.rdb create mode 100644 scipion_home/.plugins_revision create mode 100644 scipion_home/.plugins_revision.lock create mode 100644 scipion_home/config/backup/scipion.conf.20260307213812 create mode 100644 scipion_home/config/backup/scipion.conf.20260307214508 create mode 100644 scipion_home/config/backup/scipion.conf.20260307215448 create mode 100644 scipion_home/config/backup/scipion.conf.20260307215946 create mode 100644 scipion_home/config/backup/scipion.conf.20260307225850 create mode 100644 scipion_home/config/backup/scipion.conf.20260320071727 create mode 100644 scipion_home/config/backup/scipion.conf.20260320071826 create mode 100644 scipion_home/config/backup/scipion.conf.20260415111435 create mode 100644 scipion_home/config/backup/scipion.conf.20260415111706 create mode 100644 scipion_home/config/backup/scipion.conf.20260416211619 create mode 100644 scipion_home/config/backup/scipion.conf.20260421114030 create mode 100644 scipion_home/config/backup/scipion.conf.20260421114143 create mode 100644 scipion_home/config/backup/scipion.conf.20260421114423 create mode 100644 scipion_home/config/backup/scipion.conf.20260421114636 create mode 100644 scipion_home/config/backup/scipion.conf.20260421115104 create mode 100644 scipion_home/config/backup/scipion.conf.20260421115119 create mode 100644 scipion_home/config/backup/scipion.conf.20260421122630 create mode 100644 scipion_home/config/backup/scipion.conf.20260421122638 create mode 100644 scipion_home/config/backup/scipion.conf.20260421124315 create mode 100644 scipion_home/config/backup/scipion.conf.20260421124338 create mode 100644 scipion_home/config/backup/scipion.conf.20260422190617 create mode 100644 scipion_home/config/backup/scipion.conf.20260513164709 create mode 100644 scipion_home/config/backup/scipion.conf.20260513165026 create mode 100644 scipion_home/config/backup/scipion.conf.20260513165138 create mode 100644 scipion_home/config/backup/scipion.conf.20260514101059 create mode 100644 scipion_home/config/backup/scipion.conf.20260519133809 create mode 100644 scipion_home/config/backup/scipion.conf.20260520151542 create mode 100644 scipion_home/config/backup/scipion.conf.20260520154051 create mode 100644 scipion_home/config/backup/scipion.conf.20260520154226 create mode 100644 scipion_home/config/backup/scipion.conf.20260520154544 create mode 100644 scipion_home/config/backup/scipion.conf.20260520161814 create mode 100644 scipion_home/config/backup/scipion.conf.20260603210905 create mode 100644 scipion_home/config/backup/scipion.conf.20260604132329 create mode 100644 scipion_home/config/backup/scipion.conf.20260604133119 create mode 100644 scipion_home/config/hosts (Copy).conf create mode 100644 scipion_home/config/hosts.conf create mode 100644 scipion_home/config/scipion.conf create mode 100644 scipion_home/config/scipionweb_environment.json create mode 120000 scipion_home/software create mode 100644 scipion_home/web/devel_plugins.json diff --git a/.backend_reload_marker b/.backend_reload_marker new file mode 100644 index 00000000..b79513fc --- /dev/null +++ b/.backend_reload_marker @@ -0,0 +1 @@ +reloadMarker=1780572679.3509667 diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index af35c4ae..02564672 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,7 +41,11 @@ SELF_LABEL = "self" +<<<<<<< HEAD NESTED_LOGICAL_TABLES_VERSION = 8 +======= +NESTED_LOGICAL_TABLES_VERSION = 9 +>>>>>>> parent of 8045c89 (serve integrated tilt series from tomogram context) class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): @@ -1386,6 +1390,128 @@ def _getLinkedTomogramsSummary(self, scipionSet: Any) -> List[Dict[str, Any]]: return tomograms +<<<<<<< HEAD +======= + def _getLinkedTiltSeriesSummary(self, scipionSet: Any) -> Optional[Dict[str, Any]]: + tiltSeriesSet = self._callOptionalGetter(scipionSet, "getSetOfTiltSeries") + if tiltSeriesSet is None: + return None + + className = self._getClassName(tiltSeriesSet) + objectId = self._callOptionalGetter(tiltSeriesSet, "getObjId") + fileName = self._callOptionalGetter(tiltSeriesSet, "getFileName") + size = self._callOptionalGetter(tiltSeriesSet, "getSize") + + itemSummaries = [] + tsIds = [] + + for index, tiltSeries in enumerate(self._iterTiltSeriesItems(tiltSeriesSet)): + item = self._buildLinkedTiltSeriesItemSummary(tiltSeries, index) + if item is None: + continue + + itemSummaries.append(item) + + tiltSeriesId = item.get("tiltSeriesId") or item.get("tsId") or item.get("id") + if tiltSeriesId is not None: + tsIds.append(str(tiltSeriesId)) + + summary: Dict[str, Any] = { + "className": className, + "objectId": str(objectId) if objectId is not None else None, + "fileName": str(fileName) if fileName else None, + "size": self._toOptionalInt(size), + "tsIds": self._uniqueTextValues(tsIds), + "items": itemSummaries, + } + + return { + key: value + for key, value in summary.items() + if value not in (None, [], "") + } + + def _iterTiltSeriesItems(self, tiltSeriesSet: Any) -> Iterable[Any]: + iterItems = getattr(tiltSeriesSet, "iterItems", None) + if callable(iterItems): + try: + return iterItems(iterate=False) + except TypeError: + return iterItems() + except Exception: + pass + + try: + return iter(tiltSeriesSet) + except Exception: + return iter(()) + + def _buildLinkedTiltSeriesItemSummary(self, tiltSeries: Any, index: int) -> Optional[Dict[str, Any]]: + objectId = self._callOptionalGetter(tiltSeries, "getObjId") + tsId = ( + self._callOptionalGetter(tiltSeries, "getTsId") + or self._callOptionalGetter(tiltSeries, "getTSId") + or objectId + or index + ) + + if tsId is None: + return None + + label = ( + self._callOptionalGetter(tiltSeries, "getObjLabel") + or self._callOptionalGetter(tiltSeries, "getNameId") + or tsId + ) + + item: Dict[str, Any] = { + "id": str(tsId), + "tsId": str(tsId), + "tiltSeriesId": str(tsId), + "label": str(label), + } + + if objectId is not None: + item["objectId"] = str(objectId) + + dims = self._normalizeLinkedTomogramDims( + self._callOptionalGetter(tiltSeries, "getDim") + ) + if dims is not None: + item["dims"] = dims + + samplingRate = self._toOptionalFloat( + self._callOptionalGetter(tiltSeries, "getSamplingRate") + ) + if samplingRate is not None: + item["pixelSize"] = samplingRate + + tiltAxisAngle = self._toOptionalFloat( + self._callOptionalGetter( + self._callOptionalGetter(tiltSeries, "getAcquisition"), + "getTiltAxisAngle", + ) + ) + if tiltAxisAngle is not None: + item["tiltAxisAngle"] = tiltAxisAngle + + return item + + def _uniqueTextValues(self, values: Iterable[Any]) -> List[str]: + result = [] + seen = set() + + for value in values or []: + text = str(value).strip() if value is not None else "" + if not text or text in seen: + continue + + seen.add(text) + result.append(text) + + return result + +>>>>>>> parent of 8045c89 (serve integrated tilt series from tomogram context) def _iterLinkedTomograms(self, scipionSet: Any) -> Iterable[Any]: for methodName in ("iterTomograms", "iterVolumes"): iteratorGetter = getattr(scipionSet, methodName, None) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 64591a83..c48c1e92 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -348,6 +348,220 @@ def _addCoordinates3dRelations( label=label, ) + def _getLinkedTiltSeriesFromProperties(self, storedSet: Dict[str, Any]) -> Optional[Dict[str, Any]]: + properties = storedSet.get("properties") or {} + + if isinstance(properties, str): + try: + properties = json.loads(properties) + except Exception: + properties = {} + + linkedTiltSeries = properties.get("linkedTiltSeries") if isinstance(properties, dict) else None + if isinstance(linkedTiltSeries, dict): + return linkedTiltSeries + + for item in storedSet.get("setProperties") or []: + if str(item.get("key")) != "linkedTiltSeries": + continue + + value = item.get("value") + if isinstance(value, dict): + return value + + if isinstance(value, str): + try: + parsed = json.loads(value) + except Exception: + parsed = None + + if isinstance(parsed, dict): + return parsed + + return None + + def _listTiltSeriesStoredSetCandidates(self) -> List[Dict[str, Any]]: + try: + rows = self.db.fetchAll( + """ + SELECT + s.id, + s."projectId", + s."protocolDbId", + s."objectId", + s."outputName", + s."setClassName", + s."itemClassName", + s.properties, + s."createdAt", + s."updatedAt", + p."protocolId" AS "publicProtocolId", + p."protocolClassName" + FROM scipion_sets s + JOIN protocols p + ON p.id = s."protocolDbId" + AND p."projectId" = s."projectId" + WHERE s."projectId" = %s + AND ( + LOWER(COALESCE(s."setClassName", '')) LIKE '%%setoftiltseries%%' + OR LOWER(COALESCE(s."itemClassName", '')) LIKE '%%tiltseries%%' + ) + ORDER BY s."protocolDbId" ASC, s."outputName" ASC + """, + (self.projectId,), + ) + except Exception: + logger.debug( + "Could not list PostgreSQL tilt-series candidates for integrated context. projectId=%s", + self.projectId, + exc_info=True, + ) + return [] + + return [dict(row) for row in rows or []] + + def _getStoredSetPropertiesDict(self, storedSet: Dict[str, Any]) -> Dict[str, Any]: + properties = storedSet.get("properties") or {} + + if isinstance(properties, str): + try: + properties = json.loads(properties) + except Exception: + properties = {} + + return properties if isinstance(properties, dict) else {} + + def _scoreLinkedTiltSeriesCandidate( + self, + candidate: Dict[str, Any], + linkedTiltSeries: Dict[str, Any], + relationKeys: Optional[set], + ) -> int: + score = 0 + candidateProperties = self._getStoredSetPropertiesDict(candidate) + + linkedObjectId = linkedTiltSeries.get("objectId") + candidateObjectIds = [ + candidate.get("objectId"), + candidateProperties.get("scipionObjId"), + candidateProperties.get("objectId"), + ] + + if linkedObjectId is not None and any(str(value) == str(linkedObjectId) for value in candidateObjectIds if value is not None): + score += 1000 + + linkedFileName = linkedTiltSeries.get("fileName") + candidateFileName = candidateProperties.get("fileName") + if linkedFileName and candidateFileName and str(linkedFileName) == str(candidateFileName): + score += 300 + + linkedTsIds = { + str(value).strip() + for value in linkedTiltSeries.get("tsIds") or [] + if value is not None and str(value).strip() + } + + if relationKeys: + linkedTsIds.update( + str(value).strip() + for value in relationKeys + if value is not None and str(value).strip() + ) + + if linkedTsIds: + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=candidate.get("protocolDbId"), + outputName=candidate.get("outputName"), + ) + candidateItems = reader.listTiltSeries() + candidateTsIds = { + str(item.get("tiltSeriesId") or item.get("tsId") or item.get("id")).strip() + for item in candidateItems or [] + if str(item.get("tiltSeriesId") or item.get("tsId") or item.get("id") or "").strip() + } + + intersection = candidateTsIds.intersection(linkedTsIds) + if intersection: + score += 10 * len(intersection) + + return score + + def _resolveLinkedTiltSeriesStoredSet( + self, + linkedTiltSeries: Optional[Dict[str, Any]], + relationKeys: Optional[set], + ) -> Optional[Dict[str, Any]]: + if not linkedTiltSeries: + return None + + bestCandidate = None + bestScore = 0 + + for candidate in self._listTiltSeriesStoredSetCandidates(): + score = self._scoreLinkedTiltSeriesCandidate( + candidate=candidate, + linkedTiltSeries=linkedTiltSeries, + relationKeys=relationKeys, + ) + + if score > bestScore: + bestScore = score + bestCandidate = candidate + + return bestCandidate + + def _mergeTomogramLinkedTiltSeries( + self, + storedSet: Dict[str, Any], + links: Dict[str, Optional[Dict[str, Any]]], + summaries: Dict[str, Optional[Dict[str, Any]]], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> None: + linkedTiltSeries = self._getLinkedTiltSeriesFromProperties(storedSet) + relationKeys = set(relationsByKey.keys()) + + candidate = self._resolveLinkedTiltSeriesStoredSet( + linkedTiltSeries=linkedTiltSeries, + relationKeys=relationKeys, + ) + + if candidate is None: + return + + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=candidate.get("protocolDbId"), + outputName=candidate.get("outputName"), + ) + + items = self._filterIntegratedItemsByAllowedKeys( + reader.listTiltSeries(), + relationKeys, + ) + + if not items: + return + + self._addTiltSeriesRelations(relationsByKey, items) + + links["tiltSeries"] = self._buildLink( + protocolId=self._getCandidateProtocolId(candidate), + outputName=candidate.get("outputName"), + storedSet=candidate, + statusValue="derived", + ) + links["tiltSeries"]["source"] = "tomogram" + + summaries["tiltSeries"] = self._buildSummary( + candidate, + extra={ + "source": "tomogram", + }, + ) + def _mergeRootRelations( self, rootKind: str, @@ -562,6 +776,24 @@ def _mergeRelatedStoredSets( if rootKind == "coordinates3d" and candidateKind == "tomogram": continue + if rootKind == "tomogram" and candidateKind == "tiltSeries" and links.get("tiltSeries") is not None: + continue + + allowedRelationKeys = self._getAllowedRelationKeysForRoot( + rootKind=rootKind, + relationsByKey=relationsByKey, + ) + + mergedRelations = self._mergeRelationsForCandidate( + candidate=candidate, + candidateKind=candidateKind, + relationsByKey=relationsByKey, + allowedRelationKeys=allowedRelationKeys, + ) + + if not mergedRelations: + continue + if self._shouldReplaceLink(links.get(candidateKind)): links[candidateKind] = self._buildLink( protocolId=self._getCandidateProtocolId(candidate), diff --git a/dump.rdb b/dump.rdb new file mode 100644 index 0000000000000000000000000000000000000000..052f3288d644aa93b20a4d7fa6d38c01982b9cdf GIT binary patch literal 771 zcmaiy&2G~`6os8A4G9a{0&Q4SVsLlBSo7zf1PLrk7wk|2tEz;V@wgquaUzc$RH3eT z4&ViN2dW~T015F7Jc6kJrx8Wa0;{=mbnd-#KF!wVgNJ>DFe=-binM`Kl{I|_IoK^P z&K03^a~PHjGS?$@^mP4$E?b3B5ge(}QR`jDDqDOUL&1$Yw7ywi8};LtZ#p+$)hx@x zQY9+ehdDRH%Aa!>2gr*lqBIIz=1LmGK}1;4WlDG=j&b7=47sl=ZyZ>ZkW3%74 zZv#71rbzOc_$VoMbqb8s6Fp9$8bLmpRY0dzXOgH0m_kZrOk5<%{*768o^h!Jm&0m6 zTB(&J_jD3vDy@|5o$4O(Tn(aes%AR~A|6B(68AACJ%W3LIV_-rvRerC5IUJzg&OE+ zX8&I*``~!5aqX{6JcWIqdNCF*L(;3>>2gd1D!Z{DJboX33_q{^lS_E@%6VAAb^{b? z1?$f8SuWwd)+rWn-%7y>*t+nO3Rtdf?|{YDDHX7E{VWw1ITRC$@Cg+QJ?k&bp@hz2 z{6{LFFa4IvKB%t@hU1Ya9HG-lr@PMlx73#AXLGf=IpmZ6LzOw}zszdt mU`CMZGYm0~0?4>4p+F*lzN@fHn8yV1zfbPAFRy)h{q_e$eC~h% literal 0 HcmV?d00001 diff --git a/scipion_home/.plugins_revision b/scipion_home/.plugins_revision new file mode 100644 index 00000000..a6b4ce84 --- /dev/null +++ b/scipion_home/.plugins_revision @@ -0,0 +1 @@ +176 \ No newline at end of file diff --git a/scipion_home/.plugins_revision.lock b/scipion_home/.plugins_revision.lock new file mode 100644 index 00000000..e69de29b diff --git a/scipion_home/config/backup/scipion.conf.20260307213812 b/scipion_home/config/backup/scipion.conf.20260307213812 new file mode 100644 index 00000000..9af63a92 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260307213812 @@ -0,0 +1,25 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD = eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" + +CUDA_BIN = /usr/local/cuda-11.8/bin +CUDA_LIB = /usr/local/cuda-11.8/lib64 +MPI_BINDIR = /usr/lib64/mpi/gcc/openmpi/bin +MPI_LIBDIR = /usr/lib64/mpi/gcc/openmpi/lib +MPI_INCLUDE = /usr/lib64/mpi/gcc/openmpi/include +OPENCV = False +CUDA=True + + +[PLUGINS] +CRYOSPARC_HOME = /home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_USER = cfonseca@cnb.csic.es +CRYOSPARC_PASSWORD = 12345 +RELION_HOME = /home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME = /home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELION_ENV_ACTIVATION = conda activate relion-5.0 +#CHIMERA_HOME = /home/yunior/Yunior/Projects/Scipion/software/em/chimerax-1.4 +SCIPION_DEFAULT_EXECUTION_ACTION = 1 +#SCIPION_SPRITES_FILE = /home/yunior/Imágenes/sprites_16.png +#SCIPION_GUI_CANCEL_AUTO_REFRESH = True +WARP_FORCE_MRC_FLOAT32=1 + diff --git a/scipion_home/config/backup/scipion.conf.20260307214508 b/scipion_home/config/backup/scipion.conf.20260307214508 new file mode 100644 index 00000000..9af63a92 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260307214508 @@ -0,0 +1,25 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD = eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" + +CUDA_BIN = /usr/local/cuda-11.8/bin +CUDA_LIB = /usr/local/cuda-11.8/lib64 +MPI_BINDIR = /usr/lib64/mpi/gcc/openmpi/bin +MPI_LIBDIR = /usr/lib64/mpi/gcc/openmpi/lib +MPI_INCLUDE = /usr/lib64/mpi/gcc/openmpi/include +OPENCV = False +CUDA=True + + +[PLUGINS] +CRYOSPARC_HOME = /home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_USER = cfonseca@cnb.csic.es +CRYOSPARC_PASSWORD = 12345 +RELION_HOME = /home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME = /home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELION_ENV_ACTIVATION = conda activate relion-5.0 +#CHIMERA_HOME = /home/yunior/Yunior/Projects/Scipion/software/em/chimerax-1.4 +SCIPION_DEFAULT_EXECUTION_ACTION = 1 +#SCIPION_SPRITES_FILE = /home/yunior/Imágenes/sprites_16.png +#SCIPION_GUI_CANCEL_AUTO_REFRESH = True +WARP_FORCE_MRC_FLOAT32=1 + diff --git a/scipion_home/config/backup/scipion.conf.20260307215448 b/scipion_home/config/backup/scipion.conf.20260307215448 new file mode 100644 index 00000000..b2cbfa3a --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260307215448 @@ -0,0 +1,11 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260307215946 b/scipion_home/config/backup/scipion.conf.20260307215946 new file mode 100644 index 00000000..899c67d6 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260307215946 @@ -0,0 +1,12 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +IMOD_VIEWER_BINNING=2 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260307225850 b/scipion_home/config/backup/scipion.conf.20260307225850 new file mode 100644 index 00000000..899c67d6 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260307225850 @@ -0,0 +1,12 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +IMOD_VIEWER_BINNING=2 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260320071727 b/scipion_home/config/backup/scipion.conf.20260320071727 new file mode 100644 index 00000000..d7f4cfe1 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260320071727 @@ -0,0 +1,13 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +JAVA_MAX_MEMORY=8 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +IMOD_VIEWER_BINNING=2 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260320071826 b/scipion_home/config/backup/scipion.conf.20260320071826 new file mode 100644 index 00000000..cfda30d4 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260320071826 @@ -0,0 +1,13 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +IMOD_VIEWER_BINNING=2 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOLO_CUDA_LIB=/usr/local/cuda12.4/lib64 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260415111435 b/scipion_home/config/backup/scipion.conf.20260415111435 new file mode 100644 index 00000000..ba389111 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260415111435 @@ -0,0 +1,13 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +IMOD_VIEWER_BINNING=2 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOLO_CUDA_LIB=/usr/local/cuda-12.4/lib64 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260415111706 b/scipion_home/config/backup/scipion.conf.20260415111706 new file mode 100644 index 00000000..77bde170 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260415111706 @@ -0,0 +1,14 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_PRIORITY_PACKAGE_LIST="pwem tomo pwchem" + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_VIEWER_BINNING=2 +CRYOLO_CUDA_LIB=/usr/local/cuda-12.4/lib64 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260416211619 b/scipion_home/config/backup/scipion.conf.20260416211619 new file mode 100644 index 00000000..fe4f6c6c --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260416211619 @@ -0,0 +1,13 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_VIEWER_BINNING=2 +CRYOLO_CUDA_LIB=/usr/local/cuda-12.4/lib64 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260421114030 b/scipion_home/config/backup/scipion.conf.20260421114030 new file mode 100644 index 00000000..70fb22c7 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421114030 @@ -0,0 +1,13 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home + +[PLUGINS] +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOLO_CUDA_LIB=/usr/local/cuda-12.4/lib64 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_VIEWER_BINNING=2 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260421114143 b/scipion_home/config/backup/scipion.conf.20260421114143 new file mode 100644 index 00000000..d0222551 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421114143 @@ -0,0 +1,16 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_VIEWER_BINNING=2 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +CRYOLO_CUDA_LIB=/usr/local/cuda-12.4/lib64 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260421114423 b/scipion_home/config/backup/scipion.conf.20260421114423 new file mode 100644 index 00000000..556be06f --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421114423 @@ -0,0 +1,9 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] diff --git a/scipion_home/config/backup/scipion.conf.20260421114636 b/scipion_home/config/backup/scipion.conf.20260421114636 new file mode 100644 index 00000000..829a3fe5 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421114636 @@ -0,0 +1,10 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +CHIMERA_OLD_BINARY_PATH=/home/yun diff --git a/scipion_home/config/backup/scipion.conf.20260421115104 b/scipion_home/config/backup/scipion.conf.20260421115104 new file mode 100644 index 00000000..556be06f --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421115104 @@ -0,0 +1,9 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] diff --git a/scipion_home/config/backup/scipion.conf.20260421115119 b/scipion_home/config/backup/scipion.conf.20260421115119 new file mode 100644 index 00000000..7721e45f --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421115119 @@ -0,0 +1,10 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +ATSAS_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/atsas-3.2.2 diff --git a/scipion_home/config/backup/scipion.conf.20260421122630 b/scipion_home/config/backup/scipion.conf.20260421122630 new file mode 100644 index 00000000..16d442ff --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421122630 @@ -0,0 +1,10 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +ATSAS_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/atsas-3.2.1 diff --git a/scipion_home/config/backup/scipion.conf.20260421122638 b/scipion_home/config/backup/scipion.conf.20260421122638 new file mode 100644 index 00000000..e55aa3df --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421122638 @@ -0,0 +1,12 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +ATSAS_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/atsas-3.2.1 +ARETOMO_BIN=AreTomo2_1.1.3_Cuda122 diff --git a/scipion_home/config/backup/scipion.conf.20260421124315 b/scipion_home/config/backup/scipion.conf.20260421124315 new file mode 100644 index 00000000..9af63a92 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421124315 @@ -0,0 +1,25 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD = eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" + +CUDA_BIN = /usr/local/cuda-11.8/bin +CUDA_LIB = /usr/local/cuda-11.8/lib64 +MPI_BINDIR = /usr/lib64/mpi/gcc/openmpi/bin +MPI_LIBDIR = /usr/lib64/mpi/gcc/openmpi/lib +MPI_INCLUDE = /usr/lib64/mpi/gcc/openmpi/include +OPENCV = False +CUDA=True + + +[PLUGINS] +CRYOSPARC_HOME = /home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_USER = cfonseca@cnb.csic.es +CRYOSPARC_PASSWORD = 12345 +RELION_HOME = /home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME = /home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELION_ENV_ACTIVATION = conda activate relion-5.0 +#CHIMERA_HOME = /home/yunior/Yunior/Projects/Scipion/software/em/chimerax-1.4 +SCIPION_DEFAULT_EXECUTION_ACTION = 1 +#SCIPION_SPRITES_FILE = /home/yunior/Imágenes/sprites_16.png +#SCIPION_GUI_CANCEL_AUTO_REFRESH = True +WARP_FORCE_MRC_FLOAT32=1 + diff --git a/scipion_home/config/backup/scipion.conf.20260421124338 b/scipion_home/config/backup/scipion.conf.20260421124338 new file mode 100644 index 00000000..c1930fd9 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260421124338 @@ -0,0 +1,17 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=123456 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260422190617 b/scipion_home/config/backup/scipion.conf.20260422190617 new file mode 100644 index 00000000..eca1f6a8 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260422190617 @@ -0,0 +1,17 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260513164709 b/scipion_home/config/backup/scipion.conf.20260513164709 new file mode 100644 index 00000000..7e21c530 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260513164709 @@ -0,0 +1,18 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/imod-4.11.25/IMOD +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260513165026 b/scipion_home/config/backup/scipion.conf.20260513165026 new file mode 100644 index 00000000..0804695a --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260513165026 @@ -0,0 +1,18 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/Scipion/software/em/imod-4.11.25/IMOD diff --git a/scipion_home/config/backup/scipion.conf.20260513165138 b/scipion_home/config/backup/scipion.conf.20260513165138 new file mode 100644 index 00000000..d8edc6a1 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260513165138 @@ -0,0 +1,18 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD/bin diff --git a/scipion_home/config/backup/scipion.conf.20260514101059 b/scipion_home/config/backup/scipion.conf.20260514101059 new file mode 100644 index 00000000..0a836c93 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260514101059 @@ -0,0 +1,18 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD diff --git a/scipion_home/config/backup/scipion.conf.20260519133809 b/scipion_home/config/backup/scipion.conf.20260519133809 new file mode 100644 index 00000000..ad7ba238 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260519133809 @@ -0,0 +1,19 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CHIMERA_HOME=/home/yunior/Yunior/Projects/Scipion/software/em/chimerax-1.6.1 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260520151542 b/scipion_home/config/backup/scipion.conf.20260520151542 new file mode 100644 index 00000000..564e33e7 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260520151542 @@ -0,0 +1,18 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260520154051 b/scipion_home/config/backup/scipion.conf.20260520154051 new file mode 100644 index 00000000..b12f53e7 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260520154051 @@ -0,0 +1,19 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +CHIMERA_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/chimerax-1.6.1 diff --git a/scipion_home/config/backup/scipion.conf.20260520154226 b/scipion_home/config/backup/scipion.conf.20260520154226 new file mode 100644 index 00000000..8aed58e7 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260520154226 @@ -0,0 +1,20 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +CHIMERA_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/chimerax-1.6.1 diff --git a/scipion_home/config/backup/scipion.conf.20260520154544 b/scipion_home/config/backup/scipion.conf.20260520154544 new file mode 100644 index 00000000..3417393a --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260520154544 @@ -0,0 +1,20 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +CHIMERA_OLD_BINARY_PATH=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home/software/em/chimerax-1.6.1/bin +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es diff --git a/scipion_home/config/backup/scipion.conf.20260520161814 b/scipion_home/config/backup/scipion.conf.20260520161814 new file mode 100644 index 00000000..99510a28 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260520161814 @@ -0,0 +1,20 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CHIMERA_HOME=/home/yunior/Yunior/Projects/Scipion/software/em/chimerax-1.6.1 \ No newline at end of file diff --git a/scipion_home/config/backup/scipion.conf.20260603210905 b/scipion_home/config/backup/scipion.conf.20260603210905 new file mode 100644 index 00000000..ae7e778a --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260603210905 @@ -0,0 +1,19 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/backup/scipion.conf.20260604132329 b/scipion_home/config/backup/scipion.conf.20260604132329 new file mode 100644 index 00000000..73d1bbf9 --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260604132329 @@ -0,0 +1,20 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +CHIMERA_HOME=/home/yunior/Yunior/Projects/Scipion/software/em/chimerax-1.6.1/ diff --git a/scipion_home/config/backup/scipion.conf.20260604133119 b/scipion_home/config/backup/scipion.conf.20260604133119 new file mode 100644 index 00000000..3c66f08c --- /dev/null +++ b/scipion_home/config/backup/scipion.conf.20260604133119 @@ -0,0 +1,21 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +NEXTPYP_HOME=/home/yunior/Yunior/Projects/nextPYP/nextPYP_0.7.3/config.toml +NEXTPYP_CONFIG=/home/yunior/Yunior/Projects/nextPYP/nextPYP_0.7.3 +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/hosts (Copy).conf b/scipion_home/config/hosts (Copy).conf new file mode 100644 index 00000000..24c37002 --- /dev/null +++ b/scipion_home/config/hosts (Copy).conf @@ -0,0 +1,46 @@ +; This is a comment line +[localhost] +PARALLEL_COMMAND = mpirun -np %_(JOB_NODES)d %_(COMMAND)s +NAME = PBS/TORQUE +MANDATORY = False +SUBMIT_COMMAND = qsub %_(JOB_SCRIPT)s +SUBMIT_TEMPLATE = #!/bin/bash + ### Inherit all current environment variables + #PBS -V + ### Job name + #PBS -N %_(JOB_NAME)s + ### Queue name + ###PBS -q %_(JOB_QUEUE)s + ### Standard output and standard error messages + #PBS -k eo + ### Specify the number of nodes and thread (ppn) for your job. + #PBS -l nodes=%_(JOB_NODES)d:ppn=%_(JOB_THREADS)d + ### Tell PBS the anticipated run-time for your job, where walltime=HH:MM:SS + #PBS -l walltime=%_(JOB_HOURS)d:00:00 + # Use as working dir the path where qsub was launched + WORKDIR=$PBS_O_WORKDIR + ################################# + ### Set environment variable to know running mode is non interactive + export XMIPP_IN_QUEUE=1 + ### Switch to the working directory; + cd $WORKDIR + # Make a copy of PBS_NODEFILE + cp $PBS_NODEFILE %_(JOB_NODEFILE)s + # Calculate the number of processors allocated to this run. + NPROCS=`wc -l < $PBS_NODEFILE` + # Calculate the number of nodes allocated. + NNODES=`uniq $PBS_NODEFILE | wc -l` + ### Display the job context + echo Running on host `hostname` + echo Time is `date` + echo Working directory is `pwd` + echo Using ${NPROCS} processors across ${NNODES} nodes + echo PBS_NODEFILE: + cat $PBS_NODEFILE + ################################# + %_(JOB_COMMAND)s +CANCEL_COMMAND = canceljob %_(JOB_ID)s +CHECK_COMMAND = qstat %_(JOB_ID)s +; Next variable is used to provide a regex to check if a job is finished on a queue system +JOB_DONE_REGEX = "" +QUEUES = { "default": {} } diff --git a/scipion_home/config/hosts.conf b/scipion_home/config/hosts.conf new file mode 100644 index 00000000..27a580db --- /dev/null +++ b/scipion_home/config/hosts.conf @@ -0,0 +1,30 @@ +[localhost] +PARALLEL_COMMAND = mpirun -np %_(JOB_NODES)d %_(COMMAND)s +NAME = SLURM +MANDATORY = False +SUBMIT_COMMAND = sbatch --parsable %_(JOB_SCRIPT)s +CANCEL_COMMAND = scancel %_(JOB_ID)s +CHECK_COMMAND = squeue -h -j %_(JOB_ID)s +SUBMIT_PREFIX = scipion +SUBMIT_TEMPLATE = #!/bin/bash + ### Inherit all current environment variables + #SBATCH --export=ALL + ### Job name + #SBATCH -J %_(JOB_NAME)s + ### Outputs + #SBATCH -o %_(JOB_SCRIPT)s.out + #SBATCH -e %_(JOB_SCRIPT)s.err + #SBATCH --open-mode=append + ### Partition (queue) name + #SBATCH -p %_(JOB_QUEUE)s + ### Specify time, number of nodes (tasks), cores and memory(MB) for your job + #SBATCH --time=%_(JOB_TIME)s:00:00 --ntasks=%_(JOB_NODES)d --cpus-per-task=%_(JOB_THREADS)d --mem=%_(JOB_MEMORY)s --gres=gpu:%_(GPU_COUNT)s + + %_(JOB_COMMAND)s + +QUEUES = { + "debug": [["JOB_MEMORY", "8192", "Memory (MB)", "Select amount of memory in megabytes for this job"], + ["JOB_TIME", "2", "Time (hours)", "Select the expected time in hours for this job"], + ["GPU_COUNT", "1", "Number of GPUs", "Select number of GPUs for this job"], + ["QUEUE_FOR_JOBS", "N", "Use queue for jobs", "Send individual jobs to queue"]] + } diff --git a/scipion_home/config/scipion.conf b/scipion_home/config/scipion.conf new file mode 100644 index 00000000..df60dfb8 --- /dev/null +++ b/scipion_home/config/scipion.conf @@ -0,0 +1,21 @@ +[PYWORKFLOW] +CONDA_ACTIVATION_CMD=eval "$(/home/yunior/miniconda3/bin/conda shell.bash hook)" +CUDA_BIN=/usr/local/cuda-11.8/bin +CUDA_LIB=/usr/local/cuda-11.8/lib64 +JAVA_HOME=/home/yunior/miniconda3/envs/scipion3Web/lib/jvm +SCIPION_DOMAIN=pwem +SCIPION_HOME=/home/yunior/Yunior/Projects/ScipionWeb/ScipionAPI/scipion_home +SCIPION_LOCAL_CONFIG=/home/yunior/.config/scipion/scipion.conf +SCIPION_PRIORITY_PACKAGE_LIST=pwem tomo pwchem +SCIPION_USER_DATA=/home/yunior/ScipionUserData + +[PLUGINS] +NEXTPYP_HOME=/home/yunior/Yunior/Projects/nextPYP/nextPYP_0.7.3 +NEXTPYP_CONFIG=/home/yunior/Yunior/Projects/nextPYP/nextPYP_0.7.3/config.toml +IMOD_HOME=/home/yunior/Yunior/Projects/ScipionWeb/scipion/software/em/imod-5.1.3/IMOD +BRT_ENV_ACTIVATION=conda activate teamtomoBRT-0.1.3 +CRYOSPARC_HOME=/home/yunior/Yunior/Projects/cryoSPARC +CRYOSPARC_PASSWORD=12345 +CRYOSPARC_USER=cfonseca@cnb.csic.es +RELIONTOMO_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 +RELION_HOME=/home/yunior/Yunior/Projects/relion5/relion/relion-5.0 diff --git a/scipion_home/config/scipionweb_environment.json b/scipion_home/config/scipionweb_environment.json new file mode 100644 index 00000000..45f9686a --- /dev/null +++ b/scipion_home/config/scipionweb_environment.json @@ -0,0 +1,4 @@ +{ + "PEPE_HOME": "/home/pepe_app", + "YUNIOR": "yunior/yun" +} diff --git a/scipion_home/software b/scipion_home/software new file mode 120000 index 00000000..22f6a51d --- /dev/null +++ b/scipion_home/software @@ -0,0 +1 @@ +/home/yunior/Yunior/Projects/Scipion/software/ \ No newline at end of file diff --git a/scipion_home/web/devel_plugins.json b/scipion_home/web/devel_plugins.json new file mode 100644 index 00000000..9a03260e --- /dev/null +++ b/scipion_home/web/devel_plugins.json @@ -0,0 +1,106 @@ +[ + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-aretomo", + "pipName": "scipion-em-aretomo", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-cistem", + "pipName": "scipion-em-cistem", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-cryosparc2", + "pipName": "scipion-em-cryosparc2", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-miffi", + "pipName": "scipion-em-miffi", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-motioncorr", + "pipName": "scipion-em-motioncorr", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-nextpyp", + "pipName": "scipion-em-nextpyp", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-relion", + "pipName": "scipion-em-relion", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-reliontomo", + "pipName": "scipion-em-reliontomo", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-sphire", + "pipName": "scipion-em-sphire", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-tomo", + "pipName": "scipion-em-tomo", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-warp", + "pipName": "scipion-em-warp", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-warp", + "pipName": "scipion-em-warp", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + }, + { + "installedAt": "2026-06-03T15:09:02.122009+00:00", + "mode": "devel", + "path": "/home/yunior/Yunior/Projects/Plugins/scipion-em-xmipp", + "pipName": "scipion-em-xmipp", + "taskId": "aa00ee92515c437a86a910d024dd4d75", + "updatedAt": "2026-06-03T15:09:02.122009+00:00" + } +] \ No newline at end of file From 601cbcf93eaef68bfba5b8aeaa2b96990dc36a7a Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 11:24:17 +0200 Subject: [PATCH 076/278] solving more conflicts --- app/backend/mapper/scipion_set_mapper.py | 11 +- .../postgresql_integrated_context_reader.py | 217 ------------------ 2 files changed, 4 insertions(+), 224 deletions(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index 02564672..cec9a953 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,11 +41,7 @@ SELF_LABEL = "self" -<<<<<<< HEAD -NESTED_LOGICAL_TABLES_VERSION = 8 -======= NESTED_LOGICAL_TABLES_VERSION = 9 ->>>>>>> parent of 8045c89 (serve integrated tilt series from tomogram context) class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): @@ -1373,6 +1369,10 @@ def _getSetProperties(self, scipionSet: Any) -> Dict[str, Any]: if linkedTomograms: properties["linkedTomograms"] = linkedTomograms + linkedTiltSeries = self._getLinkedTiltSeriesSummary(scipionSet) + if linkedTiltSeries: + properties["linkedTiltSeries"] = linkedTiltSeries + for attrName, attrValue in self._getAttributesToStore(scipionSet): if self._getAttributesToStore(attrValue): continue @@ -1390,8 +1390,6 @@ def _getLinkedTomogramsSummary(self, scipionSet: Any) -> List[Dict[str, Any]]: return tomograms -<<<<<<< HEAD -======= def _getLinkedTiltSeriesSummary(self, scipionSet: Any) -> Optional[Dict[str, Any]]: tiltSeriesSet = self._callOptionalGetter(scipionSet, "getSetOfTiltSeries") if tiltSeriesSet is None: @@ -1511,7 +1509,6 @@ def _uniqueTextValues(self, values: Iterable[Any]) -> List[str]: return result ->>>>>>> parent of 8045c89 (serve integrated tilt series from tomogram context) def _iterLinkedTomograms(self, scipionSet: Any) -> Iterable[Any]: for methodName in ("iterTomograms", "iterVolumes"): iteratorGetter = getattr(scipionSet, methodName, None) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index c48c1e92..89c9a3fd 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -348,220 +348,6 @@ def _addCoordinates3dRelations( label=label, ) - def _getLinkedTiltSeriesFromProperties(self, storedSet: Dict[str, Any]) -> Optional[Dict[str, Any]]: - properties = storedSet.get("properties") or {} - - if isinstance(properties, str): - try: - properties = json.loads(properties) - except Exception: - properties = {} - - linkedTiltSeries = properties.get("linkedTiltSeries") if isinstance(properties, dict) else None - if isinstance(linkedTiltSeries, dict): - return linkedTiltSeries - - for item in storedSet.get("setProperties") or []: - if str(item.get("key")) != "linkedTiltSeries": - continue - - value = item.get("value") - if isinstance(value, dict): - return value - - if isinstance(value, str): - try: - parsed = json.loads(value) - except Exception: - parsed = None - - if isinstance(parsed, dict): - return parsed - - return None - - def _listTiltSeriesStoredSetCandidates(self) -> List[Dict[str, Any]]: - try: - rows = self.db.fetchAll( - """ - SELECT - s.id, - s."projectId", - s."protocolDbId", - s."objectId", - s."outputName", - s."setClassName", - s."itemClassName", - s.properties, - s."createdAt", - s."updatedAt", - p."protocolId" AS "publicProtocolId", - p."protocolClassName" - FROM scipion_sets s - JOIN protocols p - ON p.id = s."protocolDbId" - AND p."projectId" = s."projectId" - WHERE s."projectId" = %s - AND ( - LOWER(COALESCE(s."setClassName", '')) LIKE '%%setoftiltseries%%' - OR LOWER(COALESCE(s."itemClassName", '')) LIKE '%%tiltseries%%' - ) - ORDER BY s."protocolDbId" ASC, s."outputName" ASC - """, - (self.projectId,), - ) - except Exception: - logger.debug( - "Could not list PostgreSQL tilt-series candidates for integrated context. projectId=%s", - self.projectId, - exc_info=True, - ) - return [] - - return [dict(row) for row in rows or []] - - def _getStoredSetPropertiesDict(self, storedSet: Dict[str, Any]) -> Dict[str, Any]: - properties = storedSet.get("properties") or {} - - if isinstance(properties, str): - try: - properties = json.loads(properties) - except Exception: - properties = {} - - return properties if isinstance(properties, dict) else {} - - def _scoreLinkedTiltSeriesCandidate( - self, - candidate: Dict[str, Any], - linkedTiltSeries: Dict[str, Any], - relationKeys: Optional[set], - ) -> int: - score = 0 - candidateProperties = self._getStoredSetPropertiesDict(candidate) - - linkedObjectId = linkedTiltSeries.get("objectId") - candidateObjectIds = [ - candidate.get("objectId"), - candidateProperties.get("scipionObjId"), - candidateProperties.get("objectId"), - ] - - if linkedObjectId is not None and any(str(value) == str(linkedObjectId) for value in candidateObjectIds if value is not None): - score += 1000 - - linkedFileName = linkedTiltSeries.get("fileName") - candidateFileName = candidateProperties.get("fileName") - if linkedFileName and candidateFileName and str(linkedFileName) == str(candidateFileName): - score += 300 - - linkedTsIds = { - str(value).strip() - for value in linkedTiltSeries.get("tsIds") or [] - if value is not None and str(value).strip() - } - - if relationKeys: - linkedTsIds.update( - str(value).strip() - for value in relationKeys - if value is not None and str(value).strip() - ) - - if linkedTsIds: - reader = PostgresqlTiltSeriesReader( - db=self.db, - projectId=self.projectId, - protocolId=candidate.get("protocolDbId"), - outputName=candidate.get("outputName"), - ) - candidateItems = reader.listTiltSeries() - candidateTsIds = { - str(item.get("tiltSeriesId") or item.get("tsId") or item.get("id")).strip() - for item in candidateItems or [] - if str(item.get("tiltSeriesId") or item.get("tsId") or item.get("id") or "").strip() - } - - intersection = candidateTsIds.intersection(linkedTsIds) - if intersection: - score += 10 * len(intersection) - - return score - - def _resolveLinkedTiltSeriesStoredSet( - self, - linkedTiltSeries: Optional[Dict[str, Any]], - relationKeys: Optional[set], - ) -> Optional[Dict[str, Any]]: - if not linkedTiltSeries: - return None - - bestCandidate = None - bestScore = 0 - - for candidate in self._listTiltSeriesStoredSetCandidates(): - score = self._scoreLinkedTiltSeriesCandidate( - candidate=candidate, - linkedTiltSeries=linkedTiltSeries, - relationKeys=relationKeys, - ) - - if score > bestScore: - bestScore = score - bestCandidate = candidate - - return bestCandidate - - def _mergeTomogramLinkedTiltSeries( - self, - storedSet: Dict[str, Any], - links: Dict[str, Optional[Dict[str, Any]]], - summaries: Dict[str, Optional[Dict[str, Any]]], - relationsByKey: Dict[str, Dict[str, Any]], - ) -> None: - linkedTiltSeries = self._getLinkedTiltSeriesFromProperties(storedSet) - relationKeys = set(relationsByKey.keys()) - - candidate = self._resolveLinkedTiltSeriesStoredSet( - linkedTiltSeries=linkedTiltSeries, - relationKeys=relationKeys, - ) - - if candidate is None: - return - - reader = PostgresqlTiltSeriesReader( - db=self.db, - projectId=self.projectId, - protocolId=candidate.get("protocolDbId"), - outputName=candidate.get("outputName"), - ) - - items = self._filterIntegratedItemsByAllowedKeys( - reader.listTiltSeries(), - relationKeys, - ) - - if not items: - return - - self._addTiltSeriesRelations(relationsByKey, items) - - links["tiltSeries"] = self._buildLink( - protocolId=self._getCandidateProtocolId(candidate), - outputName=candidate.get("outputName"), - storedSet=candidate, - statusValue="derived", - ) - links["tiltSeries"]["source"] = "tomogram" - - summaries["tiltSeries"] = self._buildSummary( - candidate, - extra={ - "source": "tomogram", - }, - ) - def _mergeRootRelations( self, rootKind: str, @@ -776,9 +562,6 @@ def _mergeRelatedStoredSets( if rootKind == "coordinates3d" and candidateKind == "tomogram": continue - if rootKind == "tomogram" and candidateKind == "tiltSeries" and links.get("tiltSeries") is not None: - continue - allowedRelationKeys = self._getAllowedRelationKeysForRoot( rootKind=rootKind, relationsByKey=relationsByKey, From 5fb7280fcb13580f902c53fe582c0128aaba0974 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 11:24:22 +0200 Subject: [PATCH 077/278] Revert "match integrated tilt series by tomogram ids"" This reverts commit 278ce2017532bf55f8e488d25557e7c1323beacb. --- .../postgresql_integrated_context_reader.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 89c9a3fd..64591a83 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -562,21 +562,6 @@ def _mergeRelatedStoredSets( if rootKind == "coordinates3d" and candidateKind == "tomogram": continue - allowedRelationKeys = self._getAllowedRelationKeysForRoot( - rootKind=rootKind, - relationsByKey=relationsByKey, - ) - - mergedRelations = self._mergeRelationsForCandidate( - candidate=candidate, - candidateKind=candidateKind, - relationsByKey=relationsByKey, - allowedRelationKeys=allowedRelationKeys, - ) - - if not mergedRelations: - continue - if self._shouldReplaceLink(links.get(candidateKind)): links[candidateKind] = self._buildLink( protocolId=self._getCandidateProtocolId(candidate), From 753c70e42b541b9cc6dfe80945e2c0effd52b495 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 11:37:31 +0200 Subject: [PATCH 078/278] remove broken integrated tilt series inference --- app/backend/mapper/scipion_set_mapper.py | 124 +----------------- .../postgresql_integrated_context_reader.py | 3 + 2 files changed, 4 insertions(+), 123 deletions(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index cec9a953..1e2837bf 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,7 +41,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 9 +NESTED_LOGICAL_TABLES_VERSION = 8 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): @@ -1369,9 +1369,6 @@ def _getSetProperties(self, scipionSet: Any) -> Dict[str, Any]: if linkedTomograms: properties["linkedTomograms"] = linkedTomograms - linkedTiltSeries = self._getLinkedTiltSeriesSummary(scipionSet) - if linkedTiltSeries: - properties["linkedTiltSeries"] = linkedTiltSeries for attrName, attrValue in self._getAttributesToStore(scipionSet): if self._getAttributesToStore(attrValue): @@ -1390,125 +1387,6 @@ def _getLinkedTomogramsSummary(self, scipionSet: Any) -> List[Dict[str, Any]]: return tomograms - def _getLinkedTiltSeriesSummary(self, scipionSet: Any) -> Optional[Dict[str, Any]]: - tiltSeriesSet = self._callOptionalGetter(scipionSet, "getSetOfTiltSeries") - if tiltSeriesSet is None: - return None - - className = self._getClassName(tiltSeriesSet) - objectId = self._callOptionalGetter(tiltSeriesSet, "getObjId") - fileName = self._callOptionalGetter(tiltSeriesSet, "getFileName") - size = self._callOptionalGetter(tiltSeriesSet, "getSize") - - itemSummaries = [] - tsIds = [] - - for index, tiltSeries in enumerate(self._iterTiltSeriesItems(tiltSeriesSet)): - item = self._buildLinkedTiltSeriesItemSummary(tiltSeries, index) - if item is None: - continue - - itemSummaries.append(item) - - tiltSeriesId = item.get("tiltSeriesId") or item.get("tsId") or item.get("id") - if tiltSeriesId is not None: - tsIds.append(str(tiltSeriesId)) - - summary: Dict[str, Any] = { - "className": className, - "objectId": str(objectId) if objectId is not None else None, - "fileName": str(fileName) if fileName else None, - "size": self._toOptionalInt(size), - "tsIds": self._uniqueTextValues(tsIds), - "items": itemSummaries, - } - - return { - key: value - for key, value in summary.items() - if value not in (None, [], "") - } - - def _iterTiltSeriesItems(self, tiltSeriesSet: Any) -> Iterable[Any]: - iterItems = getattr(tiltSeriesSet, "iterItems", None) - if callable(iterItems): - try: - return iterItems(iterate=False) - except TypeError: - return iterItems() - except Exception: - pass - - try: - return iter(tiltSeriesSet) - except Exception: - return iter(()) - - def _buildLinkedTiltSeriesItemSummary(self, tiltSeries: Any, index: int) -> Optional[Dict[str, Any]]: - objectId = self._callOptionalGetter(tiltSeries, "getObjId") - tsId = ( - self._callOptionalGetter(tiltSeries, "getTsId") - or self._callOptionalGetter(tiltSeries, "getTSId") - or objectId - or index - ) - - if tsId is None: - return None - - label = ( - self._callOptionalGetter(tiltSeries, "getObjLabel") - or self._callOptionalGetter(tiltSeries, "getNameId") - or tsId - ) - - item: Dict[str, Any] = { - "id": str(tsId), - "tsId": str(tsId), - "tiltSeriesId": str(tsId), - "label": str(label), - } - - if objectId is not None: - item["objectId"] = str(objectId) - - dims = self._normalizeLinkedTomogramDims( - self._callOptionalGetter(tiltSeries, "getDim") - ) - if dims is not None: - item["dims"] = dims - - samplingRate = self._toOptionalFloat( - self._callOptionalGetter(tiltSeries, "getSamplingRate") - ) - if samplingRate is not None: - item["pixelSize"] = samplingRate - - tiltAxisAngle = self._toOptionalFloat( - self._callOptionalGetter( - self._callOptionalGetter(tiltSeries, "getAcquisition"), - "getTiltAxisAngle", - ) - ) - if tiltAxisAngle is not None: - item["tiltAxisAngle"] = tiltAxisAngle - - return item - - def _uniqueTextValues(self, values: Iterable[Any]) -> List[str]: - result = [] - seen = set() - - for value in values or []: - text = str(value).strip() if value is not None else "" - if not text or text in seen: - continue - - seen.add(text) - result.append(text) - - return result - def _iterLinkedTomograms(self, scipionSet: Any) -> Iterable[Any]: for methodName in ("iterTomograms", "iterVolumes"): iteratorGetter = getattr(scipionSet, methodName, None) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 64591a83..d9e98fff 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -562,6 +562,9 @@ def _mergeRelatedStoredSets( if rootKind == "coordinates3d" and candidateKind == "tomogram": continue + if rootKind in {"coordinates3d", "tomogram"} and candidateKind == "tiltSeries": + continue + if self._shouldReplaceLink(links.get(candidateKind)): links[candidateKind] = self._buildLink( protocolId=self._getCandidateProtocolId(candidate), From 09af1589f04bc4c78cd6ee0502c2660c2693b284 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 12:09:31 +0200 Subject: [PATCH 079/278] add protocol input refs table --- .../2fc2cd4da2e2_add_protocol_input_refs.py | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 alembic/versions/2fc2cd4da2e2_add_protocol_input_refs.py diff --git a/alembic/versions/2fc2cd4da2e2_add_protocol_input_refs.py b/alembic/versions/2fc2cd4da2e2_add_protocol_input_refs.py new file mode 100644 index 00000000..1801356e --- /dev/null +++ b/alembic/versions/2fc2cd4da2e2_add_protocol_input_refs.py @@ -0,0 +1,89 @@ +"""add protocol input refs + +Revision ID: 2fc2cd4da2e2 +Revises: 33ffae69565b +Create Date: 2026-06-22 12:06:28.813389 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '2fc2cd4da2e2' +down_revision: Union[str, None] = '33ffae69565b' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "protocol_input_refs", + sa.Column("projectId", sa.Integer(), nullable=False), + sa.Column("protocolDbId", sa.Integer(), nullable=False), + sa.Column("protocolId", sa.Text(), nullable=False), + sa.Column("inputName", sa.Text(), nullable=False), + sa.Column("itemIndex", sa.Integer(), nullable=False, server_default="0"), + sa.Column("parentProtocolDbId", sa.Integer(), nullable=True), + sa.Column("parentProtocolId", sa.Text(), nullable=True), + sa.Column("parentOutputName", sa.Text(), nullable=True), + sa.Column("objectClassName", sa.Text(), nullable=True), + sa.Column("objectId", sa.Text(), nullable=True), + sa.Column("createdAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updatedAt", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint( + "projectId", + "protocolDbId", + "inputName", + "itemIndex", + name="protocol_input_refs_pkey", + ), + sa.ForeignKeyConstraint( + ["projectId"], + ["projects.id"], + name="protocol_input_refs_projectId_fkey", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["projectId", "protocolDbId"], + ["protocols.projectId", "protocols.id"], + name="protocol_input_refs_protocolDbId_fkey", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["projectId", "parentProtocolDbId"], + ["protocols.projectId", "protocols.id"], + name="protocol_input_refs_parentProtocolDbId_fkey", + ondelete="CASCADE", + ), + ) + + op.create_index( + "idx_protocol_input_refs_protocol", + "protocol_input_refs", + ["projectId", "protocolDbId"], + unique=False, + ) + + op.create_index( + "idx_protocol_input_refs_parent", + "protocol_input_refs", + ["projectId", "parentProtocolDbId", "parentOutputName"], + unique=False, + ) + + op.create_index( + "idx_protocol_input_refs_parent_protocol_id", + "protocol_input_refs", + ["projectId", "parentProtocolId", "parentOutputName"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index("idx_protocol_input_refs_parent_protocol_id", table_name="protocol_input_refs") + op.drop_index("idx_protocol_input_refs_parent", table_name="protocol_input_refs") + op.drop_index("idx_protocol_input_refs_protocol", table_name="protocol_input_refs") + op.drop_table("protocol_input_refs") From 3dc4cb2556a92d44bf68b8f3d6a803fca13d327d Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 12:25:02 +0200 Subject: [PATCH 080/278] store protocol input refs during sync --- app/backend/api/services/project_service.py | 143 +++++++++++++++ app/backend/mapper/postgresql.py | 187 ++++++++++++++++++++ app/backend/mapper/scipion_set_mapper.py | 2 +- 3 files changed, 331 insertions(+), 1 deletion(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 03dce8dc..9431162d 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -599,6 +599,130 @@ def _shouldRegisterProtocolOutputs(self, protocol: Any) -> bool: return False + def _safeCall(self, obj: Any, methodName: str, default: Any = None) -> Any: + try: + method = getattr(obj, methodName, None) + if method is None: + return default + return method() + except Exception: + return default + + def _getScipionClassName(self, obj: Any) -> Optional[str]: + if obj is None: + return None + + className = self._safeCall(obj, "getClassName", None) + if className: + return str(className) + + return obj.__class__.__name__ + + def _getScipionObjectId(self, obj: Any) -> Optional[Any]: + return self._safeCall(obj, "getObjId", None) + + def _iterProtocolInputPointers(self, pointer: Any) -> List[Any]: + if pointer is None: + return [] + + if isinstance(pointer, (list, tuple, set)): + return list(pointer) + + try: + if not isinstance(pointer, (str, bytes, dict)): + items = list(pointer) + if items: + return items + except Exception: + pass + + return [pointer] + + def _getPointerTargetObject(self, pointer: Any) -> Any: + if pointer is None: + return None + + target = self._safeCall(pointer, "get", None) + if target is not None: + return target + + return pointer + + def _getPointerParentProtocolId(self, pointer: Any, targetObj: Any) -> Optional[Any]: + pointerObj = self._safeCall(pointer, "getObjValue", None) + parentProtocolId = self._safeCall(pointerObj, "getObjId", None) + if parentProtocolId is not None: + return parentProtocolId + + parentObj = self._safeCall(targetObj, "getObjParent", None) + parentProtocolId = self._safeCall(parentObj, "getObjId", None) + if parentProtocolId is not None: + return parentProtocolId + + return None + + def _getPointerOutputName(self, pointer: Any) -> Optional[str]: + outputName = self._safeCall(pointer, "getExtended", None) + if outputName is None: + return None + + outputNameText = str(outputName).strip() + return outputNameText or None + + def _buildProtocolInputRefsForPostgresql( + self, + projectId: int, + protocol: Any, + protocolDbIdByScipionId: Dict[str, int], + ) -> List[Dict[str, Any]]: + protocolId = self._getScipionObjectId(protocol) + if protocolId is None: + return [] + + protocolIdText = str(protocolId) + protocolDbId = protocolDbIdByScipionId.get(protocolIdText) + if protocolDbId is None: + return [] + + try: + inputAttributes = list(protocol.iterInputAttributes()) + except Exception: + return [] + + refs: List[Dict[str, Any]] = [] + + for inputName, pointer in inputAttributes: + pointerItems = self._iterProtocolInputPointers(pointer) + + for itemIndex, pointerItem in enumerate(pointerItems): + targetObj = self._getPointerTargetObject(pointerItem) + if targetObj is None: + continue + + parentProtocolId = self._getPointerParentProtocolId(pointerItem, targetObj) + parentProtocolIdText = str(parentProtocolId) if parentProtocolId is not None else None + parentProtocolDbId = ( + protocolDbIdByScipionId.get(parentProtocolIdText) + if parentProtocolIdText is not None + else None + ) + + refs.append({ + "projectId": int(projectId), + "protocolDbId": int(protocolDbId), + "protocolId": protocolIdText, + "inputName": str(inputName), + "itemIndex": int(itemIndex), + "parentProtocolDbId": parentProtocolDbId, + "parentProtocolId": parentProtocolIdText, + "parentOutputName": self._getPointerOutputName(pointerItem), + "objectClassName": self._getScipionClassName(targetObj), + "objectId": str(self._getScipionObjectId(targetObj)) + if self._getScipionObjectId(targetObj) is not None else None, + }) + + return refs + def syncProjectProtocolsAndDependencies( self, mapper: PostgresqlFlatMapper, @@ -617,6 +741,7 @@ def syncProjectProtocolsAndDependencies( protocolDbIdByScipionId: Dict[str, int] = {} currentProtocolIds: Set[str] = set() + protocolsByScipionId: Dict[str, Any] = {} # 1) Save all protocol nodes that are currently present in the real Scipion graph for nodeId, nodeObj in nodesDict.items(): @@ -642,6 +767,7 @@ def syncProjectProtocolsAndDependencies( currentProtocolIds.add(nodeIdText) protocolDbIdByScipionId[nodeIdText] = int(protocolDbId) + protocolsByScipionId[nodeIdText] = protocol # 2) Purge stale protocol rows that are no longer present in the real graph mapper.deleteProjectProtocolsNotInProtocolIds( @@ -670,9 +796,26 @@ def syncProjectProtocolsAndDependencies( savedEdges = mapper.replaceProjectProtocolDependencies(projectId, edges) + inputRefs: List[Dict[str, Any]] = [] + + for protocolIdText, protocol in protocolsByScipionId.items(): + inputRefs.extend( + self._buildProtocolInputRefsForPostgresql( + projectId=projectId, + protocol=protocol, + protocolDbIdByScipionId=protocolDbIdByScipionId, + ) + ) + + savedInputRefs = 0 + replaceInputRefs = getattr(mapper, "replaceProjectProtocolInputRefs", None) + if callable(replaceInputRefs): + savedInputRefs = replaceInputRefs(projectId, inputRefs) + return { "protocols": len(protocolDbIdByScipionId), "dependencies": int(savedEdges), + "inputRefs": int(savedInputRefs), } def syncProjectGraphAfterMutation( diff --git a/app/backend/mapper/postgresql.py b/app/backend/mapper/postgresql.py index f7236bf7..00a66d90 100644 --- a/app/backend/mapper/postgresql.py +++ b/app/backend/mapper/postgresql.py @@ -164,6 +164,44 @@ def initTables(self): """ ) + self.db.execute( + """ + CREATE TABLE IF NOT EXISTS protocol_input_refs ( + "projectId" INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + "protocolDbId" INTEGER NOT NULL, + "protocolId" TEXT NOT NULL, + "inputName" TEXT NOT NULL, + "itemIndex" INTEGER NOT NULL DEFAULT 0, + "parentProtocolDbId" INTEGER, + "parentProtocolId" TEXT, + "parentOutputName" TEXT, + "objectClassName" TEXT, + "objectId" TEXT, + "createdAt" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "updatedAt" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + PRIMARY KEY ("projectId", "protocolDbId", "inputName", "itemIndex"), + + FOREIGN KEY ("projectId", "protocolDbId") + REFERENCES protocols("projectId", id) + ON DELETE CASCADE, + + FOREIGN KEY ("projectId", "parentProtocolDbId") + REFERENCES protocols("projectId", id) + ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS idx_protocol_input_refs_protocol + ON protocol_input_refs("projectId", "protocolDbId"); + + CREATE INDEX IF NOT EXISTS idx_protocol_input_refs_parent + ON protocol_input_refs("projectId", "parentProtocolDbId", "parentOutputName"); + + CREATE INDEX IF NOT EXISTS idx_protocol_input_refs_parent_protocol_id + ON protocol_input_refs("projectId", "parentProtocolId", "parentOutputName"); + """ + ) + self.db.execute(""" CREATE TABLE IF NOT EXISTS protocol_steps ( id SERIAL PRIMARY KEY, @@ -965,6 +1003,155 @@ def getProjectProtocolDbIdMap(self, projectId: int) -> Dict[str, int]: if row.get("protocolId") is not None and row.get("id") is not None } + def replaceProjectProtocolInputRefs( + self, + projectId: int, + refs: List[Dict[str, Any]], + ) -> int: + def toOptionalInt(value: Any) -> Optional[int]: + if value is None or value == "": + return None + + try: + return int(value) + except Exception: + try: + return int(float(value)) + except Exception: + return None + + self.db.execute( + """ + DELETE FROM protocol_input_refs + WHERE "projectId" = %s + """, + (projectId,), + ) + + cleanRefs: List[Dict[str, Any]] = [] + seen = set() + + for ref in refs or []: + protocolDbId = toOptionalInt(ref.get("protocolDbId")) + if protocolDbId is None or protocolDbId <= 0: + continue + + inputName = str(ref.get("inputName") or "").strip() + if not inputName: + continue + + itemIndex = toOptionalInt(ref.get("itemIndex")) + if itemIndex is None or itemIndex < 0: + itemIndex = 0 + + protocolId = str(ref.get("protocolId") or "").strip() + if not protocolId: + continue + + key = (protocolDbId, inputName, itemIndex) + if key in seen: + continue + + seen.add(key) + + parentProtocolDbId = toOptionalInt(ref.get("parentProtocolDbId")) + if parentProtocolDbId is not None and parentProtocolDbId <= 0: + parentProtocolDbId = None + + cleanRefs.append({ + "projectId": int(projectId), + "protocolDbId": protocolDbId, + "protocolId": protocolId, + "inputName": inputName, + "itemIndex": itemIndex, + "parentProtocolDbId": parentProtocolDbId, + "parentProtocolId": str(ref.get("parentProtocolId")).strip() + if ref.get("parentProtocolId") not in (None, "") else None, + "parentOutputName": str(ref.get("parentOutputName")).strip() + if ref.get("parentOutputName") not in (None, "") else None, + "objectClassName": str(ref.get("objectClassName")).strip() + if ref.get("objectClassName") not in (None, "") else None, + "objectId": str(ref.get("objectId")).strip() + if ref.get("objectId") not in (None, "") else None, + }) + + if not cleanRefs: + return 0 + + valuesSql = ",".join(["(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"] * len(cleanRefs)) + params: List[Any] = [] + + for ref in cleanRefs: + params.extend([ + ref["projectId"], + ref["protocolDbId"], + ref["protocolId"], + ref["inputName"], + ref["itemIndex"], + ref["parentProtocolDbId"], + ref["parentProtocolId"], + ref["parentOutputName"], + ref["objectClassName"], + ref["objectId"], + ]) + + self.db.execute( + f""" + INSERT INTO protocol_input_refs ( + "projectId", + "protocolDbId", + "protocolId", + "inputName", + "itemIndex", + "parentProtocolDbId", + "parentProtocolId", + "parentOutputName", + "objectClassName", + "objectId" + ) + VALUES {valuesSql} + ON CONFLICT ("projectId", "protocolDbId", "inputName", "itemIndex") + DO UPDATE SET + "protocolId" = EXCLUDED."protocolId", + "parentProtocolDbId" = EXCLUDED."parentProtocolDbId", + "parentProtocolId" = EXCLUDED."parentProtocolId", + "parentOutputName" = EXCLUDED."parentOutputName", + "objectClassName" = EXCLUDED."objectClassName", + "objectId" = EXCLUDED."objectId", + "updatedAt" = NOW() + """, + tuple(params), + ) + + return len(cleanRefs) + + def listProtocolInputRefs( + self, + projectId: int, + protocolDbId: Optional[int] = None, + ) -> List[Dict[str, Any]]: + if protocolDbId is None: + return self.db.fetchAll( + """ + SELECT * + FROM protocol_input_refs + WHERE "projectId" = %s + ORDER BY "protocolDbId", "inputName", "itemIndex" + """, + (projectId,), + ) + + return self.db.fetchAll( + """ + SELECT * + FROM protocol_input_refs + WHERE "projectId" = %s + AND "protocolDbId" = %s + ORDER BY "inputName", "itemIndex" + """, + (projectId, protocolDbId), + ) + def replaceProjectProtocolDependencies( self, projectId: int, diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index 1e2837bf..51c24fc2 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,7 +41,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 8 +NESTED_LOGICAL_TABLES_VERSION = 9 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): From e3068d3d0dab99952dd3efb41d3e003bac8337df Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 12:35:35 +0200 Subject: [PATCH 081/278] Resolve integrated context from input refs --- .../postgresql_integrated_context_reader.py | 394 +++++++++++++++++- 1 file changed, 376 insertions(+), 18 deletions(-) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index d9e98fff..f7df3d20 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -93,6 +93,14 @@ def getContext(self) -> Optional[Dict[str, Any]]: summaries=summaries, ) + self._mergeExactInputRefs( + rootKind=rootKind, + rootStoredSet=storedSet, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + self._mergeRelatedStoredSets( rootStoredSet=storedSet, links=links, @@ -144,18 +152,22 @@ def _getIntegratedKind(self, storedSet: Dict[str, Any]) -> Optional[str]: storedSet.get("setClassName") or "", storedSet.get("itemClassName") or "", ) - classText = classText.replace(" ", "").lower() - if "setofcoordinates3d" in classText or "coordinate3d" in classText: + return self._getIntegratedKindFromText(classText) + + def _getIntegratedKindFromText(self, classText: Any) -> Optional[str]: + text = str(classText or "").replace(" ", "").lower() + + if "setofcoordinates3d" in text or "coordinate3d" in text: return "coordinates3d" - if "setofctftomoseries" in classText or "ctftomoseries" in classText: + if "setofctftomoseries" in text or "ctftomoseries" in text: return "ctf" - if "setoftiltseries" in classText or "tiltseries" in classText: + if "setoftiltseries" in text or "tiltseries" in text: return "tiltSeries" - if "setoftomograms" in classText or "tomogram" in classText: + if "setoftomograms" in text or "tomogram" in text: return "tomogram" return None @@ -223,6 +235,340 @@ def _buildSummary( return summary + def _listProtocolInputRefs(self, protocolDbId: Any) -> List[Dict[str, Any]]: + if protocolDbId is None: + return [] + + try: + rows = self.db.fetchAll( + """ + SELECT + "projectId", + "protocolDbId", + "protocolId", + "inputName", + "itemIndex", + "parentProtocolDbId", + "parentProtocolId", + "parentOutputName", + "objectClassName", + "objectId", + "createdAt", + "updatedAt" + FROM protocol_input_refs + WHERE "projectId" = %s + AND "protocolDbId" = %s + ORDER BY "inputName", "itemIndex" + """, + (self.projectId, int(protocolDbId)), + ) + except Exception: + logger.debug( + "Could not list protocol input refs. projectId=%s protocolDbId=%s", + self.projectId, + protocolDbId, + exc_info=True, + ) + return [] + + return [dict(row) for row in rows or []] + + def _getInputRefKind(self, inputRef: Dict[str, Any]) -> Optional[str]: + return self._getIntegratedKindFromText(inputRef.get("objectClassName")) + + def _findInputRefByKind( + self, + inputRefs: List[Dict[str, Any]], + kind: str, + ) -> Optional[Dict[str, Any]]: + for inputRef in inputRefs or []: + if self._getInputRefKind(inputRef) == kind: + return inputRef + + return None + + def _expandInputRefOutputNames(self, outputName: Any) -> List[str]: + outputNameText = str(outputName or "").strip() + if not outputNameText: + return [] + + outputNames = [outputNameText] + + if "." in outputNameText: + outputNames.append(outputNameText.split(".", 1)[0]) + + result = [] + seen = set() + + for item in outputNames: + if item in seen: + continue + + seen.add(item) + result.append(item) + + return result + + def _getStoredSetFromInputRef(self, inputRef: Dict[str, Any]) -> Optional[Dict[str, Any]]: + parentProtocolDbId = inputRef.get("parentProtocolDbId") + if parentProtocolDbId is None: + return None + + for outputName in self._expandInputRefOutputNames(inputRef.get("parentOutputName")): + storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=parentProtocolDbId, + outputName=outputName, + limit=None, + offset=0, + ) + + if storedSet is not None: + return storedSet + + return None + + def _buildInputRefLink( + self, + inputRef: Dict[str, Any], + storedSet: Dict[str, Any], + ) -> Dict[str, Any]: + return { + "protocolId": inputRef.get("parentProtocolId") or inputRef.get("parentProtocolDbId"), + "outputName": storedSet.get("outputName") or inputRef.get("parentOutputName"), + "itemId": storedSet.get("objectId") or storedSet.get("id") or inputRef.get("objectId"), + "label": inputRef.get("inputName") or inputRef.get("parentOutputName"), + "status": "inferred", + "source": "inputRef", + } + + def _getRelationKeySet(self, relationsByKey: Dict[str, Dict[str, Any]]) -> Optional[set]: + keys = set() + + for key, relation in (relationsByKey or {}).items(): + candidates = [ + key, + relation.get("key"), + relation.get("label"), + relation.get("tiltSeriesId"), + relation.get("tsId"), + relation.get("ctfSeriesId"), + relation.get("tomogramId"), + relation.get("coordinatesTomogramId"), + ] + + for value in candidates: + text = str(value).strip() if value is not None else "" + if text: + keys.add(text) + + return keys or None + + def _mergeInputRefRelations( + self, + kind: str, + inputRef: Dict[str, Any], + storedSet: Dict[str, Any], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> None: + protocolDbId = storedSet.get("protocolDbId") or inputRef.get("parentProtocolDbId") + outputName = storedSet.get("outputName") or inputRef.get("parentOutputName") + + if protocolDbId is None or not outputName: + return + + allowedRelationKeys = self._getRelationKeySet(relationsByKey) + + if kind == "tiltSeries": + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + items = self._filterIntegratedItemsByAllowedKeys( + reader.listTiltSeries(), + allowedRelationKeys, + ) + self._addTiltSeriesRelations(relationsByKey, items) + return + + if kind == "ctf": + reader = PostgresqlCtftomoReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + items = self._filterIntegratedItemsByAllowedKeys( + reader.listCtftomoSeries(), + allowedRelationKeys, + ) + self._addCtftomoRelations(relationsByKey, items) + return + + if kind == "tomogram": + items = self._filterIntegratedItemsByAllowedKeys( + self._buildTomogramItemsFromStoredSet(storedSet), + allowedRelationKeys, + ) + self._addTomogramRelations(relationsByKey, items) + return + + if kind == "coordinates3d": + reader = PostgresqlCoords3dReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + items = self._filterIntegratedItemsByAllowedKeys( + reader.listTomograms() or [], + allowedRelationKeys, + ) + self._addCoordinates3dRelations(relationsByKey, items) + + def _mergeKindFromInputRefs( + self, + kind: str, + inputRefs: List[Dict[str, Any]], + links: Dict[str, Optional[Dict[str, Any]]], + summaries: Dict[str, Optional[Dict[str, Any]]], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + if links.get(kind) is not None: + return None + + inputRef = self._findInputRefByKind(inputRefs, kind) + if inputRef is None: + return None + + storedSet = self._getStoredSetFromInputRef(inputRef) + if storedSet is None: + return None + + links[kind] = self._buildInputRefLink(inputRef, storedSet) + summaries[kind] = self._buildSummary( + storedSet, + extra={ + "source": "inputRef", + "inputName": inputRef.get("inputName"), + }, + ) + + self._mergeInputRefRelations( + kind=kind, + inputRef=inputRef, + storedSet=storedSet, + relationsByKey=relationsByKey, + ) + + return inputRef + + def _mergeExactInputRefs( + self, + rootKind: str, + rootStoredSet: Dict[str, Any], + links: Dict[str, Optional[Dict[str, Any]]], + summaries: Dict[str, Optional[Dict[str, Any]]], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> None: + rootProtocolDbId = rootStoredSet.get("protocolDbId") + inputRefs = self._listProtocolInputRefs(rootProtocolDbId) + + if not inputRefs: + return + + if rootKind == "ctf": + self._mergeKindFromInputRefs( + kind="tiltSeries", + inputRefs=inputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + return + + if rootKind == "tomogram": + ctfRef = self._mergeKindFromInputRefs( + kind="ctf", + inputRefs=inputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + tiltRef = self._mergeKindFromInputRefs( + kind="tiltSeries", + inputRefs=inputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + if tiltRef is None and ctfRef is not None: + ctfInputRefs = self._listProtocolInputRefs(ctfRef.get("parentProtocolDbId")) + self._mergeKindFromInputRefs( + kind="tiltSeries", + inputRefs=ctfInputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + return + + if rootKind == "coordinates3d": + ctfRef = self._mergeKindFromInputRefs( + kind="ctf", + inputRefs=inputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + tiltRef = self._mergeKindFromInputRefs( + kind="tiltSeries", + inputRefs=inputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + tomogramRef = self._findInputRefByKind(inputRefs, "tomogram") + if tomogramRef is not None: + tomogramInputRefs = self._listProtocolInputRefs( + tomogramRef.get("parentProtocolDbId") + ) + + if ctfRef is None: + ctfRef = self._mergeKindFromInputRefs( + kind="ctf", + inputRefs=tomogramInputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + if tiltRef is None: + tiltRef = self._mergeKindFromInputRefs( + kind="tiltSeries", + inputRefs=tomogramInputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + + if tiltRef is None and ctfRef is not None: + ctfInputRefs = self._listProtocolInputRefs(ctfRef.get("parentProtocolDbId")) + self._mergeKindFromInputRefs( + kind="tiltSeries", + inputRefs=ctfInputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + def _firstValueByName( self, values: Dict[str, Any], @@ -415,26 +761,38 @@ def _buildTomogramItemsFromStoredSet( for index, item in enumerate(storedSet.get("items") or []): values = item.get("values") or {} - tomogramId = self._firstValueByName( + tsId = self._firstValueByName( + values, + ["_tsId", "tsId", "tiltSeriesId", "tilt_series_id"], + ) + + tomoId = self._firstValueByName( values, - ["_tomoId", "tomoId", "tomogramId", "_tsId", "tsId"], + ["_tomoId", "tomoId", "tomogramId", "tomo_id", "tomogram_id"], ) label = self._firstValueByName( values, - ["_objLabel", "label", "_tsId", "tsId", "tomoId"], + ["_objLabel", "label", "name", "_tsId", "tsId", "tomoId"], ) - publicId = tomogramId or item.get("scipionItemId") or index + publicId = tsId or tomoId or item.get("scipionItemId") or index - items.append( - { - "id": publicId, - "tomoId": publicId, - "label": label or publicId, - "volumeId": index, - } - ) + row = { + "id": publicId, + "tomoId": publicId, + "label": label or publicId, + "volumeId": index, + } + + if tsId is not None: + row["tsId"] = tsId + row["tiltSeriesId"] = tsId + + if tomoId is not None: + row["sourceTomoId"] = tomoId + + items.append(row) return items @@ -562,7 +920,7 @@ def _mergeRelatedStoredSets( if rootKind == "coordinates3d" and candidateKind == "tomogram": continue - if rootKind in {"coordinates3d", "tomogram"} and candidateKind == "tiltSeries": + if rootKind in {"coordinates3d", "tomogram"} and candidateKind in {"tiltSeries", "ctf"}: continue if self._shouldReplaceLink(links.get(candidateKind)): From 11fce8339d4ca43b941010df80fef37562528858 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 13:22:58 +0200 Subject: [PATCH 082/278] clarify integrated dependency fallback policy --- .../postgresql_integrated_context_reader.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index f7df3d20..c7b4aad4 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -900,6 +900,19 @@ def _listRelatedStoredSets(self) -> List[Dict[str, Any]]: ) return result + def _shouldSkipDependencyCandidate( + self, + rootKind: Optional[str], + candidateKind: str, + ) -> bool: + if rootKind == "coordinates3d" and candidateKind == "tomogram": + return True + + if rootKind in {"coordinates3d", "tomogram"} and candidateKind in {"tiltSeries", "ctf"}: + return True + + return False + def _mergeRelatedStoredSets( self, rootStoredSet: Dict[str, Any], @@ -917,10 +930,7 @@ def _mergeRelatedStoredSets( if candidateKind is None or candidateKind not in links: continue - if rootKind == "coordinates3d" and candidateKind == "tomogram": - continue - - if rootKind in {"coordinates3d", "tomogram"} and candidateKind in {"tiltSeries", "ctf"}: + if self._shouldSkipDependencyCandidate(rootKind, candidateKind): continue if self._shouldReplaceLink(links.get(candidateKind)): From 01c734d8c74e816dc17bf2689e7aab62c242b0ec Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 13:41:34 +0200 Subject: [PATCH 083/278] preserve tomogram relation aliases --- .../postgresql_integrated_context_reader.py | 98 +++++++++++++++++-- 1 file changed, 91 insertions(+), 7 deletions(-) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index c7b4aad4..71cc20de 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -354,6 +354,8 @@ def _getRelationKeySet(self, relationsByKey: Dict[str, Dict[str, Any]]) -> Optio relation.get("tsId"), relation.get("ctfSeriesId"), relation.get("tomogramId"), + relation.get("sourceTomoId"), + relation.get("tomogramVolumeId"), relation.get("coordinatesTomogramId"), ] @@ -588,16 +590,72 @@ def _firstValueByName( def _normalizeName(self, value: Any) -> str: return str(value).replace("_", "").replace(".", "").replace("-", "").lower() + def _toRelationMatchText(self, value: Any) -> Optional[str]: + if value is None: + return None + + text = str(value).strip() + return text or None + + def _iterRelationMatchValues( + self, + keyValue: Any, + values: Dict[str, Any], + ) -> List[str]: + candidates = [ + keyValue, + values.get("key"), + values.get("tiltSeriesId"), + values.get("tsId"), + values.get("ctfSeriesId"), + values.get("tomogramId"), + values.get("sourceTomoId"), + values.get("coordinatesTomogramId"), + ] + + result = [] + seen = set() + + for candidate in candidates: + text = self._toRelationMatchText(candidate) + if not text or text in seen: + continue + + seen.add(text) + result.append(text) + + return result + + def _findExistingRelationKey( + self, + relationsByKey: Dict[str, Dict[str, Any]], + matchValues: List[str], + ) -> Optional[str]: + matchSet = set(matchValues or []) + if not matchSet: + return None + + for key, relation in (relationsByKey or {}).items(): + existingValues = self._iterRelationMatchValues(key, relation) + if any(value in matchSet for value in existingValues): + return key + + return None + def _addRelation( self, relationsByKey: Dict[str, Dict[str, Any]], keyValue: Any, **values: Any, ) -> None: - key = str(keyValue) if keyValue is not None else "" - if not key: + requestedKey = self._toRelationMatchText(keyValue) + if not requestedKey: return + matchValues = self._iterRelationMatchValues(requestedKey, values) + existingKey = self._findExistingRelationKey(relationsByKey, matchValues) + key = existingKey or requestedKey + relation = relationsByKey.setdefault( key, { @@ -649,22 +707,38 @@ def _addTomogramRelations( items: List[Dict[str, Any]], ) -> None: for index, item in enumerate(items or []): - tomogramId = ( - item.get("tomoId") + tiltSeriesId = ( + item.get("tiltSeriesId") + or item.get("tsId") + ) + + sourceTomoId = ( + item.get("sourceTomoId") or item.get("tomogramId") + or item.get("tomoId") + ) + + tomogramId = ( + sourceTomoId or item.get("id") or item.get("label") or index ) + relationKey = tiltSeriesId or tomogramId + ctfSeriesId = item.get("ctfSeriesId") or tiltSeriesId label = item.get("label") or item.get("name") or str(tomogramId) volumeId = item.get("tomogramVolumeId") or item.get("volumeId") or index self._addRelation( relationsByKey, - tomogramId, + relationKey, tomogramId=tomogramId, + sourceTomoId=sourceTomoId, tomogramVolumeId=volumeId, + tiltSeriesId=tiltSeriesId, + tsId=tiltSeriesId, + ctfSeriesId=ctfSeriesId, label=label, ) @@ -780,14 +854,17 @@ def _buildTomogramItemsFromStoredSet( row = { "id": publicId, - "tomoId": publicId, + "tomoId": tomoId or publicId, + "tomogramId": tomoId or publicId, "label": label or publicId, "volumeId": index, + "tomogramVolumeId": index, } if tsId is not None: row["tsId"] = tsId row["tiltSeriesId"] = tsId + row["ctfSeriesId"] = tsId if tomoId is not None: row["sourceTomoId"] = tomoId @@ -973,9 +1050,16 @@ def _filterIntegratedItemsByAllowedKeys( item.get("tiltSeriesId"), item.get("ctfSeriesId"), item.get("coordinatesTomogramId"), + item.get("tsId"), + item.get("sourceTomoId"), + item.get("tomogramVolumeId"), ] - if any(str(value) in allowedRelationKeys for value in candidates if value is not None): + if any( + str(value).strip() in allowedRelationKeys + for value in candidates + if value is not None and str(value).strip() + ): filteredItems.append(item) return filteredItems From f06d4d79bef3ff5fcb2f96e5e879bba7b87ffadc Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 14:04:45 +0200 Subject: [PATCH 084/278] avoid legacy ctftomo fallback for postgresql outputs --- app/backend/api/services/project_service.py | 5 +++ .../viewers/postgresql_volume_reader.py | 32 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 9431162d..3e0048ed 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6754,6 +6754,11 @@ def getCtftomoSeriesViewsService( if payload is not None: return payload + raise HTTPException( + status_code=404, + detail="CTF tomo series not found in PostgreSQL metadata", + ) + protocol, output = self._resolveOutputForCtftomoSeries(protocolId, outputName) targetKey = str(tiltSeriesId) diff --git a/app/backend/viewers/postgresql_volume_reader.py b/app/backend/viewers/postgresql_volume_reader.py index d479cb71..068e5ba1 100644 --- a/app/backend/viewers/postgresql_volume_reader.py +++ b/app/backend/viewers/postgresql_volume_reader.py @@ -197,6 +197,22 @@ def _buildVolumeFromSetItem( if locationIndex is not None: volume["locationIndex"] = locationIndex + tsId = self._firstValueBySuffix( + values, + ["tsId", "tiltSeriesId"], + ) + if tsId is not None: + volume["tsId"] = tsId + volume["tiltSeriesId"] = tsId + + tomoId = self._firstValueBySuffix( + values, + ["tomoId", "tomogramId"], + ) + if tomoId is not None: + volume["tomoId"] = tomoId + volume["tomogramId"] = tomoId + dims = self._extractDims(values) if dims is not None: volume["dims"] = dims @@ -259,6 +275,22 @@ def _buildSingleVolumeFromObjectTree( if locationIndex is not None: volume["locationIndex"] = locationIndex + tsId = self._firstValueBySuffix( + valuesByPath, + ["tsId", "tiltSeriesId"], + ) + if tsId is not None: + volume["tsId"] = tsId + volume["tiltSeriesId"] = tsId + + tomoId = self._firstValueBySuffix( + valuesByPath, + ["tomoId", "tomogramId"], + ) + if tomoId is not None: + volume["tomoId"] = tomoId + volume["tomogramId"] = tomoId + dims = self._extractDims(valuesByPath) if dims is not None: volume["dims"] = dims From 3029034cb0f1ec4ce879e0cfbf41048e9629795e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 14:14:42 +0200 Subject: [PATCH 085/278] avoid unsafe legacy ctftomo views fallback --- app/backend/api/services/project_service.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 3e0048ed..02aabf90 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6539,8 +6539,6 @@ def resolveAnalyzeViewerDecision(self, projectId: int, protocolId: int, ctx: Dic # ====================================================================== # Analyze Results: CTF Tomography (CTFTomoSeries) # ====================================================================== - - @lru_cache def _resolveOutputForCtftomoSeries(self, protocolId: int, outputName: str): """ Resolve protocol + CTFTomoSeries-like output for CTF tomography operations. @@ -6758,6 +6756,19 @@ def getCtftomoSeriesViewsService( status_code=404, detail="CTF tomo series not found in PostgreSQL metadata", ) + if mapper is not None: + logger.warning( + "PostgreSQL CTFTomo reader is not available. Skipping legacy fallback. " + "projectId=%s protocolId=%s outputName=%s tiltSeriesId=%s", + projectId, + protocolId, + outputName, + tiltSeriesId, + ) + raise HTTPException( + status_code=404, + detail="CTF tomo output is not available in PostgreSQL metadata", + ) protocol, output = self._resolveOutputForCtftomoSeries(protocolId, outputName) From 54d10d1b29fad12902b0c0c460f820b80d2e1805 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 14:19:22 +0200 Subject: [PATCH 086/278] match tomogram relations by item index --- .../postgresql_integrated_context_reader.py | 23 +++++++++++++++++++ .../viewers/postgresql_tiltseries_reader.py | 1 + 2 files changed, 24 insertions(+) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 71cc20de..b7500326 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -356,6 +356,9 @@ def _getRelationKeySet(self, relationsByKey: Dict[str, Dict[str, Any]]) -> Optio relation.get("tomogramId"), relation.get("sourceTomoId"), relation.get("tomogramVolumeId"), + relation.get("sourceTomoId"), + relation.get("tomogramVolumeId"), + relation.get("relationIndex"), relation.get("coordinatesTomogramId"), ] @@ -610,6 +613,8 @@ def _iterRelationMatchValues( values.get("ctfSeriesId"), values.get("tomogramId"), values.get("sourceTomoId"), + values.get("tomogramVolumeId"), + values.get("relationIndex"), values.get("coordinatesTomogramId"), ] @@ -676,11 +681,16 @@ def _addTiltSeriesRelations( for index, item in enumerate(items or []): tiltSeriesId = item.get("tiltSeriesId") or item.get("id") or index label = item.get("label") or str(tiltSeriesId) + relationIndex = item.get("relationIndex") + if relationIndex is None: + relationIndex = item.get("index", index) self._addRelation( relationsByKey, tiltSeriesId, tiltSeriesId=tiltSeriesId, + tsId=tiltSeriesId, + relationIndex=relationIndex, label=label, ) @@ -692,12 +702,17 @@ def _addCtftomoRelations( for index, item in enumerate(items or []): tiltSeriesId = item.get("tiltSeriesId") or item.get("id") or index label = item.get("label") or str(tiltSeriesId) + relationIndex = item.get("relationIndex") + if relationIndex is None: + relationIndex = item.get("index", index) self._addRelation( relationsByKey, tiltSeriesId, ctfSeriesId=tiltSeriesId, tiltSeriesId=tiltSeriesId, + tsId=tiltSeriesId, + relationIndex=relationIndex, label=label, ) @@ -729,6 +744,9 @@ def _addTomogramRelations( ctfSeriesId = item.get("ctfSeriesId") or tiltSeriesId label = item.get("label") or item.get("name") or str(tomogramId) volumeId = item.get("tomogramVolumeId") or item.get("volumeId") or index + relationIndex = item.get("relationIndex") + if relationIndex is None: + relationIndex = item.get("index", volumeId) self._addRelation( relationsByKey, @@ -740,6 +758,7 @@ def _addTomogramRelations( tsId=tiltSeriesId, ctfSeriesId=ctfSeriesId, label=label, + relationIndex=relationIndex, ) def _addCoordinates3dRelations( @@ -859,6 +878,8 @@ def _buildTomogramItemsFromStoredSet( "label": label or publicId, "volumeId": index, "tomogramVolumeId": index, + "index": index, + "relationIndex": index, } if tsId is not None: @@ -1052,6 +1073,8 @@ def _filterIntegratedItemsByAllowedKeys( item.get("coordinatesTomogramId"), item.get("tsId"), item.get("sourceTomoId"), + item.get("index"), + item.get("relationIndex"), item.get("tomogramVolumeId"), ] diff --git a/app/backend/viewers/postgresql_tiltseries_reader.py b/app/backend/viewers/postgresql_tiltseries_reader.py index af44bbf0..8cf5830d 100644 --- a/app/backend/viewers/postgresql_tiltseries_reader.py +++ b/app/backend/viewers/postgresql_tiltseries_reader.py @@ -134,6 +134,7 @@ def _buildTiltSeriesSummary(self, item: Dict[str, Any], index: int) -> Dict[str, summary: Dict[str, Any] = { "tiltSeriesId": str(tiltSeriesId), "label": "TiltSeries %s" % str(tiltSeriesId), + "index": index, } childTable = self._findChildTableForParentItem(itemId) From 65519f264a775993ff2475e87dcd655880bae6fc Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 14:38:34 +0200 Subject: [PATCH 087/278] match postgresql ctftomo series by aliases --- .../viewers/postgresql_ctftomo_reader.py | 33 +++++++++++++++++-- .../postgresql_integrated_context_reader.py | 23 ------------- .../viewers/postgresql_tiltseries_reader.py | 1 - 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/app/backend/viewers/postgresql_ctftomo_reader.py b/app/backend/viewers/postgresql_ctftomo_reader.py index b9ff73ba..7b2f3158 100644 --- a/app/backend/viewers/postgresql_ctftomo_reader.py +++ b/app/backend/viewers/postgresql_ctftomo_reader.py @@ -115,6 +115,7 @@ def _buildCtftomoSeriesSummary(self, item: Dict[str, Any], index: int) -> Dict[s label = str(item.get("label") or "CTFTomoSeries %s" % str(tiltSeriesId)) summary: Dict[str, Any] = { + "ctfSeriesId": str(tiltSeriesId), "tiltSeriesId": str(tiltSeriesId), "label": str(label), "index": index, @@ -147,15 +148,41 @@ def _findCtftomoSeriesItem(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: if storedSet is None: return None - targetKey = str(tiltSeriesId) + targetKey = str(tiltSeriesId).strip() + if not targetKey: + return None for index, item in enumerate(storedSet.get("items") or []): - itemTiltSeriesId = self._getTiltSeriesIdFromItem(item, index) - if str(itemTiltSeriesId) == targetKey: + if targetKey in self._getCtftomoSeriesItemMatchKeys(item, index): return item return None + def _getCtftomoSeriesItemMatchKeys(self, item: Dict[str, Any], index: int) -> set: + values = item.get("values") or {} + + candidates = [ + self._getTiltSeriesIdFromItem(item, index), + self._firstValue(values, ["_tsId", "tsId", "tiltSeriesId", "ctfSeriesId", "id"]), + self._firstValueBySuffix(values, ["tsid", "tiltseriesid", "ctfseriesid"]), + self._firstValueBySuffix(values, ["objlabel", "label", "name"]), + item.get("label"), + item.get("scipionItemId"), + item.get("id"), + index, + ] + + keys = set() + for candidate in candidates: + if candidate is None: + continue + + text = str(candidate).strip() + if text: + keys.add(text) + + return keys + def _getTiltSeriesIdFromItem(self, item: Dict[str, Any], index: int) -> Any: values = item.get("values") or {} tiltSeriesId = self._firstValue( diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index b7500326..71cc20de 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -356,9 +356,6 @@ def _getRelationKeySet(self, relationsByKey: Dict[str, Dict[str, Any]]) -> Optio relation.get("tomogramId"), relation.get("sourceTomoId"), relation.get("tomogramVolumeId"), - relation.get("sourceTomoId"), - relation.get("tomogramVolumeId"), - relation.get("relationIndex"), relation.get("coordinatesTomogramId"), ] @@ -613,8 +610,6 @@ def _iterRelationMatchValues( values.get("ctfSeriesId"), values.get("tomogramId"), values.get("sourceTomoId"), - values.get("tomogramVolumeId"), - values.get("relationIndex"), values.get("coordinatesTomogramId"), ] @@ -681,16 +676,11 @@ def _addTiltSeriesRelations( for index, item in enumerate(items or []): tiltSeriesId = item.get("tiltSeriesId") or item.get("id") or index label = item.get("label") or str(tiltSeriesId) - relationIndex = item.get("relationIndex") - if relationIndex is None: - relationIndex = item.get("index", index) self._addRelation( relationsByKey, tiltSeriesId, tiltSeriesId=tiltSeriesId, - tsId=tiltSeriesId, - relationIndex=relationIndex, label=label, ) @@ -702,17 +692,12 @@ def _addCtftomoRelations( for index, item in enumerate(items or []): tiltSeriesId = item.get("tiltSeriesId") or item.get("id") or index label = item.get("label") or str(tiltSeriesId) - relationIndex = item.get("relationIndex") - if relationIndex is None: - relationIndex = item.get("index", index) self._addRelation( relationsByKey, tiltSeriesId, ctfSeriesId=tiltSeriesId, tiltSeriesId=tiltSeriesId, - tsId=tiltSeriesId, - relationIndex=relationIndex, label=label, ) @@ -744,9 +729,6 @@ def _addTomogramRelations( ctfSeriesId = item.get("ctfSeriesId") or tiltSeriesId label = item.get("label") or item.get("name") or str(tomogramId) volumeId = item.get("tomogramVolumeId") or item.get("volumeId") or index - relationIndex = item.get("relationIndex") - if relationIndex is None: - relationIndex = item.get("index", volumeId) self._addRelation( relationsByKey, @@ -758,7 +740,6 @@ def _addTomogramRelations( tsId=tiltSeriesId, ctfSeriesId=ctfSeriesId, label=label, - relationIndex=relationIndex, ) def _addCoordinates3dRelations( @@ -878,8 +859,6 @@ def _buildTomogramItemsFromStoredSet( "label": label or publicId, "volumeId": index, "tomogramVolumeId": index, - "index": index, - "relationIndex": index, } if tsId is not None: @@ -1073,8 +1052,6 @@ def _filterIntegratedItemsByAllowedKeys( item.get("coordinatesTomogramId"), item.get("tsId"), item.get("sourceTomoId"), - item.get("index"), - item.get("relationIndex"), item.get("tomogramVolumeId"), ] diff --git a/app/backend/viewers/postgresql_tiltseries_reader.py b/app/backend/viewers/postgresql_tiltseries_reader.py index 8cf5830d..af44bbf0 100644 --- a/app/backend/viewers/postgresql_tiltseries_reader.py +++ b/app/backend/viewers/postgresql_tiltseries_reader.py @@ -134,7 +134,6 @@ def _buildTiltSeriesSummary(self, item: Dict[str, Any], index: int) -> Dict[str, summary: Dict[str, Any] = { "tiltSeriesId": str(tiltSeriesId), "label": "TiltSeries %s" % str(tiltSeriesId), - "index": index, } childTable = self._findChildTableForParentItem(itemId) From 12e5cf9b588f4e6671e251e012787e7dbc1d5f2b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 15:17:04 +0200 Subject: [PATCH 088/278] fix integrated tomogram and ctftomo relations --- .../viewers/postgresql_ctftomo_reader.py | 8 +- .../postgresql_integrated_context_reader.py | 132 ++++++++++++++++-- 2 files changed, 129 insertions(+), 11 deletions(-) diff --git a/app/backend/viewers/postgresql_ctftomo_reader.py b/app/backend/viewers/postgresql_ctftomo_reader.py index 7b2f3158..f3cf4106 100644 --- a/app/backend/viewers/postgresql_ctftomo_reader.py +++ b/app/backend/viewers/postgresql_ctftomo_reader.py @@ -69,11 +69,13 @@ def getCtftomoSeriesViews(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: summary["frames"] = frames summary["tiltSeriesId"] = summary.get("tiltSeriesId") or str(tiltSeriesId) + summary["ctfSeriesId"] = ( + summary.get("ctfSeriesId") + or summary.get("tiltSeriesId") + or str(tiltSeriesId) + ) summary["nViews"] = len(frames) - if not self._hasCtftomoViewerContract(summary): - return None - return summary def _getStoredSet(self) -> Optional[Dict[str, Any]]: diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 71cc20de..85bffeeb 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -93,6 +93,12 @@ def getContext(self) -> Optional[Dict[str, Any]]: summaries=summaries, ) + if rootKind == "tomogram": + self._mergeRootTomogramRelationsFromInputRefs( + rootStoredSet=storedSet, + relationsByKey=relationsByKey, + ) + self._mergeExactInputRefs( rootKind=rootKind, rootStoredSet=storedSet, @@ -334,7 +340,8 @@ def _buildInputRefLink( storedSet: Dict[str, Any], ) -> Dict[str, Any]: return { - "protocolId": inputRef.get("parentProtocolId") or inputRef.get("parentProtocolDbId"), + "protocolId": inputRef.get("parentProtocolDbId") or inputRef.get("parentProtocolId"), + "publicProtocolId": inputRef.get("parentProtocolId"), "outputName": storedSet.get("outputName") or inputRef.get("parentOutputName"), "itemId": storedSet.get("objectId") or storedSet.get("id") or inputRef.get("objectId"), "label": inputRef.get("inputName") or inputRef.get("parentOutputName"), @@ -430,6 +437,103 @@ def _mergeInputRefRelations( ) self._addCoordinates3dRelations(relationsByKey, items) + def _listTiltSeriesItemsFromInputRefs( + self, + inputRefs: List[Dict[str, Any]], + ) -> List[Dict[str, Any]]: + tiltRef = self._findInputRefByKind(inputRefs, "tiltSeries") + + if tiltRef is None: + ctfRef = self._findInputRefByKind(inputRefs, "ctf") + if ctfRef is not None: + ctfInputRefs = self._listProtocolInputRefs( + ctfRef.get("parentProtocolDbId") + ) + tiltRef = self._findInputRefByKind(ctfInputRefs, "tiltSeries") + + if tiltRef is None: + return [] + + storedSet = self._getStoredSetFromInputRef(tiltRef) + if storedSet is None: + return [] + + protocolDbId = storedSet.get("protocolDbId") or tiltRef.get("parentProtocolDbId") + outputName = storedSet.get("outputName") or tiltRef.get("parentOutputName") + + if protocolDbId is None or not outputName: + return [] + + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + + return reader.listTiltSeries() or [] + + def _mergeRootTomogramRelationsFromInputRefs( + self, + rootStoredSet: Dict[str, Any], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> None: + inputRefs = self._listProtocolInputRefs(rootStoredSet.get("protocolDbId")) + if not inputRefs: + return + + tiltSeriesItems = self._listTiltSeriesItemsFromInputRefs(inputRefs) + if not tiltSeriesItems: + return + + tomogramItems = self._buildTomogramItemsFromStoredSet(rootStoredSet) + + for index, tomogramItem in enumerate(tomogramItems): + if index >= len(tiltSeriesItems): + break + + tiltSeriesItem = tiltSeriesItems[index] + tiltSeriesId = ( + tiltSeriesItem.get("tiltSeriesId") + or tiltSeriesItem.get("tsId") + or tiltSeriesItem.get("id") + ) + + if tiltSeriesId is None: + continue + + tomogramId = ( + tomogramItem.get("tomoId") + or tomogramItem.get("tomogramId") + or tomogramItem.get("id") + or tomogramItem.get("label") + or index + ) + + volumeId = ( + tomogramItem.get("tomogramVolumeId") + or tomogramItem.get("volumeId") + or index + ) + + label = ( + tomogramItem.get("label") + or tomogramItem.get("name") + or str(tomogramId) + ) + + self._addRelation( + relationsByKey, + tomogramId, + tomogramId=tomogramId, + sourceTomoId=tomogramItem.get("sourceTomoId"), + tomogramVolumeId=volumeId, + tiltSeriesId=tiltSeriesId, + tsId=tiltSeriesId, + ctfSeriesId=tiltSeriesId, + label=label, + ) + def _mergeKindFromInputRefs( self, kind: str, @@ -610,6 +714,7 @@ def _iterRelationMatchValues( values.get("ctfSeriesId"), values.get("tomogramId"), values.get("sourceTomoId"), + values.get("tomogramVolumeId"), values.get("coordinatesTomogramId"), ] @@ -626,21 +731,23 @@ def _iterRelationMatchValues( return result - def _findExistingRelationKey( + def _findExistingRelationKeys( self, relationsByKey: Dict[str, Dict[str, Any]], matchValues: List[str], - ) -> Optional[str]: + ) -> List[str]: matchSet = set(matchValues or []) if not matchSet: - return None + return [] + + keys = [] for key, relation in (relationsByKey or {}).items(): existingValues = self._iterRelationMatchValues(key, relation) if any(value in matchSet for value in existingValues): - return key + keys.append(key) - return None + return keys def _addRelation( self, @@ -653,8 +760,8 @@ def _addRelation( return matchValues = self._iterRelationMatchValues(requestedKey, values) - existingKey = self._findExistingRelationKey(relationsByKey, matchValues) - key = existingKey or requestedKey + existingKeys = self._findExistingRelationKeys(relationsByKey, matchValues) + key = existingKeys[0] if existingKeys else requestedKey relation = relationsByKey.setdefault( key, @@ -664,6 +771,15 @@ def _addRelation( }, ) + for otherKey in existingKeys[1:]: + otherRelation = relationsByKey.pop(otherKey, None) + if not otherRelation: + continue + + for name, value in otherRelation.items(): + if value is not None and name not in relation: + relation[name] = value + for name, value in values.items(): if value is not None: relation[name] = value From 619f8618bb61e07442e6450925c40f5081b3c45d Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 15:20:40 +0200 Subject: [PATCH 089/278] fix integrated tomogram and ctftomo relations --- app/backend/mapper/scipion_set_mapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index 51c24fc2..50e48dcc 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,7 +41,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 9 +NESTED_LOGICAL_TABLES_VERSION = 10 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): From ff0827b6dd2f2f6f78e370452e244a9eeffb9395 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 16:01:58 +0200 Subject: [PATCH 090/278] Resolve ctftomo tilt angles from associated tilt series --- app/backend/api/services/project_service.py | 26 ++- app/backend/mapper/scipion_set_mapper.py | 2 +- .../viewers/postgresql_ctftomo_reader.py | 213 +++++++++++++++++- .../postgresql_integrated_context_reader.py | 19 +- .../viewers/postgresql_tiltseries_reader.py | 21 +- 5 files changed, 273 insertions(+), 8 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 02aabf90..aed7a1f9 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -8109,7 +8109,19 @@ def renderTiltSeriesImageService( return self._storeTiltSeriesPreviewInCache(cacheKey, response) - except Exception: + + except HTTPException: + logger.exception( + "Failed to render TiltSeries image from PostgreSQL. projectId=%s protocolId=%s outputName=%s tiltSeriesId=%s index=%s", + projectId, + protocolId, + outputName, + tiltSeriesId, + index, + ) + raise + + except Exception as e: logger.exception( "Failed to render TiltSeries image from PostgreSQL. projectId=%s protocolId=%s outputName=%s tiltSeriesId=%s index=%s", projectId, @@ -8119,6 +8131,12 @@ def renderTiltSeriesImageService( index, ) + raise HTTPException( + status_code=500, + detail=f"Failed to render TiltSeries image from PostgreSQL metadata: {e}", + + ) + protocol, setOfTiltSeries = self._resolveOutputForTiltSeries(protocolId, outputName) ts = setOfTiltSeries.getItem('_tsId', tiltSeriesId) @@ -8568,6 +8586,12 @@ def listCoordinates3dTomogramsService( getattr(pgReader, "lastSkipReason", None), ) + if mapper is not None: + raise HTTPException( + status_code=404, + detail="TiltSeries frame not found in PostgreSQL metadata", + ) + try: protocol = self.currentProject.getProtocol(int(protocolId)) except Exception: diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index 50e48dcc..1900be8c 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,7 +41,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 10 +NESTED_LOGICAL_TABLES_VERSION = 11 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): diff --git a/app/backend/viewers/postgresql_ctftomo_reader.py b/app/backend/viewers/postgresql_ctftomo_reader.py index f3cf4106..d50ffd15 100644 --- a/app/backend/viewers/postgresql_ctftomo_reader.py +++ b/app/backend/viewers/postgresql_ctftomo_reader.py @@ -26,6 +26,7 @@ from typing import Any, Dict, List, Optional from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper +from app.backend.viewers.postgresql_tiltseries_reader import PostgresqlTiltSeriesReader class PostgresqlCtftomoReader: @@ -37,6 +38,7 @@ def __init__(self, db, projectId: int, protocolId: int, outputName: str): self.setMapper = ScipionSetPostgresqlMapper(db) self._storedSet = None self._logicalTables = None + self._associatedTiltSeriesFramesBySeriesId = {} def hasOutput(self) -> bool: return self._getStoredSet() is not None @@ -61,11 +63,22 @@ def getCtftomoSeriesViews(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: summary = self._buildCtftomoSeriesSummary(seriesItem, 0) childTable = self._findChildTableForParentItem(seriesItem.get("scipionItemId")) + associatedTiltFrames = self._getAssociatedTiltSeriesFrames( + summary.get("tiltSeriesId") or tiltSeriesId + ) + frames: List[Dict[str, Any]] = [] if childTable is not None: childItems = self.setMapper.getStoredSetTableItems(int(childTable["id"])) for index, item in enumerate(childItems): - frames.append(self._buildCtftomoMeasurementFrame(item, index)) + frame = self._buildCtftomoMeasurementFrame(item, index) + tiltFrame = self._findAssociatedTiltSeriesFrame( + ctfFrame=frame, + position=index, + tiltFrames=associatedTiltFrames, + ) + self._mergeTiltSeriesFrameIntoCtftomoFrame(frame, tiltFrame) + frames.append(frame) summary["frames"] = frames summary["tiltSeriesId"] = summary.get("tiltSeriesId") or str(tiltSeriesId) @@ -98,6 +111,204 @@ def _getLogicalTables(self) -> List[Dict[str, Any]]: ) return self._logicalTables + def _getAssociatedTiltSeriesFrames(self, tiltSeriesId: Any) -> List[Dict[str, Any]]: + seriesKey = str(tiltSeriesId) + if seriesKey in self._associatedTiltSeriesFramesBySeriesId: + return self._associatedTiltSeriesFramesBySeriesId[seriesKey] + + frames = self._loadAssociatedTiltSeriesFrames(tiltSeriesId) + self._associatedTiltSeriesFramesBySeriesId[seriesKey] = frames + return frames + + def _loadAssociatedTiltSeriesFrames(self, tiltSeriesId: Any) -> List[Dict[str, Any]]: + storedSet = self._getStoredSet() + rootProtocolDbId = storedSet.get("protocolDbId") if storedSet else self.protocolId + + tiltRef = self._findTiltSeriesInputRefForProtocol(rootProtocolDbId) + if tiltRef is None: + return [] + + tiltStoredSet = self._getStoredSetFromInputRef(tiltRef) + if tiltStoredSet is None: + return [] + + protocolDbId = tiltStoredSet.get("protocolDbId") or tiltRef.get("parentProtocolDbId") + outputName = tiltStoredSet.get("outputName") or tiltRef.get("parentOutputName") + + if protocolDbId is None or not outputName: + return [] + + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + + payload = reader.getTiltSeriesFrames(tiltSeriesId) + if not payload: + return [] + + return payload.get("frames") or [] + + def _findTiltSeriesInputRefForProtocol( + self, + protocolDbId: Any, + visited: Optional[set] = None, + ) -> Optional[Dict[str, Any]]: + if protocolDbId is None: + return None + + if visited is None: + visited = set() + + protocolKey = str(protocolDbId) + if protocolKey in visited: + return None + + visited.add(protocolKey) + + inputRefs = self._listProtocolInputRefs(protocolDbId) + for inputRef in inputRefs: + if self._getInputRefKind(inputRef) == "tiltSeries": + return inputRef + + for inputRef in inputRefs: + if self._getInputRefKind(inputRef) != "ctf": + continue + + parentProtocolDbId = inputRef.get("parentProtocolDbId") + tiltRef = self._findTiltSeriesInputRefForProtocol( + parentProtocolDbId, + visited=visited, + ) + + if tiltRef is not None: + return tiltRef + + return None + + def _listProtocolInputRefs(self, protocolDbId: Any) -> List[Dict[str, Any]]: + if protocolDbId is None: + return [] + + try: + rows = self.db.fetchAll( + """ + SELECT + "projectId", + "protocolDbId", + "protocolId", + "inputName", + "itemIndex", + "parentProtocolDbId", + "parentProtocolId", + "parentOutputName", + "objectClassName", + "objectId" + FROM protocol_input_refs + WHERE "projectId" = %s + AND "protocolDbId" = %s + ORDER BY "inputName", "itemIndex" + """, + (self.projectId, int(protocolDbId)), + ) + except Exception: + return [] + + return [dict(row) for row in rows or []] + + def _getInputRefKind(self, inputRef: Dict[str, Any]) -> Optional[str]: + text = str(inputRef.get("objectClassName") or "").replace(" ", "").lower() + + if "ctftomo" in text: + return "ctf" + + if "tiltseries" in text: + return "tiltSeries" + + return None + + def _getStoredSetFromInputRef(self, inputRef: Dict[str, Any]) -> Optional[Dict[str, Any]]: + parentProtocolDbId = inputRef.get("parentProtocolDbId") + if parentProtocolDbId is None: + return None + + for outputName in self._expandInputRefOutputNames(inputRef.get("parentOutputName")): + storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=parentProtocolDbId, + outputName=outputName, + limit=None, + offset=0, + ) + + if storedSet is not None: + return storedSet + + return None + + def _expandInputRefOutputNames(self, outputName: Any) -> List[str]: + outputNameText = str(outputName or "").strip() + if not outputNameText: + return [] + + outputNames = [outputNameText] + + if "." in outputNameText: + outputNames.append(outputNameText.split(".", 1)[0]) + + result = [] + seen = set() + + for item in outputNames: + if item in seen: + continue + + seen.add(item) + result.append(item) + + return result + + def _findAssociatedTiltSeriesFrame( + self, + ctfFrame: Dict[str, Any], + position: int, + tiltFrames: List[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + if not tiltFrames: + return None + + orderKey = self._toTextKey(ctfFrame.get("order")) + if orderKey: + for tiltFrame in tiltFrames: + if self._toTextKey(tiltFrame.get("order")) == orderKey: + return tiltFrame + + if 0 <= position < len(tiltFrames): + return tiltFrames[position] + + return None + + def _mergeTiltSeriesFrameIntoCtftomoFrame( + self, + ctfFrame: Dict[str, Any], + tiltFrame: Optional[Dict[str, Any]], + ) -> None: + if tiltFrame is None: + return + + for key in ("tiltAngle", "dose", "order"): + if ctfFrame.get(key) is None and tiltFrame.get(key) is not None: + ctfFrame[key] = tiltFrame.get(key) + + def _toTextKey(self, value: Any) -> Optional[str]: + if value is None: + return None + + text = str(value).strip() + return text or None + def _buildCtftomoSeriesSummary(self, item: Dict[str, Any], index: int) -> Dict[str, Any]: values = item.get("values") or {} itemId = item.get("scipionItemId") diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 85bffeeb..4d70081e 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -586,13 +586,28 @@ def _mergeExactInputRefs( return if rootKind == "ctf": - self._mergeKindFromInputRefs( + tiltRef = self._mergeKindFromInputRefs( kind="tiltSeries", inputRefs=inputRefs, links=links, summaries=summaries, relationsByKey=relationsByKey, ) + + if tiltRef is None: + ctfRef = self._findInputRefByKind(inputRefs, "ctf") + if ctfRef is not None: + ctfInputRefs = self._listProtocolInputRefs( + ctfRef.get("parentProtocolDbId") + ) + self._mergeKindFromInputRefs( + kind="tiltSeries", + inputRefs=ctfInputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + return if rootKind == "tomogram": @@ -797,6 +812,7 @@ def _addTiltSeriesRelations( relationsByKey, tiltSeriesId, tiltSeriesId=tiltSeriesId, + tsId=tiltSeriesId, label=label, ) @@ -814,6 +830,7 @@ def _addCtftomoRelations( tiltSeriesId, ctfSeriesId=tiltSeriesId, tiltSeriesId=tiltSeriesId, + tsId=tiltSeriesId, label=label, ) diff --git a/app/backend/viewers/postgresql_tiltseries_reader.py b/app/backend/viewers/postgresql_tiltseries_reader.py index af44bbf0..5d121b08 100644 --- a/app/backend/viewers/postgresql_tiltseries_reader.py +++ b/app/backend/viewers/postgresql_tiltseries_reader.py @@ -43,11 +43,12 @@ def __init__(self, db, projectId: int, protocolId: int, outputName: str): self._logicalTables = None def hasOutput(self) -> bool: - return self._getStoredSet() is not None + storedSet = self._getStoredSet() + return storedSet is not None and self._isTiltSeriesStoredSet(storedSet) def listTiltSeries(self) -> List[Dict[str, Any]]: storedSet = self._getStoredSet() - if storedSet is None: + if storedSet is None or not self._isTiltSeriesStoredSet(storedSet): return [] result = [] @@ -57,10 +58,12 @@ def listTiltSeries(self) -> List[Dict[str, Any]]: return result def getTiltSeriesFrames(self, tiltSeriesId: Any) -> Optional[Dict[str, Any]]: - seriesItem = self._findTiltSeriesItem(tiltSeriesId) - if seriesItem is None: + storedSet = self._getStoredSet() + if storedSet is None or not self._isTiltSeriesStoredSet(storedSet): return None + seriesItem = self._findTiltSeriesItem(tiltSeriesId) + seriesSummary = self._buildTiltSeriesSummary(seriesItem, 0) childTable = self._findChildTableForParentItem(seriesItem.get("scipionItemId")) @@ -109,6 +112,16 @@ def _getStoredSet(self) -> Optional[Dict[str, Any]]: ) return self._storedSet + def _isTiltSeriesStoredSet(self, storedSet: Dict[str, Any]) -> bool: + classText = self._getStoredSetClassText(storedSet) + return "tiltseries" in classText and "ctftomo" not in classText + + def _getStoredSetClassText(self, storedSet: Dict[str, Any]) -> str: + return ("%s %s" % ( + storedSet.get("setClassName") or "", + storedSet.get("itemClassName") or "", + )).replace(" ", "").lower() + def _getLogicalTables(self) -> List[Dict[str, Any]]: if self._logicalTables is None: storedSet = self._getStoredSet() From 27077d256a7c3d68d2c179535523c64657233aa8 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 16:48:53 +0200 Subject: [PATCH 091/278] resolve ctftomo relations from regular tilt series --- .../viewers/postgresql_ctftomo_reader.py | 95 ++++++++-- .../postgresql_integrated_context_reader.py | 167 ++++++++++++++++-- 2 files changed, 229 insertions(+), 33 deletions(-) diff --git a/app/backend/viewers/postgresql_ctftomo_reader.py b/app/backend/viewers/postgresql_ctftomo_reader.py index d50ffd15..2250f2fb 100644 --- a/app/backend/viewers/postgresql_ctftomo_reader.py +++ b/app/backend/viewers/postgresql_ctftomo_reader.py @@ -124,16 +124,12 @@ def _loadAssociatedTiltSeriesFrames(self, tiltSeriesId: Any) -> List[Dict[str, A storedSet = self._getStoredSet() rootProtocolDbId = storedSet.get("protocolDbId") if storedSet else self.protocolId - tiltRef = self._findTiltSeriesInputRefForProtocol(rootProtocolDbId) - if tiltRef is None: - return [] - - tiltStoredSet = self._getStoredSetFromInputRef(tiltRef) + tiltStoredSet = self._findRegularTiltSeriesStoredSetForProtocol(rootProtocolDbId) if tiltStoredSet is None: return [] - protocolDbId = tiltStoredSet.get("protocolDbId") or tiltRef.get("parentProtocolDbId") - outputName = tiltStoredSet.get("outputName") or tiltRef.get("parentOutputName") + protocolDbId = tiltStoredSet.get("protocolDbId") + outputName = tiltStoredSet.get("outputName") if protocolDbId is None or not outputName: return [] @@ -151,7 +147,7 @@ def _loadAssociatedTiltSeriesFrames(self, tiltSeriesId: Any) -> List[Dict[str, A return payload.get("frames") or [] - def _findTiltSeriesInputRefForProtocol( + def _findRegularTiltSeriesStoredSetForProtocol( self, protocolDbId: Any, visited: Optional[set] = None, @@ -168,23 +164,51 @@ def _findTiltSeriesInputRefForProtocol( visited.add(protocolKey) + sameProtocolStoredSet = self._findRegularTiltSeriesStoredSetInProtocol(protocolDbId) + if sameProtocolStoredSet is not None: + return sameProtocolStoredSet + inputRefs = self._listProtocolInputRefs(protocolDbId) + for inputRef in inputRefs: - if self._getInputRefKind(inputRef) == "tiltSeries": - return inputRef + if self._getInputRefKind(inputRef) != "tiltSeries": + continue + + storedSet = self._getStoredSetFromInputRef(inputRef) + if self._isRegularTiltSeriesStoredSet(storedSet): + return storedSet for inputRef in inputRefs: if self._getInputRefKind(inputRef) != "ctf": continue parentProtocolDbId = inputRef.get("parentProtocolDbId") - tiltRef = self._findTiltSeriesInputRefForProtocol( + storedSet = self._findRegularTiltSeriesStoredSetForProtocol( parentProtocolDbId, visited=visited, ) - if tiltRef is not None: - return tiltRef + if storedSet is not None: + return storedSet + + return None + + def _findRegularTiltSeriesStoredSetInProtocol( + self, + protocolDbId: Any, + ) -> Optional[Dict[str, Any]]: + try: + storedSets = self.setMapper.listProtocolStoredSets( + projectId=self.projectId, + protocolDbId=int(protocolDbId), + ) + except Exception: + return None + + for storedSet in storedSets or []: + storedSetDict = dict(storedSet) + if self._isRegularTiltSeriesStoredSet(storedSetDict): + return storedSetDict return None @@ -219,16 +243,57 @@ def _listProtocolInputRefs(self, protocolDbId: Any) -> List[Dict[str, Any]]: return [dict(row) for row in rows or []] def _getInputRefKind(self, inputRef: Dict[str, Any]) -> Optional[str]: - text = str(inputRef.get("objectClassName") or "").replace(" ", "").lower() + text = self._normalizeClassText(inputRef.get("objectClassName")) if "ctftomo" in text: return "ctf" - if "tiltseries" in text: + if self._isTiltSeriesMClassText(text): + return "tiltSeriesM" + + if self._isRegularTiltSeriesClassText(text): return "tiltSeries" return None + def _isRegularTiltSeriesStoredSet(self, storedSet: Optional[Dict[str, Any]]) -> bool: + if storedSet is None: + return False + + classText = self._normalizeClassText( + "%s %s" % ( + storedSet.get("setClassName") or "", + storedSet.get("itemClassName") or "", + ) + ) + + return self._isRegularTiltSeriesClassText(classText) + + def _isRegularTiltSeriesClassText(self, text: Any) -> bool: + classText = self._normalizeClassText(text) + + if self._isTiltSeriesMClassText(classText): + return False + + if "ctftomo" in classText: + return False + + return "tiltseries" in classText + + def _isTiltSeriesMClassText(self, text: Any) -> bool: + classText = self._normalizeClassText(text) + + return ( + "setoftiltseriesm" in classText + or "tiltseriesm" in classText + or "tiltimagem" in classText + or "movie" in classText + or "movies" in classText + ) + + def _normalizeClassText(self, value: Any) -> str: + return str(value or "").replace(" ", "").replace("_", "").replace(".", "").lower() + def _getStoredSetFromInputRef(self, inputRef: Dict[str, Any]) -> Optional[Dict[str, Any]]: parentProtocolDbId = inputRef.get("parentProtocolDbId") if parentProtocolDbId is None: diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 4d70081e..5591655b 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -178,6 +178,39 @@ def _getIntegratedKindFromText(self, classText: Any) -> Optional[str]: return None + def _normalizeClassText(self, value: Any) -> str: + return str(value or "").replace(" ", "").replace("_", "").replace(".", "").lower() + + def _isTiltSeriesMClassText(self, value: Any) -> bool: + text = self._normalizeClassText(value) + + return ( + "setoftiltseriesm" in text + or "tiltseriesm" in text + or "tiltimagem" in text + or "movie" in text + or "movies" in text + ) + + def _isRegularTiltSeriesStoredSet(self, storedSet: Optional[Dict[str, Any]]) -> bool: + if storedSet is None: + return False + + text = self._normalizeClassText( + "%s %s" % ( + storedSet.get("setClassName") or "", + storedSet.get("itemClassName") or "", + ) + ) + + if self._isTiltSeriesMClassText(text): + return False + + if "ctftomo" in text: + return False + + return "tiltseries" in text + def _getStoredSetProperty( self, storedSet: Dict[str, Any], @@ -534,6 +567,120 @@ def _mergeRootTomogramRelationsFromInputRefs( label=label, ) + def _findRegularTiltSeriesStoredSetForProtocol( + self, + protocolDbId: Any, + visited: Optional[set] = None, + ) -> Optional[Dict[str, Any]]: + if protocolDbId is None: + return None + + if visited is None: + visited = set() + + protocolKey = str(protocolDbId) + if protocolKey in visited: + return None + + visited.add(protocolKey) + + sameProtocolStoredSet = self._findRegularTiltSeriesStoredSetInProtocol(protocolDbId) + if sameProtocolStoredSet is not None: + return sameProtocolStoredSet + + inputRefs = self._listProtocolInputRefs(protocolDbId) + + for inputRef in inputRefs: + if self._getInputRefKind(inputRef) != "tiltSeries": + continue + + storedSet = self._getStoredSetFromInputRef(inputRef) + if self._isRegularTiltSeriesStoredSet(storedSet): + return storedSet + + for inputRef in inputRefs: + if self._getInputRefKind(inputRef) != "ctf": + continue + + storedSet = self._findRegularTiltSeriesStoredSetForProtocol( + inputRef.get("parentProtocolDbId"), + visited=visited, + ) + + if storedSet is not None: + return storedSet + + return None + + def _findRegularTiltSeriesStoredSetInProtocol( + self, + protocolDbId: Any, + ) -> Optional[Dict[str, Any]]: + try: + storedSets = self.setMapper.listProtocolStoredSets( + projectId=self.projectId, + protocolDbId=int(protocolDbId), + ) + except Exception: + return None + + for storedSet in storedSets or []: + storedSetDict = dict(storedSet) + if self._isRegularTiltSeriesStoredSet(storedSetDict): + return storedSetDict + + return None + + def _mergeRegularTiltSeriesForCtftomo( + self, + rootProtocolDbId: Any, + links: Dict[str, Optional[Dict[str, Any]]], + summaries: Dict[str, Optional[Dict[str, Any]]], + relationsByKey: Dict[str, Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + if links.get("tiltSeries") is not None: + return None + + storedSet = self._findRegularTiltSeriesStoredSetForProtocol(rootProtocolDbId) + if storedSet is None: + return None + + links["tiltSeries"] = self._buildLink( + protocolId=storedSet.get("protocolDbId"), + outputName=storedSet.get("outputName"), + storedSet=storedSet, + label=storedSet.get("outputName"), + statusValue="inferred", + ) + + summaries["tiltSeries"] = self._buildSummary( + storedSet, + extra={ + "source": "ctfAssociatedTiltSeries", + }, + ) + + protocolDbId = storedSet.get("protocolDbId") + outputName = storedSet.get("outputName") + + if protocolDbId is None or not outputName: + return storedSet + + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + + items = self._filterIntegratedItemsByAllowedKeys( + reader.listTiltSeries(), + self._getRelationKeySet(relationsByKey), + ) + self._addTiltSeriesRelations(relationsByKey, items) + + return storedSet + def _mergeKindFromInputRefs( self, kind: str, @@ -586,28 +733,12 @@ def _mergeExactInputRefs( return if rootKind == "ctf": - tiltRef = self._mergeKindFromInputRefs( - kind="tiltSeries", - inputRefs=inputRefs, + self._mergeRegularTiltSeriesForCtftomo( + rootProtocolDbId=rootProtocolDbId, links=links, summaries=summaries, relationsByKey=relationsByKey, ) - - if tiltRef is None: - ctfRef = self._findInputRefByKind(inputRefs, "ctf") - if ctfRef is not None: - ctfInputRefs = self._listProtocolInputRefs( - ctfRef.get("parentProtocolDbId") - ) - self._mergeKindFromInputRefs( - kind="tiltSeries", - inputRefs=ctfInputRefs, - links=links, - summaries=summaries, - relationsByKey=relationsByKey, - ) - return if rootKind == "tomogram": From ae01f95d4fba9c942decb7a715d2fc3cbb28aed7 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 18:03:33 +0200 Subject: [PATCH 092/278] resolve tomogram relations from regular tilt series --- .../postgresql_integrated_context_reader.py | 78 ++++++++++++++----- 1 file changed, 59 insertions(+), 19 deletions(-) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 5591655b..8c505599 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -313,7 +313,35 @@ def _listProtocolInputRefs(self, protocolDbId: Any) -> List[Dict[str, Any]]: return [dict(row) for row in rows or []] def _getInputRefKind(self, inputRef: Dict[str, Any]) -> Optional[str]: - return self._getIntegratedKindFromText(inputRef.get("objectClassName")) + text = self._normalizeClassText(inputRef.get("objectClassName")) + + if "ctftomo" in text: + return "ctf" + + if self._isTiltSeriesMClassText(text): + return "tiltSeriesM" + + if self._isRegularTiltSeriesClassText(text): + return "tiltSeries" + + if "setoftomograms" in text or "tomogram" in text: + return "tomogram" + + if "setofcoordinates3d" in text or "coordinate3d" in text: + return "coordinates3d" + + return None + + def _isRegularTiltSeriesClassText(self, value: Any) -> bool: + text = self._normalizeClassText(value) + + if self._isTiltSeriesMClassText(text): + return False + + if "ctftomo" in text: + return False + + return "tiltseries" in text def _findInputRefByKind( self, @@ -470,6 +498,29 @@ def _mergeInputRefRelations( ) self._addCoordinates3dRelations(relationsByKey, items) + def _listRegularTiltSeriesItemsForProtocol( + self, + protocolDbId: Any, + ) -> List[Dict[str, Any]]: + storedSet = self._findRegularTiltSeriesStoredSetForProtocol(protocolDbId) + if storedSet is None: + return [] + + protocolDbId = storedSet.get("protocolDbId") + outputName = storedSet.get("outputName") + + if protocolDbId is None or not outputName: + return [] + + reader = PostgresqlTiltSeriesReader( + db=self.db, + projectId=self.projectId, + protocolId=protocolDbId, + outputName=outputName, + ) + + return reader.listTiltSeries() or [] + def _listTiltSeriesItemsFromInputRefs( self, inputRefs: List[Dict[str, Any]], @@ -511,11 +562,11 @@ def _mergeRootTomogramRelationsFromInputRefs( rootStoredSet: Dict[str, Any], relationsByKey: Dict[str, Dict[str, Any]], ) -> None: - inputRefs = self._listProtocolInputRefs(rootStoredSet.get("protocolDbId")) - if not inputRefs: + rootProtocolDbId = rootStoredSet.get("protocolDbId") + if rootProtocolDbId is None: return - tiltSeriesItems = self._listTiltSeriesItemsFromInputRefs(inputRefs) + tiltSeriesItems = self._listRegularTiltSeriesItemsForProtocol(rootProtocolDbId) if not tiltSeriesItems: return @@ -742,32 +793,21 @@ def _mergeExactInputRefs( return if rootKind == "tomogram": - ctfRef = self._mergeKindFromInputRefs( - kind="ctf", - inputRefs=inputRefs, + self._mergeRegularTiltSeriesForCtftomo( + rootProtocolDbId=rootProtocolDbId, links=links, summaries=summaries, relationsByKey=relationsByKey, ) - tiltRef = self._mergeKindFromInputRefs( - kind="tiltSeries", + self._mergeKindFromInputRefs( + kind="ctf", inputRefs=inputRefs, links=links, summaries=summaries, relationsByKey=relationsByKey, ) - if tiltRef is None and ctfRef is not None: - ctfInputRefs = self._listProtocolInputRefs(ctfRef.get("parentProtocolDbId")) - self._mergeKindFromInputRefs( - kind="tiltSeries", - inputRefs=ctfInputRefs, - links=links, - summaries=summaries, - relationsByKey=relationsByKey, - ) - return if rootKind == "coordinates3d": From 8318fd764c7a975639c859f241dd9c41fa4d5236 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 18:22:44 +0200 Subject: [PATCH 093/278] match tomogram relations by semantic ids --- app/backend/mapper/scipion_set_mapper.py | 2 +- .../postgresql_integrated_context_reader.py | 115 ++++++++++++++++-- 2 files changed, 104 insertions(+), 13 deletions(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index 1900be8c..e09c3841 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,7 +41,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 11 +NESTED_LOGICAL_TABLES_VERSION = 12 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 8c505599..6913ee0b 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -423,7 +423,6 @@ def _getRelationKeySet(self, relationsByKey: Dict[str, Dict[str, Any]]) -> Optio relation.get("ctfSeriesId"), relation.get("tomogramId"), relation.get("sourceTomoId"), - relation.get("tomogramVolumeId"), relation.get("coordinatesTomogramId"), ] @@ -557,6 +556,81 @@ def _listTiltSeriesItemsFromInputRefs( return reader.listTiltSeries() or [] + def _getItemMatchKeys( + self, + item: Dict[str, Any], + fields: List[str], + ) -> List[str]: + result = [] + seen = set() + + for field in fields: + text = self._toRelationMatchText((item or {}).get(field)) + if not text or text in seen: + continue + + seen.add(text) + result.append(text) + + return result + + def _getTiltSeriesItemMatchKeys(self, item: Dict[str, Any]) -> List[str]: + return self._getItemMatchKeys( + item, + [ + "tiltSeriesId", + "tsId", + "id", + "label", + "name", + ], + ) + + def _getTomogramItemMatchKeys(self, item: Dict[str, Any]) -> List[str]: + return self._getItemMatchKeys( + item, + [ + "tiltSeriesId", + "tsId", + "label", + "name", + "tomoId", + "tomogramId", + "sourceTomoId", + "id", + ], + ) + + def _buildItemMapByMatchKeys( + self, + items: List[Dict[str, Any]], + keyGetter, + ) -> Dict[str, Dict[str, Any]]: + result = {} + + for item in items or []: + for key in keyGetter(item): + result.setdefault(key, item) + + return result + + def _findTiltSeriesItemForTomogram( + self, + tomogramItem: Dict[str, Any], + tiltSeriesByKey: Dict[str, Dict[str, Any]], + tiltSeriesItems: List[Dict[str, Any]], + index: int, + ) -> Optional[Dict[str, Any]]: + for key in self._getTomogramItemMatchKeys(tomogramItem): + match = tiltSeriesByKey.get(key) + if match is not None: + return match + + if 0 <= index < len(tiltSeriesItems): + return tiltSeriesItems[index] + + return None + def _mergeRootTomogramRelationsFromInputRefs( self, rootStoredSet: Dict[str, Any], @@ -570,17 +644,29 @@ def _mergeRootTomogramRelationsFromInputRefs( if not tiltSeriesItems: return + tiltSeriesByKey = self._buildItemMapByMatchKeys( + tiltSeriesItems, + self._getTiltSeriesItemMatchKeys, + ) + tomogramItems = self._buildTomogramItemsFromStoredSet(rootStoredSet) for index, tomogramItem in enumerate(tomogramItems): - if index >= len(tiltSeriesItems): - break + tiltSeriesItem = self._findTiltSeriesItemForTomogram( + tomogramItem=tomogramItem, + tiltSeriesByKey=tiltSeriesByKey, + tiltSeriesItems=tiltSeriesItems, + index=index, + ) + + if tiltSeriesItem is None: + continue - tiltSeriesItem = tiltSeriesItems[index] tiltSeriesId = ( tiltSeriesItem.get("tiltSeriesId") or tiltSeriesItem.get("tsId") or tiltSeriesItem.get("id") + or tiltSeriesItem.get("label") ) if tiltSeriesId is None: @@ -595,11 +681,14 @@ def _mergeRootTomogramRelationsFromInputRefs( ) volumeId = ( - tomogramItem.get("tomogramVolumeId") - or tomogramItem.get("volumeId") - or index + tomogramItem.get("tomogramVolumeId") + if tomogramItem.get("tomogramVolumeId") is not None + else tomogramItem.get("volumeId") ) + if volumeId is None: + volumeId = index + label = ( tomogramItem.get("label") or tomogramItem.get("name") @@ -900,7 +989,6 @@ def _iterRelationMatchValues( values.get("ctfSeriesId"), values.get("tomogramId"), values.get("sourceTomoId"), - values.get("tomogramVolumeId"), values.get("coordinatesTomogramId"), ] @@ -1289,7 +1377,10 @@ def _shouldSkipDependencyCandidate( if rootKind == "coordinates3d" and candidateKind == "tomogram": return True - if rootKind in {"coordinates3d", "tomogram"} and candidateKind in {"tiltSeries", "ctf"}: + if rootKind == "coordinates3d" and candidateKind in {"tiltSeries", "ctf"}: + return True + + if rootKind == "tomogram" and candidateKind == "tiltSeries": return True return False @@ -1324,8 +1415,8 @@ def _mergeRelatedStoredSets( summaries[candidateKind] = self._buildSummary(candidate) allowedRelationKeys = None - if rootKind == "coordinates3d": - allowedRelationKeys = set(relationsByKey.keys()) + if rootKind in {"coordinates3d", "tomogram"}: + allowedRelationKeys = self._getRelationKeySet(relationsByKey) self._mergeRelationsForCandidate( candidate=candidate, @@ -1348,6 +1439,7 @@ def _filterIntegratedItemsByAllowedKeys( candidates = [ item.get("key"), item.get("label"), + item.get("name"), item.get("id"), item.get("tomoId"), item.get("tomogramId"), @@ -1356,7 +1448,6 @@ def _filterIntegratedItemsByAllowedKeys( item.get("coordinatesTomogramId"), item.get("tsId"), item.get("sourceTomoId"), - item.get("tomogramVolumeId"), ] if any( From 3918cded01e67815b8b1c7be5fc8b7707f7246c9 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 19:44:00 +0200 Subject: [PATCH 094/278] Store explicit tomogram tilt series ids --- app/backend/mapper/scipion_set_mapper.py | 57 ++++++++++++++++++- .../postgresql_integrated_context_reader.py | 9 --- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index e09c3841..de795622 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,7 +41,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 12 +NESTED_LOGICAL_TABLES_VERSION = 13 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): @@ -1015,6 +1015,11 @@ def _getItemValues(self, item: Any, scipionSet: Optional[Any] = None) -> Dict[st if str(label) != SELF_LABEL } + self._addRelationIdentityValues( + item=item, + values=values, + ) + classSize = self._getClassItemSize(item) if classSize is not None and "_size" not in values: values["_size"] = classSize @@ -1027,6 +1032,56 @@ def _getItemValues(self, item: Any, scipionSet: Optional[Any] = None) -> Dict[st return values + def _addRelationIdentityValues( + self, + item: Any, + values: Dict[str, Any], + ) -> None: + tsId = self._getFirstGetterValue( + item, + ("getTsId", "getTiltSeriesId"), + ) + if tsId is not None and not values.get("_tsId"): + values["_tsId"] = self._toJsonValue(tsId) + + tomoId = self._getFirstGetterValue( + item, + ("getTomoId",), + ) + if tomoId is not None and not values.get("_tomoId"): + values["_tomoId"] = self._toJsonValue(tomoId) + + def _getFirstGetterValue( + self, + item: Any, + getterNames: Tuple[str, ...], + ) -> Optional[Any]: + for getterName in getterNames: + getter = getattr(item, getterName, None) + if not callable(getter): + continue + + try: + value = getter() + except Exception: + continue + + getterValue = getattr(value, "get", None) + if callable(getterValue): + try: + value = getterValue() + except Exception: + continue + + if value is None: + continue + + text = str(value).strip() + if text: + return value + + return None + def _getClassItemSize(self, item: Any) -> Optional[int]: className = self._getClassName(item) or item.__class__.__name__ if not str(className or "").startswith("Class"): diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index 6913ee0b..c00ef1d6 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -592,12 +592,6 @@ def _getTomogramItemMatchKeys(self, item: Dict[str, Any]) -> List[str]: [ "tiltSeriesId", "tsId", - "label", - "name", - "tomoId", - "tomogramId", - "sourceTomoId", - "id", ], ) @@ -626,9 +620,6 @@ def _findTiltSeriesItemForTomogram( if match is not None: return match - if 0 <= index < len(tiltSeriesItems): - return tiltSeriesItems[index] - return None def _mergeRootTomogramRelationsFromInputRefs( From 2ed2264f9540d3387608ff4e8e9438c1bd28ae41 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 22 Jun 2026 20:56:14 +0200 Subject: [PATCH 095/278] preserve tomogram volume selectors --- .../viewers/postgresql_integrated_context_reader.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index c00ef1d6..c6bf9e94 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -1111,7 +1111,14 @@ def _addTomogramRelations( relationKey = tiltSeriesId or tomogramId ctfSeriesId = item.get("ctfSeriesId") or tiltSeriesId label = item.get("label") or item.get("name") or str(tomogramId) - volumeId = item.get("tomogramVolumeId") or item.get("volumeId") or index + volumeId = ( + item.get("tomogramVolumeId") + if item.get("tomogramVolumeId") is not None + else item.get("volumeId") + ) + + if volumeId is None: + volumeId = index self._addRelation( relationsByKey, @@ -1140,14 +1147,12 @@ def _addCoordinates3dRelations( ) label = item.get("label") or item.get("name") or str(tomogramId) - volumeId = item.get("tomogramVolumeId") or item.get("volumeId") or index self._addRelation( relationsByKey, tomogramId, coordinatesTomogramId=tomogramId, tomogramId=tomogramId, - tomogramVolumeId=volumeId, label=label, ) From fbc2435df33e4ebc35a42d97695150c6e33e6319 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 10:23:44 +0200 Subject: [PATCH 096/278] Fix integrated metadata links and schema resolution --- app/backend/api/routers/project_router.py | 47 +++++++++++++++---- app/backend/api/services/project_service.py | 26 ---------- app/backend/mapper/scipion_set_mapper.py | 2 +- .../postgresql_integrated_context_reader.py | 26 ++++++++-- 4 files changed, 60 insertions(+), 41 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index fa0a3e57..298b3a20 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2554,7 +2554,7 @@ def listOutputMetadataTables( """ List logical metadata tables (blocks) associated with a given output. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2591,7 +2591,7 @@ def getMetadataTableSchema( """ Return logical schema for one metadata table: columns, renderers, flags. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2627,7 +2627,7 @@ def runMetadataTableAction( service: ProjectService = Depends(getProjectService), ): # runMetadataTableAction - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2720,7 +2720,7 @@ def getMetadataTablePage( """ Return one logical page of rows for a metadata table. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2779,7 +2779,7 @@ def exportMetadataTable( """ Export a metadata table (full or subset) as CSV/XLSX. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2865,7 +2865,7 @@ def renderMetadataImageCell( """ Render one image cell from a metadata table using the same logic as ImageRenderer. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2928,7 +2928,7 @@ def getMetadataTableWindow( """ Return a window (offset + limit) of rows for a metadata table. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) if not project: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") @@ -2979,6 +2979,31 @@ def listExternalViewers( refresh=False, checkPid=False, ) + if not project: + raise + +@router.get( + "/{projectId}/protocols/{protocolId}/outputs/{outputName}/external-viewers", + response_model=Any, + status_code=status.HTTP_200_OK, +) +def listExternalViewers( + projectId: int, + protocolId: int, + outputName: str, + objectId: Optional[Union[str, int]] = Query(None), + objectKind: Optional[str] = Query(None), + currentUser=Depends(getCurrentUser), + mapper: PostgresqlFlatMapper = Depends(getMapper), + service: ProjectService = Depends(getProjectService), +): + project = service.getProjectById( + mapper, + projectId, + currentUser, + refresh=True, + checkPid=False, + ) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2989,7 +3014,9 @@ def listExternalViewers( objectId=objectId, objectKind=objectKind, ) - return {"viewers": viewers} + + return {"viewers": viewers or []} + except HTTPException: raise except Exception as e: @@ -3019,7 +3046,7 @@ def launchExternalViewer( mapper, projectId, currentUser, - refresh=False, + refresh=True, checkPid=False, ) if not project: @@ -3036,6 +3063,7 @@ def launchExternalViewer( objectKind=payload.objectKind, params=payload.params or {}, ) + except HTTPException: raise except Exception as e: @@ -3044,7 +3072,6 @@ def launchExternalViewer( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to launch external viewer: {e}", ) - # ====================================================================== # PROTOCOL TAGS # ====================================================================== diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index aed7a1f9..13ca5c3b 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -10069,32 +10069,6 @@ def getMetadataTableSchemaService(self, projectId: int, visibleLabels = [] orderLabels = [] renderLabels = [] - - if table.getName() == SQLITE_OBJECT_TABLE: - from pwem.viewers.viewers_data import RegistryViewerConfig - protocol = self.currentProject.getProtocol(int(protocolId)) - output = getattr(protocol, outputName) - - config = RegistryViewerConfig.getConfig(type(output)) or {} - - fileNameLabel = ' _filename' - stackLabel = ' stack' - - visibleLabelsStr = config.get(VISIBLE, '') - orderLabelsStr = config.get(ORDER, '') - renderLabelsStr = config.get(RENDER, '') - - orderLabelsStr = orderLabelsStr.replace(fileNameLabel, stackLabel, 1) - renderLabelsStr = renderLabelsStr.replace(fileNameLabel, stackLabel, 1) - - if fileNameLabel in visibleLabelsStr and stackLabel not in renderLabelsStr: - renderLabelsStr += stackLabel - visibleLabelsStr += stackLabel - - visibleLabels = visibleLabelsStr.split() - orderLabels = orderLabelsStr.split() - renderLabels = renderLabelsStr.split() - actions = self._getMetadataTableActionNames(table) try: diff --git a/app/backend/mapper/scipion_set_mapper.py b/app/backend/mapper/scipion_set_mapper.py index de795622..719c4c88 100644 --- a/app/backend/mapper/scipion_set_mapper.py +++ b/app/backend/mapper/scipion_set_mapper.py @@ -41,7 +41,7 @@ SELF_LABEL = "self" -NESTED_LOGICAL_TABLES_VERSION = 13 +NESTED_LOGICAL_TABLES_VERSION = 14 class ScipionSetPostgresqlMapper(ScipionObjectPostgresqlMapper): diff --git a/app/backend/viewers/postgresql_integrated_context_reader.py b/app/backend/viewers/postgresql_integrated_context_reader.py index c6bf9e94..d68e5bdb 100644 --- a/app/backend/viewers/postgresql_integrated_context_reader.py +++ b/app/backend/viewers/postgresql_integrated_context_reader.py @@ -242,7 +242,7 @@ def _buildLink( label: Optional[str] = None, statusValue: str = "available", ) -> Dict[str, Any]: - return { + link = { "protocolId": protocolId, "outputName": outputName, "itemId": storedSet.get("objectId") or storedSet.get("id"), @@ -250,6 +250,12 @@ def _buildLink( "status": statusValue, } + publicProtocolId = storedSet.get("publicProtocolId") or storedSet.get("protocolId") + if publicProtocolId is not None and str(publicProtocolId) != str(protocolId): + link["publicProtocolId"] = publicProtocolId + + return link + def _buildSummary( self, storedSet: Dict[str, Any], @@ -820,7 +826,8 @@ def _mergeKindFromInputRefs( summaries: Dict[str, Optional[Dict[str, Any]]], relationsByKey: Dict[str, Dict[str, Any]], ) -> Optional[Dict[str, Any]]: - if links.get(kind) is not None: + existingLink = links.get(kind) + if existingLink is not None and not self._shouldReplaceLink(existingLink): return None inputRef = self._findInputRefByKind(inputRefs, kind) @@ -891,6 +898,14 @@ def _mergeExactInputRefs( return if rootKind == "coordinates3d": + tomogramRef = self._mergeKindFromInputRefs( + kind="tomogram", + inputRefs=inputRefs, + links=links, + summaries=summaries, + relationsByKey=relationsByKey, + ) + ctfRef = self._mergeKindFromInputRefs( kind="ctf", inputRefs=inputRefs, @@ -907,7 +922,6 @@ def _mergeExactInputRefs( relationsByKey=relationsByKey, ) - tomogramRef = self._findInputRefByKind(inputRefs, "tomogram") if tomogramRef is not None: tomogramInputRefs = self._listProtocolInputRefs( tomogramRef.get("parentProtocolDbId") @@ -1548,14 +1562,18 @@ def _shouldReplaceLink( protocolId = existingLink.get("protocolId") outputName = existingLink.get("outputName") statusValue = str(existingLink.get("status") or "") + sourceValue = str(existingLink.get("source") or "") if protocolId is None and outputName is None: return True + if statusValue == "derived" and sourceValue == "coordinates3d": + return True + return statusValue == "inferred" and protocolId is None def _getCandidateProtocolId(self, candidate: Dict[str, Any]) -> Any: - return candidate.get("publicProtocolId") or candidate.get("protocolDbId") + return candidate.get("protocolDbId") or candidate.get("publicProtocolId") def _safeValue(self, value: Any) -> Any: if isinstance(value, dict): From d39d346db228b5fdc3d2266811284a8c4e557527 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 10:50:37 +0200 Subject: [PATCH 097/278] Sync filtered TiltSeries outputs with PostgreSQL --- app/backend/api/routers/project_router.py | 9 +- app/backend/api/services/project_service.py | 111 ++++++++++++++++++-- 2 files changed, 108 insertions(+), 12 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 298b3a20..f850b302 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1958,14 +1958,14 @@ def createNewSetOfTiltSeries( ): """ Create a new tilt-series output applying the given exclusions. - The backend is expected to duplicate the SetOfTiltSeries and - remove excluded views, optionally restacking files on disk. + The backend duplicates the SetOfTiltSeries and removes excluded views, + optionally restacking files on disk. """ project = service.getProjectById( mapper, projectId, currentUser, - refresh=False, + refresh=True, checkPid=False, ) if not project: @@ -1978,6 +1978,7 @@ def createNewSetOfTiltSeries( outputName=outputName, exclusions=payload.exclusions, restack=payload.restack, + mapper=mapper, ) from fastapi.responses import JSONResponse @@ -1998,10 +1999,10 @@ def createNewSetOfTiltSeries( detail=f"Failed to create new tilt-series set: {e}", ) - # ============================================================================== # ANALYZE RESULTS: CTF TOMOGRAPHY (SetOfCTFTomoSeries) # ============================================================================== + class CtftomoNewSetRequest(BaseModel): """ Request payload for creating a new SetOfCTFTomoSeries based on exclusions. diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 13ca5c3b..bbf017be 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6444,16 +6444,66 @@ def writeRemoteFileService( # ---------------------------------------------------------------------- # Internal helpers for TiltSeries (SetOfTiltSeries) # ---------------------------------------------------------------------- + def _resolveScipionProtocolId( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + ) -> int: + if mapper is None or projectId is None: + return int(protocolId) + + try: + row = mapper.db.fetchOne( + """ + SELECT "protocolId" + FROM protocols + WHERE "projectId" = %s + AND (id = %s OR "protocolId" = %s) + LIMIT 1 + """, + (projectId, protocolId, str(protocolId)), + ) + + if row: + value = row.get("protocolId") if isinstance(row, dict) else row[0] + if value is not None: + return int(value) + + except Exception: + logger.exception( + "Failed to resolve Scipion protocol id from PostgreSQL. projectId=%s protocolId=%s", + projectId, + protocolId, + ) - @lru_cache - def _resolveOutputForTiltSeries(self, protocolId: int, outputName: str): + return int(protocolId) + + def _resolveOutputForTiltSeries( + self, + protocolId: int, + outputName: str, + projectId: Optional[int] = None, + mapper=None, + ): """ Resolve protocol + SetOfTiltSeries-like output for tilt series operations. + + protocolId can be either the PostgreSQL protocols.id or the Scipion protocolId. """ + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + try: - protocol = self.currentProject.getProtocol(int(protocolId)) + protocol = self.currentProject.getProtocol(int(scipionProtocolId)) except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + raise HTTPException( + status_code=404, + detail=f"Protocol not found: {protocolId}", + ) if not hasattr(protocol, outputName): raise HTTPException( @@ -7642,7 +7692,12 @@ def listOutputTiltSeriesService( if pgReader is not None: return pgReader.listTiltSeries() - _, setOfTiltSeries = self._resolveOutputForTiltSeries(protocolId, outputName) + _, setOfTiltSeries = self._resolveOutputForTiltSeries( + protocolId=protocolId, + outputName=outputName, + projectId=projectId, + mapper=mapper, + ) seriesList: List[Dict[str, Any]] = [] for idx, ts in enumerate(setOfTiltSeries.iterItems(iterate=False)): @@ -7699,7 +7754,12 @@ def getTiltSeriesFramesService( if payload is not None: return payload - protocol, setOfTiltSeries = self._resolveOutputForTiltSeries(protocolId, outputName) + protocol, setOfTiltSeries = self._resolveOutputForTiltSeries( + protocolId=protocolId, + outputName=outputName, + projectId=projectId, + mapper=mapper, + ) targetKey = str(tiltSeriesId) selectedSummary: Optional[Dict[str, Any]] = None selectedSeries = setOfTiltSeries.getItem('_tsId', targetKey) @@ -8137,7 +8197,12 @@ def renderTiltSeriesImageService( ) - protocol, setOfTiltSeries = self._resolveOutputForTiltSeries(protocolId, outputName) + protocol, setOfTiltSeries = self._resolveOutputForTiltSeries( + protocolId=protocolId, + outputName=outputName, + projectId=projectId, + mapper=mapper, + ) ts = setOfTiltSeries.getItem('_tsId', tiltSeriesId) if ts is None: @@ -8299,6 +8364,7 @@ def createNewSetOfTiltSeriesService( outputName: str, exclusions: Dict[str, Any], restack: bool, + mapper=None ) -> Dict[str, Any]: """ Create a new SetOfTiltSeries applying per-tilt-series and per-tilt exclusions. @@ -8315,7 +8381,12 @@ def createNewSetOfTiltSeriesService( Returns a JSON-serializable payload for the web UI with basic metadata. """ # Resolve protocol and input set for the given output - protocol, inputSet = self._resolveOutputForTiltSeries(protocolId, outputName) + protocol, inputSet = self._resolveOutputForTiltSeries( + protocolId=protocolId, + outputName=outputName, + projectId=projectId, + mapper=mapper, + ) hasOddEven = inputSet.hasOddEven() # Normalize exclusions keys to strings (tsId can be int/str on the backend) @@ -8495,6 +8566,29 @@ def createNewSetOfTiltSeriesService( outputSet.write() protocol._defineOutputs(**{newOutputName: outputSet}) protocol._store() + + postgresqlSync = None + + if mapper is not None: + try: + from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + setMapper = ScipionSetPostgresqlMapper(mapper.db) + postgresqlSync = setMapper.storeSet( + projectId=projectId, + protocolDbId=protocolId, + outputName=newOutputName, + scipionSet=outputSet, + ) + + except Exception: + logger.exception( + "Failed to persist created TiltSeries output to PostgreSQL. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + newOutputName, + ) + logger.info("The new set (%s) has been created successfully", newOutputName) return { @@ -8503,6 +8597,7 @@ def createNewSetOfTiltSeriesService( "createdTiltSeries": createdCount, "hasOddEven": bool(hasOddEven), "restack": bool(restack), + "postgresqlSync": postgresqlSync, } def _cloneTiltImage(self, ti, included): From 5c1563c4825e62d306b890093e741b0fea8d3576 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 11:43:59 +0200 Subject: [PATCH 098/278] Encapsulate Scipion runtime protocol resolution --- app/backend/api/services/project_service.py | 71 ++++++++++++++++++--- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index bbf017be..f5f86d5e 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6479,6 +6479,67 @@ def _resolveScipionProtocolId( return int(protocolId) + def _resolveScipionProtocolId( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + ) -> int: + if mapper is None or projectId is None: + return int(protocolId) + + try: + row = mapper.db.fetchOne( + """ + SELECT "protocolId" + FROM protocols + WHERE "projectId" = %s + AND (id = %s OR "protocolId" = %s) + LIMIT 1 + """, + (projectId, protocolId, str(protocolId)), + ) + + if row: + value = row.get("protocolId") if isinstance(row, dict) else row[0] + if value is not None: + return int(value) + + except Exception: + logger.exception( + "Failed to resolve Scipion protocol id from PostgreSQL. projectId=%s protocolId=%s", + projectId, + protocolId, + ) + + return int(protocolId) + + def _getScipionProtocolForRuntime( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + ): + if self.currentProject is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="No current Scipion project loaded", + ) + + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + try: + return self.currentProject.getProtocol(int(scipionProtocolId)) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol not found in Scipion runtime: {protocolId}. {e}", + ) + def _resolveOutputForTiltSeries( self, protocolId: int, @@ -6491,20 +6552,12 @@ def _resolveOutputForTiltSeries( protocolId can be either the PostgreSQL protocols.id or the Scipion protocolId. """ - scipionProtocolId = self._resolveScipionProtocolId( + protocol = self._getScipionProtocolForRuntime( mapper=mapper, projectId=projectId, protocolId=protocolId, ) - try: - protocol = self.currentProject.getProtocol(int(scipionProtocolId)) - except Exception: - raise HTTPException( - status_code=404, - detail=f"Protocol not found: {protocolId}", - ) - if not hasattr(protocol, outputName): raise HTTPException( status_code=404, From 159771d04929859d479b7f472765551d189334c1 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 12:10:38 +0200 Subject: [PATCH 099/278] Centralize Scipion runtime protocol resolution --- app/backend/api/services/project_service.py | 247 ++++++++++++-------- 1 file changed, 144 insertions(+), 103 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index f5f86d5e..bbee6af7 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -661,6 +661,148 @@ def _getPointerParentProtocolId(self, pointer: Any, targetObj: Any) -> Optional[ return None + # ------------------------------------------------------------------ + # Scipion runtime protocol resolution + # ------------------------------------------------------------------ + def _resolveScipionProtocolId( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + ) -> int: + rawProtocolId = str(protocolId).strip() + + if not rawProtocolId: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Missing protocol id", + ) + + if mapper is None or projectId is None: + try: + return int(rawProtocolId) + except Exception: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid Scipion protocol id: {protocolId}", + ) + + protocolDbIdCandidate = None + try: + protocolDbIdCandidate = int(rawProtocolId) + except Exception: + protocolDbIdCandidate = None + + try: + if protocolDbIdCandidate is not None: + row = mapper.db.fetchOne( + """ + SELECT "protocolId" + FROM protocols + WHERE "projectId" = %s + AND (id = %s OR "protocolId" = %s) + LIMIT 1 + """, + (projectId, protocolDbIdCandidate, rawProtocolId), + ) + else: + row = mapper.db.fetchOne( + """ + SELECT "protocolId" + FROM protocols + WHERE "projectId" = %s + AND "protocolId" = %s + LIMIT 1 + """, + (projectId, rawProtocolId), + ) + + if row: + value = row.get("protocolId") if isinstance(row, dict) else row[0] + if value is not None: + return int(value) + + except HTTPException: + raise + except Exception: + logger.exception( + "Failed to resolve Scipion protocol id from PostgreSQL. projectId=%s protocolId=%s", + projectId, + protocolId, + ) + + try: + return int(rawProtocolId) + except Exception: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol not found in PostgreSQL: {protocolId}", + ) + + def _getScipionProtocolByRuntimeId( + self, + protocolId: Union[int, str], + ): + if self.currentProject is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="No current Scipion project loaded", + ) + + try: + protocol = self.currentProject.getProtocol(int(protocolId)) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol not found in Scipion runtime: {protocolId}. {e}", + ) + + if protocol is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol not found in Scipion runtime: {protocolId}", + ) + + return protocol + + def _tryGetScipionProtocolByRuntimeId( + self, + protocolId: Union[int, str], + ): + try: + return self._getScipionProtocolByRuntimeId(protocolId) + except Exception: + return None + + def _getScipionProtocolForRuntime( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + ): + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + return self._getScipionProtocolByRuntimeId(scipionProtocolId) + + def _tryGetScipionProtocolForRuntime( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + ): + try: + return self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + except Exception: + return None + def _getPointerOutputName(self, pointer: Any) -> Optional[str]: outputName = self._safeCall(pointer, "getExtended", None) if outputName is None: @@ -751,10 +893,7 @@ def syncProjectProtocolsAndDependencies( protocol = getattr(nodeObj, "run", None) if protocol is None: - try: - protocol = self.currentProject.getProtocol(int(nodeId)) - except Exception: - protocol = None + protocol = self._tryGetScipionProtocolByRuntimeId(nodeId) if protocol is None: continue @@ -1654,10 +1793,7 @@ def sortKey(row: Dict[str, Any]): protocol = liveRuns.get(nodeId) if protocol is None: - try: - protocol = self.currentProject.getProtocol(int(nodeId)) - except Exception: - protocol = None + protocol = self._tryGetScipionProtocolByRuntimeId(nodeId) if protocol is not None: try: @@ -6444,101 +6580,6 @@ def writeRemoteFileService( # ---------------------------------------------------------------------- # Internal helpers for TiltSeries (SetOfTiltSeries) # ---------------------------------------------------------------------- - def _resolveScipionProtocolId( - self, - mapper, - projectId: Optional[int], - protocolId: Union[int, str], - ) -> int: - if mapper is None or projectId is None: - return int(protocolId) - - try: - row = mapper.db.fetchOne( - """ - SELECT "protocolId" - FROM protocols - WHERE "projectId" = %s - AND (id = %s OR "protocolId" = %s) - LIMIT 1 - """, - (projectId, protocolId, str(protocolId)), - ) - - if row: - value = row.get("protocolId") if isinstance(row, dict) else row[0] - if value is not None: - return int(value) - - except Exception: - logger.exception( - "Failed to resolve Scipion protocol id from PostgreSQL. projectId=%s protocolId=%s", - projectId, - protocolId, - ) - - return int(protocolId) - - def _resolveScipionProtocolId( - self, - mapper, - projectId: Optional[int], - protocolId: Union[int, str], - ) -> int: - if mapper is None or projectId is None: - return int(protocolId) - - try: - row = mapper.db.fetchOne( - """ - SELECT "protocolId" - FROM protocols - WHERE "projectId" = %s - AND (id = %s OR "protocolId" = %s) - LIMIT 1 - """, - (projectId, protocolId, str(protocolId)), - ) - - if row: - value = row.get("protocolId") if isinstance(row, dict) else row[0] - if value is not None: - return int(value) - - except Exception: - logger.exception( - "Failed to resolve Scipion protocol id from PostgreSQL. projectId=%s protocolId=%s", - projectId, - protocolId, - ) - - return int(protocolId) - - def _getScipionProtocolForRuntime( - self, - mapper, - projectId: Optional[int], - protocolId: Union[int, str], - ): - if self.currentProject is None: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="No current Scipion project loaded", - ) - - scipionProtocolId = self._resolveScipionProtocolId( - mapper=mapper, - projectId=projectId, - protocolId=protocolId, - ) - - try: - return self.currentProject.getProtocol(int(scipionProtocolId)) - except Exception as e: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found in Scipion runtime: {protocolId}. {e}", - ) def _resolveOutputForTiltSeries( self, From 43e1661e61f3d980c588be397bec451a0609dc8b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 12:21:26 +0200 Subject: [PATCH 100/278] Resolve runtime protocols in log and rename services --- app/backend/api/routers/project_router.py | 4 ++ app/backend/api/routers/protocol_router.py | 9 +++- app/backend/api/services/project_service.py | 60 +++++++++++++++------ 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index f850b302..7b7b8c07 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -592,6 +592,8 @@ def renameProtocol( ) service.renameProtocol( + mapper, + projectId, protocolId, newNameText.strip(), str(newComment or "").strip(), @@ -1009,6 +1011,7 @@ def listProtocolLogChannels( channels = service.listProtocolLogChannelsService( projectId=projectId, protocolId=protocolId, + mapper=mapper, ) # normalizeChannels @@ -1072,6 +1075,7 @@ def pollProtocolLogs( offsets=offsets, maxBytes=maxBytes, maxLines=maxLines, + mapper=mapper, ) # normalizePollResponse diff --git a/app/backend/api/routers/protocol_router.py b/app/backend/api/routers/protocol_router.py index 3d264448..f5ce57eb 100644 --- a/app/backend/api/routers/protocol_router.py +++ b/app/backend/api/routers/protocol_router.py @@ -101,4 +101,11 @@ async def getProtocolLogs( if not project: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") - return service.getProtocolLogs(projectId, protocolId, offset, errOffset, scheduleOffset) + return service.getProtocolLogs( + projectId, + protocolId, + offset, + errOffset, + scheduleOffset, + mapper=mapper, + ) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index bbee6af7..b971bce0 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -4801,14 +4801,20 @@ def getProtocolName(self, node): return 'default' - def listProtocolLogChannelsService(self, projectId: int, protocolId: int): + def listProtocolLogChannelsService( + self, + projectId: int, + protocolId: int, + mapper=None, + ): """ Return available log channels for a protocol, including paths and basic file stats. """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) # Resolve log paths from Scipion protocol object stdoutPath = protocol.getStdoutLog() if hasattr(protocol, "getStdoutLog") else None @@ -4862,14 +4868,16 @@ def pollProtocolLogsService( offsets: Dict[str, int], maxBytes: Optional[int] = 65536, maxLines: Optional[int] = 2000, + mapper=None, ): """ Incrementally read protocol logs from the given offsets, applying maxBytes and maxLines limits. """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) stdoutPath = protocol.getStdoutLog() if hasattr(protocol, "getStdoutLog") else None stderrPath = protocol.getStderrLog() if hasattr(protocol, "getStderrLog") else None @@ -5028,11 +5036,20 @@ def readChunk(filePath: Optional[str], startOffset: int) -> Dict[str, Any]: }, } - def getProtocolLogs(self, projectId: int, protocolId: int, - offset: int = 0, - errOffset: int = 0, - scheduleOffset: int = 0): - protocol = self.currentProject.getProtocol(int(protocolId)) + def getProtocolLogs( + self, + projectId: int, + protocolId: int, + offset: int = 0, + errOffset: int = 0, + scheduleOffset: int = 0, + mapper=None, + ): + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) logPath = protocol.getStdoutLog() errLogPath = protocol.getStderrLog() scheduleLogPath = protocol.getScheduleLog() @@ -5089,8 +5106,19 @@ def _buildProtocolMutationResult(message: str, **extra) -> Dict[str, Any]: return result - def renameProtocol(self, protocolId, newName, newComment): - protocol = self.currentProject.getProtocol(int(protocolId)) + def renameProtocol( + self, + mapper, + projectId: int, + protocolId, + newName, + newComment, + ): + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) if protocol is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, From 7128d0d983519d821b32d3d8b53f47d3d6ff17dc Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 12:38:04 +0200 Subject: [PATCH 101/278] Purge PostgreSQL outputs before protocol restart --- app/backend/api/routers/project_router.py | 4 +- app/backend/api/services/project_service.py | 330 ++++++++++++++++++-- 2 files changed, 312 insertions(+), 22 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 7b7b8c07..decd718e 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -740,7 +740,7 @@ def restartProtocolAll( ) try: - service.restartProtocolAll(protocolId) + service.restartProtocolAll(mapper, projectId, protocolId) service.syncProjectGraphAfterMutation( mapper, projectId, @@ -832,7 +832,7 @@ def resetProtocolFrom( ) try: - service.resetProtocolFrom(protocolId) + service.resetProtocolFrom(mapper, projectId, protocolId) service.syncProjectGraphAfterMutation( mapper, projectId, diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index b971bce0..e8ee1d7c 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -803,6 +803,250 @@ def _tryGetScipionProtocolForRuntime( except Exception: return None + def _resolvePostgresqlProtocolDbId( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + ) -> Optional[int]: + if mapper is None or projectId is None or protocolId is None: + return None + + rawProtocolId = str(protocolId).strip() + if not rawProtocolId: + return None + + protocolDbIdCandidate = None + try: + protocolDbIdCandidate = int(rawProtocolId) + except Exception: + protocolDbIdCandidate = None + + try: + if protocolDbIdCandidate is not None: + row = mapper.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE "projectId" = %s + AND (id = %s OR "protocolId" = %s) + LIMIT 1 + """, + (projectId, protocolDbIdCandidate, rawProtocolId), + ) + else: + row = mapper.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE "projectId" = %s + AND "protocolId" = %s + LIMIT 1 + """, + (projectId, rawProtocolId), + ) + + if row: + value = row.get("id") if isinstance(row, dict) else row[0] + if value is not None: + return int(value) + + except Exception: + logger.exception( + "Failed to resolve PostgreSQL protocol id. projectId=%s protocolId=%s", + projectId, + protocolId, + ) + + return None + + + def _deletePersistedProtocolOutputsFromPostgresql( + self, + mapper, + projectId: int, + protocolId: Union[int, str], + ) -> Dict[str, Any]: + protocolDbId = self._resolvePostgresqlProtocolDbId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + if protocolDbId is None: + return { + "protocolDbId": None, + "setsDeleted": 0, + "objectsDeleted": 0, + "skipped": True, + "reason": "protocol_not_found", + } + + setRows = mapper.db.fetchAll( + """ + SELECT id + FROM scipion_sets + WHERE "projectId" = %s + AND "protocolDbId" = %s + """, + (projectId, protocolDbId), + ) + + setIds = [ + int(row.get("id") if isinstance(row, dict) else row[0]) + for row in (setRows or []) + if (row.get("id") if isinstance(row, dict) else row[0]) is not None + ] + + setsDeleted = 0 + objectsDeleted = 0 + + with mapper.db.transaction(): + if setIds: + mapper.db.execute( + """ + DELETE FROM scipion_set_table_items + WHERE "tableId" IN ( + SELECT id + FROM scipion_set_tables + WHERE "setId" = ANY(%s) + ) + """, + (setIds,), + commit=False, + ) + + mapper.db.execute( + """ + DELETE FROM scipion_set_table_columns + WHERE "tableId" IN ( + SELECT id + FROM scipion_set_tables + WHERE "setId" = ANY(%s) + ) + """, + (setIds,), + commit=False, + ) + + mapper.db.execute( + """ + DELETE FROM scipion_set_tables + WHERE "setId" = ANY(%s) + """, + (setIds,), + commit=False, + ) + + mapper.db.execute( + """ + DELETE FROM scipion_set_items + WHERE "setId" = ANY(%s) + """, + (setIds,), + commit=False, + ) + + mapper.db.execute( + """ + DELETE FROM scipion_set_columns + WHERE "setId" = ANY(%s) + """, + (setIds,), + commit=False, + ) + + mapper.db.execute( + """ + DELETE FROM scipion_set_properties + WHERE "setId" = ANY(%s) + """, + (setIds,), + commit=False, + ) + + cur = mapper.db.execute( + """ + DELETE FROM scipion_sets + WHERE id = ANY(%s) + """, + (setIds,), + commit=False, + ) + setsDeleted = int(cur.rowcount or 0) + + cur = mapper.db.execute( + """ + WITH RECURSIVE object_tree AS ( + SELECT id + FROM scipion_objects + WHERE "projectId" = %s + AND "protocolDbId" = %s + + UNION ALL + + SELECT child.id + FROM scipion_objects child + JOIN object_tree parent + ON child."parentObjectId" = parent.id + ) + DELETE FROM scipion_objects + WHERE id IN (SELECT id FROM object_tree) + """, + (projectId, protocolDbId), + commit=False, + ) + objectsDeleted = int(cur.rowcount or 0) + + return { + "protocolDbId": protocolDbId, + "setsDeleted": setsDeleted, + "objectsDeleted": objectsDeleted, + "skipped": False, + } + + def _deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql( + self, + mapper, + projectId: int, + protocols: List[Any], + ) -> Dict[str, Any]: + cleanupItems = [] + totalSetsDeleted = 0 + totalObjectsDeleted = 0 + + for protocol in protocols or []: + protocolId = None + + try: + protocolId = protocol.getObjId() + except Exception: + protocolId = protocol + + if protocolId is None: + continue + + cleanupInfo = self._deletePersistedProtocolOutputsFromPostgresql( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + cleanupItems.append({ + "protocolId": str(protocolId), + **cleanupInfo, + }) + + totalSetsDeleted += int(cleanupInfo.get("setsDeleted") or 0) + totalObjectsDeleted += int(cleanupInfo.get("objectsDeleted") or 0) + + return { + "protocolsCount": len(cleanupItems), + "setsDeleted": totalSetsDeleted, + "objectsDeleted": totalObjectsDeleted, + "items": cleanupItems, + } + def _getPointerOutputName(self, pointer: Any) -> Optional[str]: outputName = self._safeCall(pointer, "getExtended", None) if outputName is None: @@ -4496,6 +4740,20 @@ def launchProtocol(self, mapper, projectId, protocolId, protocolClassName, param } runMode = modeToRunMode[executeMode] protocol.runMode.set(runMode) + + if executeMode == "restart": + cleanupInfo = self._deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql( + mapper=mapper, + projectId=projectId, + protocols=[protocol], + ) + logger.info( + "Deleted persisted protocol outputs before restart. projectId=%s protocolId=%s cleanup=%s", + projectId, + getattr(protocol, "getObjId", lambda: protocolId)(), + cleanupInfo, + ) + self.currentProject.launchProtocol(protocol) self.syncProjectProtocolsAndDependencies( @@ -5241,19 +5499,19 @@ def deleteProtocol(self, mapper, projectId, protocols: Any): except Exception as e: raise HTTPException(status_code=500, detail=str(e)) - def restartProtocolAll(self, protocolId): - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + def restartProtocolAll(self, mapper, projectId: int, protocolId): + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) try: workflowProtocolList, _activeProtocolList = self.currentProject._getSubworkflow(protocol) except Exception as e: logger.exception( - "Failed to resolve subworkflow for restart-all. protocolId=%s", + "Failed to resolve subworkflow for restart-all. projectId=%s protocolId=%s", + projectId, protocolId, ) raise HTTPException( @@ -5261,12 +5519,25 @@ def restartProtocolAll(self, protocolId): detail=f"Failed to resolve protocol subworkflow: {e}", ) + cleanupInfo = self._deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql( + mapper=mapper, + projectId=projectId, + protocols=workflowProtocolList, + ) + logger.info( + "Deleted persisted protocol outputs before restart-all. projectId=%s protocolId=%s cleanup=%s", + projectId, + protocolId, + cleanupInfo, + ) + errorList = [] try: self.currentProject._restartWorkflow(errorList, workflowProtocolList) except Exception as e: logger.exception( - "Failed to restart workflow subtree. protocolId=%s", + "Failed to restart workflow subtree. projectId=%s protocolId=%s", + projectId, protocolId, ) raise HTTPException( @@ -5280,7 +5551,10 @@ def restartProtocolAll(self, protocolId): detail=[str(e) for e in errorList], ) - return self._buildProtocolMutationResult("Protocol subtree restarted successfully") + return self._buildProtocolMutationResult( + "Protocol subtree restarted successfully", + postgresqlCleanup=cleanupInfo, + ) def continueProtocolAll(self, mapper, projectId, protocolId, currentUser): protocol = self.currentProject.getProtocol(int(protocolId)) @@ -5352,19 +5626,19 @@ def continueProtocolAll(self, mapper, projectId, protocolId, currentUser): return self._buildProtocolMutationResult("Protocol subtree continued successfully") - def resetProtocolFrom(self, protocolId): - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + def resetProtocolFrom(self, mapper, projectId: int, protocolId): + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) try: workflowProtocolList, _activeProtocolList = self.currentProject._getSubworkflow(protocol) except Exception as e: logger.exception( - "Failed to resolve subworkflow for reset-from. protocolId=%s", + "Failed to resolve subworkflow for reset-from. projectId=%s protocolId=%s", + projectId, protocolId, ) raise HTTPException( @@ -5372,11 +5646,24 @@ def resetProtocolFrom(self, protocolId): detail=f"Failed to resolve protocol subworkflow: {e}", ) + cleanupInfo = self._deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql( + mapper=mapper, + projectId=projectId, + protocols=workflowProtocolList, + ) + logger.info( + "Deleted persisted protocol outputs before reset-from. projectId=%s protocolId=%s cleanup=%s", + projectId, + protocolId, + cleanupInfo, + ) + try: resetErrors = self.currentProject.resetWorkFlow(workflowProtocolList) or [] except Exception as e: logger.exception( - "Failed to reset workflow subtree. protocolId=%s", + "Failed to reset workflow subtree. projectId=%s protocolId=%s", + projectId, protocolId, ) raise HTTPException( @@ -5390,7 +5677,10 @@ def resetProtocolFrom(self, protocolId): detail=[str(e) for e in resetErrors], ) - return self._buildProtocolMutationResult("Protocol subtree reset successfully") + return self._buildProtocolMutationResult( + "Protocol subtree reset successfully", + postgresqlCleanup=cleanupInfo, + ) def stopProtocol(self, protocolIds): resolvedProtocols = [] From ba26524677a7e1964d52d198f89367d2940e0b78 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 12:48:16 +0200 Subject: [PATCH 102/278] Resolve protocol params through runtime helper --- app/backend/api/routers/project_router.py | 12 +- app/backend/api/routers/protocol_router.py | 6 +- app/backend/api/services/project_service.py | 119 +++++++++++++------- 3 files changed, 95 insertions(+), 42 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index decd718e..73880588 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -345,7 +345,11 @@ async def loadProtocol( if not project: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") - return service.getProtocolParams(projectId, protocolId) + return service.getProtocolParams( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) class ProtocolStepStatusUpdate(BaseModel): @@ -544,7 +548,11 @@ def suggestionProtocol( "errors": ["Project not found"], "workflow": []}, ) - return service.getNextProtocolSuggestions(protocolId) + return service.getNextProtocolSuggestions( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) @router.put("/{projectId}/protocols/{protocolId}/rename", response_model=Any, status_code=status.HTTP_200_OK) diff --git a/app/backend/api/routers/protocol_router.py b/app/backend/api/routers/protocol_router.py index f5ce57eb..68769ffc 100644 --- a/app/backend/api/routers/protocol_router.py +++ b/app/backend/api/routers/protocol_router.py @@ -48,7 +48,11 @@ async def loadProtocol( if not project: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") - return service.getProtocolParams(projectId, protocolId) + return service.getProtocolParams( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) @router.get("/{projectId}/protclass/{protClassName}", response_model=Any) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index e8ee1d7c..4418e418 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -4255,22 +4255,33 @@ def getNewProtocolParams(self, projectId, protocolClassName: str) -> dict: return copy.deepcopy(ctx) - def getProtocolParams(self, projectId: int, protocolId: int) -> dict: + def getProtocolParams( + self, + mapper, + projectId: int, + protocolId: int, + ) -> dict: """ Returns the parameters of an existing protocol given its ID. """ - protocol = self.currentProject.getProtocol(int(protocolId)) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + protocol.getPlugin() self.currentProject._fixProtParamsConfiguration(protocol) return self._buildProtocolContext(projectId, protocol) - def getNextProtocolSuggestions(self, protocolId): + def getNextProtocolSuggestions(self, mapper, projectId, protocolId): """ Returns the suggestions from the Scipion website for the next protocols to the protocol passed """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) protName = protocol.getClassName() try: url = Config.SCIPION_STATS_SUGGESTION % protName # protocol.getClassName() @@ -4353,7 +4364,13 @@ def castParamValue(self, param, rawValue): else: return rawValue - def applyParamsToProtocol(self, protocol, params): + def applyParamsToProtocol( + self, + mapper, + projectId: int, + protocol, + params, + ): """Apply pointer parameters to protocol.""" errorList = [] for key, value in params.items(): @@ -4369,7 +4386,12 @@ def applyParamsToProtocol(self, protocol, params): parentId, rawValue = v.split('.') if v else ("", "") if rawValue: try: - parentProtocol = self.currentProject.getProtocol(int(parentId)) + parentScipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=parentId, + ) + parentProtocol = self._getScipionProtocolByRuntimeId(parentScipionProtocolId) extended = rawValue newInputs.append(Pointer(parentProtocol, extended=extended)) logger.info(f"[INFO] Pointer param {key} set from parent {parentId} output {rawValue}") @@ -4386,9 +4408,14 @@ def applyParamsToProtocol(self, protocol, params): parentId, rawValue = value.split('.') if value else ("", "") if rawValue: try: - parentProtocol = self.currentProject.getProtocol(int(parentId)) - val = value - output = val.split('.')[-1] + parentScipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=parentId, + ) + parentProtocol = self._getScipionProtocolByRuntimeId(parentScipionProtocolId) + output = rawValue + val = f"{parentScipionProtocolId}.{output}" param.set(val) parentOutput = hasattr(parentProtocol, output) if parentOutput: @@ -4424,16 +4451,37 @@ def applyParamsToProtocol(self, protocol, params): param.set(None) return errorList - def setPointerParam(self, protocol, key, value, parentId): + def setPointerParam( + self, + mapper, + projectId: int, + protocol, + key, + value, + parentId, + ): """Resolve and set a pointer param from parent protocol outputs.""" param = protocol.getParam(key) if not isinstance(param, PointerParam): logger.warning(f"[WARN] Param {key} is not a PointerParam") return - parentProtocol = self.currentProject.getProtocol(int(parentId)) - param.set(value['editableValue']) + + parentScipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=parentId, + ) + parentProtocol = self._getScipionProtocolByRuntimeId(parentScipionProtocolId) + + editableValue = value.get("editableValue") if isinstance(value, dict) else value + + if editableValue and "." in str(editableValue): + _rawParentId, outputName = str(editableValue).split(".", 1) + editableValue = f"{parentScipionProtocolId}.{outputName}" + + param.set(editableValue) protocol.setAttributeValue(key, parentProtocol) - param.default.set(value['editableValue']) + param.default.set(editableValue) def saveProtocol(self, mapper, projectId, protocolId, protocolClassName, params, setToSave=True): errorList = [] @@ -4447,12 +4495,11 @@ def saveProtocol(self, mapper, projectId, protocolId, protocolClassName, params, ) protocol = self.currentProject.newProtocol(protClass) else: - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) protectedParams = ['_objComment', '_useQueue', '_prerequisites', 'gpuList', 'numberOfThreads'] for paramName in protectedParams: @@ -4493,7 +4540,12 @@ def saveProtocol(self, mapper, projectId, protocolId, protocolClassName, params, cleaned = re.sub(r'[^A-Za-z0-9\s+\-*/=<>!&|^%()\[\]{}_,.;:]', '', str(e)) errorList.append('**' + param.label.get() + '** ' + cleaned) - errorList += self.applyParamsToProtocol(protocol, params) + errorList += self.applyParamsToProtocol( + mapper=mapper, + projectId=projectId, + protocol=protocol, + params=params, + ) # Persist protocol in Scipion always. # The setToSave flag only controls whether we also sync the graph to PostgreSQL now. @@ -4561,22 +4613,11 @@ def updateProtocolStepStatusService( targetStatus = statusMap[normalizedStatus] - if self.currentProject is None: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="No current project loaded", - ) - - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - protocol = None - - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) try: steps = protocol.loadSteps() or [] From fd81785359f5dca333983c8b53391b7c70d54694 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 12:54:03 +0200 Subject: [PATCH 103/278] Resolve workflow mutations through runtime helper --- app/backend/api/routers/project_router.py | 2 +- app/backend/api/services/project_service.py | 77 +++++++++++---------- 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 73880588..6e34aa80 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -890,7 +890,7 @@ def stopProtocol( "workflow": []}, ) - service.stopProtocol(protocolIds) + service.stopProtocol(mapper, projectId, protocolIds) service.syncProjectGraphAfterMutation( mapper, projectId, diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 4418e418..eabb951d 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -4703,7 +4703,7 @@ def launchProtocol(self, mapper, projectId, protocolId, protocolClassName, param if executeMode == "stop": try: - self.stopProtocol([protocolId]) + self.stopProtocol(mapper, projectId, [protocolId]) self.syncProjectProtocolsAndDependencies( mapper, @@ -5452,12 +5452,11 @@ def duplicateProtocol(self, mapper, projectId, protocols): if protocolId is None: continue sourceIds.append(protocolId) - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) protocolList.append(protocol) @@ -5518,8 +5517,20 @@ def duplicateProtocol(self, mapper, projectId, protocols): def deleteProtocol(self, mapper, projectId, protocols: Any): try: protList = [] - for protocol in protocols: - protList.append(self.currentProject.getProtocol(int(protocol))) + + for protocolId in protocols or []: + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + protList.append(protocol) + + if not protList: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="No valid protocols to delete", + ) self.currentProject.deleteProtocol(*protList) mapper.deleteProtocol(projectId, protList) @@ -5537,6 +5548,9 @@ def deleteProtocol(self, mapper, projectId, protocols: Any): "protocolsCount": syncInfo.get("protocols"), "dependenciesCount": syncInfo.get("dependencies"), } + + except HTTPException: + raise except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -5598,12 +5612,11 @@ def restartProtocolAll(self, mapper, projectId: int, protocolId): ) def continueProtocolAll(self, mapper, projectId, protocolId, currentUser): - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) try: workflowProtocolList, activeProtocolList = self.currentProject._getSubworkflow(protocol) @@ -5626,19 +5639,11 @@ def continueProtocolAll(self, mapper, projectId, protocolId, currentUser): protocolToLaunch = item if not hasattr(protocolToLaunch, "runMode"): - try: - protocolToLaunch = self.currentProject.getProtocol(int(item)) - except Exception: - logger.exception( - "Failed to resolve protocol to continue. projectId=%s protocolId=%s item=%s", - projectId, - protocolId, - item, - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to resolve protocol to continue: {item}", - ) + protocolToLaunch = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=item, + ) try: protocolToLaunch.runMode.set(MODE_RESUME) @@ -5723,16 +5728,15 @@ def resetProtocolFrom(self, mapper, projectId: int, protocolId): postgresqlCleanup=cleanupInfo, ) - def stopProtocol(self, protocolIds): + def stopProtocol(self, mapper, projectId: int, protocolIds): resolvedProtocols = [] for protocolId in protocolIds or []: - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) resolvedProtocols.append(protocol) if not resolvedProtocols: @@ -5746,7 +5750,8 @@ def stopProtocol(self, protocolIds): self.currentProject.stopProtocol(protocol) except Exception as e: logger.exception( - "Failed to stop protocols. protocolIds=%s", + "Failed to stop protocols. projectId=%s protocolIds=%s", + projectId, protocolIds, ) raise HTTPException( From c71828556326efeda55b9f7f514bda3264d4ccfa Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 13:04:56 +0200 Subject: [PATCH 104/278] Resolve file browser protocol ids through runtime helper --- app/backend/api/routers/project_router.py | 35 +++++- app/backend/api/services/project_service.py | 130 +++++++++++++++++--- 2 files changed, 144 insertions(+), 21 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 6e34aa80..4a0f176d 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1195,7 +1195,11 @@ async def getProtocolPath( service: ProjectService = Depends(getProjectService), ): _ensureProjectForFsRequest(projectId, protocolId, currentUser, mapper, service) - return service.getProtocolPath(protocolId) + return service.getProtocolPath( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) @router.get("/{projectId}/protocols/{protocolId}/fs/list", response_model=Any) @@ -1215,7 +1219,12 @@ async def listProtocolDir( service: ProjectService = Depends(getProjectService), ): _ensureProjectForFsRequest(projectId, protocolId, currentUser, mapper, service) - return service.listProtocolDir(protocolId, path) + return service.listProtocolDir( + protocolId=protocolId, + path=path, + mapper=mapper, + projectId=projectId, + ) @router.get("/{projectId}/protocols/{protocolId}/fs/preview2", response_model=None) @@ -1228,7 +1237,12 @@ async def previewProtocolText( service: ProjectService = Depends(getProjectService), ): _ensureProjectForFsRequest(projectId, protocolId, currentUser, mapper, service) - return service.previewProtocolTextFile(protocolId, path) + return service.previewProtocolTextFile( + protocolId=protocolId, + path=path, + mapper=mapper, + projectId=projectId, + ) @router.get("/{projectId}/protocols/{protocolId}/fs/preview", response_model=None) @@ -1249,7 +1263,12 @@ def previewRemoteEntry( refresh=False, checkPid=False, ) - return service.previewRemoteEntry(protocolId, path) + return service.previewRemoteEntry( + protocolId=protocolId, + path=path, + mapper=mapper, + projectId=projectId, + ) @router.get("/{projectId}/protocols/{protocolId}/fs/download", response_model=None) @@ -1263,7 +1282,13 @@ async def previewProtocolImageFile( service: ProjectService = Depends(getProjectService), ): _ensureProjectForFsRequest(projectId, protocolId, currentUser, mapper, service) - return service.previewProtocolImageFile(protocolId, path, inline) + return service.previewProtocolImageFile( + protocolId=protocolId, + path=path, + inline=inline, + mapper=mapper, + projectId=projectId, + ) @router.post( diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index eabb951d..79c31a9d 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6288,8 +6288,17 @@ def _sanitizeExportFilename(self, rawFilename: str) -> str: return filename - def _resolveFsRootForWrite(self, protocolId: Union[int, str]) -> Path: - pathInfo = self.getProtocolPath(protocolId) + def _resolveFsRootForWrite( + self, + protocolId: Union[int, str], + mapper=None, + projectId: Optional[int] = None, + ) -> Path: + pathInfo = self.getProtocolPath( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) rootAbs = str((pathInfo or {}).get("rootAbs") or "").strip() if not rootAbs: raise HTTPException( @@ -6326,16 +6335,27 @@ def _guardFsPathWithinRootForWrite( return candidate - def getProtocolPath(self, protocolId): + def getProtocolPath( + self, + protocolId, + mapper=None, + projectId: Optional[int] = None, + ): if self._isGlobalFsBrowserMode(protocolId): root = self._getGlobalFsBrowserRoot() return { "rootAbs": str(root), "startPath": "", } + fakeProtocolId = 'fake-protocol-id-for-browser-paths-resolution' + if protocolId != fakeProtocolId: - protocol = self.currentProject.getProtocol(int(protocolId)) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) protocolAbsPath = os.path.abspath(protocol.getPath()) rootAbsPath = self._inferProjectRootAbs(protocolAbsPath) else: @@ -6382,14 +6402,26 @@ def _inferProjectRootAbs(self, protocolAbsPath: str) -> str: return os.path.abspath(projectPath) if projectPath else "" - def _protocolRoot(self, protocolId: Union[int, str]) -> FsPath: + def _protocolRoot( + self, + protocolId: Union[int, str], + mapper=None, + projectId: Optional[int] = None, + ) -> FsPath: """ Resolve the absolute root folder for a protocol, using your service. """ - root = self.getProtocolPath(str(protocolId)) - if not root: + pathInfo = self.getProtocolPath( + protocolId=str(protocolId), + mapper=mapper, + projectId=projectId, + ) + + rootAbs = str((pathInfo or {}).get("rootAbs") or "").strip() + if not rootAbs: raise HTTPException(status_code=404, detail="Protocol path not found") - return FsPath(root).resolve() + + return FsPath(rootAbs).resolve() @staticmethod def _guardJoin(root: FsPath, relPath: str) -> FsPath: @@ -6410,7 +6442,30 @@ def _guessMime(p: FsPath) -> str: mt, _ = mimetypes.guess_type(str(p)) return mt or "application/octet-stream" - def listProtocolDir(self, protocolId: str, path: str): + def _resolveRuntimeProtocolIdForFs( + self, + protocolId: Union[int, str], + mapper=None, + projectId: Optional[int] = None, + ) -> str: + if self._isGlobalFsBrowserMode(protocolId): + return str(protocolId) + + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + return str(scipionProtocolId) + + def listProtocolDir( + self, + protocolId: str, + path: str, + mapper=None, + projectId: Optional[int] = None, + ): """Return the directory file list.""" fileHandlers = FileHandlers(self.currentProject) @@ -6418,9 +6473,21 @@ def listProtocolDir(self, protocolId: str, path: str): root = self._getGlobalFsBrowserRoot() return fileHandlers.listRemoteDirectoryUnderRoot(root, path) - return fileHandlers.listProtocolDir(protocolId, path) + runtimeProtocolId = self._resolveRuntimeProtocolIdForFs( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) + + return fileHandlers.listProtocolDir(runtimeProtocolId, path) - def previewProtocolTextFile(self, protocolId: str, path: str): + def previewProtocolTextFile( + self, + protocolId: str, + path: str, + mapper=None, + projectId: Optional[int] = None, + ): """ Return a lightweight preview for a file inside a protocol workspace. """ @@ -6430,9 +6497,21 @@ def previewProtocolTextFile(self, protocolId: str, path: str): root = self._getGlobalFsBrowserRoot() return fileHandlers.previewTextFileUnderRoot(root, path) - return fileHandlers.previewProtocolTextFile(protocolId, path) + runtimeProtocolId = self._resolveRuntimeProtocolIdForFs( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) - def previewRemoteEntry(self, protocolId: str, path: str): + return fileHandlers.previewProtocolTextFile(runtimeProtocolId, path) + + def previewRemoteEntry( + self, + protocolId: str, + path: str, + mapper=None, + projectId: Optional[int] = None, + ): """ Return a preview. """ @@ -6446,13 +6525,26 @@ def previewRemoteEntry(self, protocolId: str, path: str): databaseInspector=self._inspectScipionSqliteDatabase, ) + runtimeProtocolId = self._resolveRuntimeProtocolIdForFs( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) + return fileHandlers.previewProtocolRemoteEntry( - protocolId, + runtimeProtocolId, path, databaseInspector=self._inspectScipionSqliteDatabase, ) - def previewProtocolImageFile(self, protocolId, path, inline: bool): + def previewProtocolImageFile( + self, + protocolId, + path, + inline: bool, + mapper=None, + projectId: Optional[int] = None, + ): """ inline == False: - attachment download (binary as-is) @@ -6468,7 +6560,13 @@ def previewProtocolImageFile(self, protocolId, path, inline: bool): root = self._getGlobalFsBrowserRoot() return fileHandlers.previewImageFileUnderRoot(root, path, inline) - return fileHandlers.previewProtocolImageFile(protocolId, path, inline) + runtimeProtocolId = self._resolveRuntimeProtocolIdForFs( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) + + return fileHandlers.previewProtocolImageFile(runtimeProtocolId, path, inline) def outputPreview(self, protocolId: int, outputName: str, requestHeaders: dict = None, colormap: str = None): protocol = self.currentProject.getProtocol(protocolId) From 8522ee740bf4f7989bc90db43118ab5a03f7f786 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 13:12:28 +0200 Subject: [PATCH 105/278] Resolve preview and export protocols through runtime helper --- app/backend/api/routers/project_router.py | 4 + app/backend/api/services/project_service.py | 129 ++++++++++++-------- 2 files changed, 84 insertions(+), 49 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 4a0f176d..f4645839 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1443,6 +1443,8 @@ async def writeRemoteFile( return service.writeRemoteFileService( protocolId=protocolId, payload=payload, + mapper=mapper, + projectId=projectId, ) except HTTPException: raise @@ -1482,6 +1484,8 @@ async def previewOutput( outputName=outputName, requestHeaders=dict(request.headers), colormap=cmapHeader or cmapQuery, + mapper=mapper, + projectId=projectId, ) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 79c31a9d..af919dbc 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6266,6 +6266,44 @@ def _normalizeProtocolIdsForExport( return out + def _resolveRuntimeProtocolsForExport( + self, + mapper, + projectId: int, + protocolIds: List[Union[int, str]], + ) -> List[Any]: + protocolList = [] + missing = [] + + for protocolId in protocolIds or []: + try: + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + except HTTPException as e: + if e.status_code == status.HTTP_404_NOT_FOUND: + missing.append(str(protocolId)) + continue + raise + + protocolList.append(protocol) + + if missing: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol(s) not found: {', '.join(missing)}", + ) + + if not protocolList: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="No valid protocols to export", + ) + + return protocolList + def _sanitizeExportFilename(self, rawFilename: str) -> str: filename = str(rawFilename or "").strip() filename = filename.replace("\\", "/").split("/")[-1].strip() @@ -6568,8 +6606,27 @@ def previewProtocolImageFile( return fileHandlers.previewProtocolImageFile(runtimeProtocolId, path, inline) - def outputPreview(self, protocolId: int, outputName: str, requestHeaders: dict = None, colormap: str = None): - protocol = self.currentProject.getProtocol(protocolId) + def outputPreview( + self, + protocolId: int, + outputName: str, + requestHeaders: dict = None, + colormap: str = None, + mapper=None, + projectId: Optional[int] = None, + ): + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + if not hasattr(protocol, outputName): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Output not found: {outputName}", + ) + output = getattr(protocol, outputName) outputPreview = OutputsPreview( @@ -6652,34 +6709,20 @@ def exportProtocolsService( ) try: - protocolIdInts = [int(pid) for pid in protocolIds] - except Exception: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail="protocolIds must be numeric", + protocolList = self._resolveRuntimeProtocolsForExport( + mapper=mapper, + projectId=projectId, + protocolIds=protocolIds, ) - try: - protocolList = [] - missing: List[str] = [] - - for protId in protocolIdInts: - protocol = self.currentProject.getProtocol(protId) - if protocol is None: - missing.append(str(protId)) - continue - protocolList.append(protocol) - - if missing: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol(s) not found: {', '.join(missing)}", - ) - rawExport = self.currentProject.getProtocolsJson(protocolList) content = self._buildWorkflowExportJsonContent(rawExport, protocolList) - rootPath = self._resolveFsRootForWrite("-1") + rootPath = self._resolveFsRootForWrite( + "-1", + mapper=mapper, + projectId=projectId, + ) targetDir = self._guardFsPathWithinRootForWrite(rootPath, directoryPath) if targetDir.exists() and not targetDir.is_dir(): @@ -6895,29 +6938,11 @@ def exportWorkflowProtocolsService( detail="Missing protocolIds", ) - try: - protocolIdInts = [int(pid) for pid in protocolIds] - except Exception: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail="protocolIds must be numeric", - ) - - protocolList = [] - missing: List[str] = [] - - for protocolId in protocolIdInts: - protocol = self.currentProject.getProtocol(protocolId) - if protocol is None: - missing.append(str(protocolId)) - continue - protocolList.append(protocol) - - if missing: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol(s) not found: {', '.join(missing)}", - ) + protocolList = self._resolveRuntimeProtocolsForExport( + mapper=mapper, + projectId=projectId, + protocolIds=protocolIds, + ) rawExport = self.currentProject.getProtocolsJson(protocolList) workflow = self._decodeExportJsonPayload(rawExport) @@ -7012,8 +7037,14 @@ def writeRemoteFileService( self, protocolId: Union[int, str], payload: Any, + mapper=None, + projectId: Optional[int] = None, ) -> Dict[str, Any]: - rootPath = self._resolveFsRootForWrite(protocolId) + rootPath = self._resolveFsRootForWrite( + protocolId=protocolId, + mapper=mapper, + projectId=projectId, + ) targetPath = self._guardFsPathWithinRootForWrite( rootPath, getattr(payload, "path", ""), From 34f4a35c569aa2311869c204a9348402201cfc25 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 17:02:39 +0200 Subject: [PATCH 106/278] Resolve viewer fallbacks through runtime helper --- app/backend/api/services/project_service.py | 109 +++++++++++++++----- 1 file changed, 84 insertions(+), 25 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index af919dbc..533b5286 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6649,8 +6649,10 @@ def outputPreview( if not outputPath: return outputPreview.renderOutputFallbackPreview() + runtimeProtocolId = getattr(protocol, "getObjId", lambda: protocolId)() + objMgr = self._createObjectManager() - return outputPreview.preview(protocolId, outputPath, objMgr) + return outputPreview.preview(runtimeProtocolId, outputPath, objMgr) def buildProtocolThumbnail( self, @@ -7176,29 +7178,34 @@ def resolveAnalyzeViewerDecision(self, projectId: int, protocolId: int, ctx: Dic # ====================================================================== # Analyze Results: CTF Tomography (CTFTomoSeries) # ====================================================================== - def _resolveOutputForCtftomoSeries(self, protocolId: int, outputName: str): + def _resolveOutputForCtftomoSeries( + self, + protocolId: int, + outputName: str, + mapper=None, + projectId: Optional[int] = None, + ): """ Resolve protocol + CTFTomoSeries-like output for CTF tomography operations. - The output can be: - * a single CTFTomoSeries (one tilt-series) - * a container of CTFTomoSeries objects (SetOfCTFTomoSeries) + protocolId can be either the PostgreSQL protocols.id or the Scipion protocolId. """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) if not hasattr(protocol, outputName): raise HTTPException( - status_code=404, + status_code=status.HTTP_404_NOT_FOUND, detail=f"Output '{outputName}' not found in protocol", ) output = getattr(protocol, outputName) if output is None: raise HTTPException( - status_code=404, + status_code=status.HTTP_404_NOT_FOUND, detail=f"Output '{outputName}' is None", ) @@ -7323,7 +7330,12 @@ def listOutputCtftomoSeriesService( if pgReader is not None: return pgReader.listCtftomoSeries() - protocol, output = self._resolveOutputForCtftomoSeries(protocolId, outputName) + protocol, output = self._resolveOutputForCtftomoSeries( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) seriesList: List[Dict[str, Any]] = [] @@ -7407,7 +7419,12 @@ def getCtftomoSeriesViewsService( detail="CTF tomo output is not available in PostgreSQL metadata", ) - protocol, output = self._resolveOutputForCtftomoSeries(protocolId, outputName) + protocol, output = self._resolveOutputForCtftomoSeries( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) targetKey = str(tiltSeriesId) setOfTiltSeries = output.getSetOfTiltSeries() @@ -7505,7 +7522,12 @@ def renderCtfTomoPsdImageService( psdPath, ) - protocol, output = self._resolveOutputForCtftomoSeries(protocolId, outputName) + protocol, output = self._resolveOutputForCtftomoSeries( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) if protocol is None: raise HTTPException( status_code=404, @@ -7617,11 +7639,18 @@ def _getPostgresqlCtftomoReaderIfAvailable( # Analyze Results: Volumes (Volume / VolumeMask / SetOfVolumes) # ====================================================================== - def _resolveOutputForVolumes(self, protocolId: int, outputName: str): - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + def _resolveOutputForVolumes( + self, + protocolId: int, + outputName: str, + mapper=None, + projectId: Optional[int] = None, + ): + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) # Try exact + common alternates (singular/plural/alias) candidates = [outputName] @@ -7729,7 +7758,12 @@ def listOutputVolumesService( getattr(pgReader, "lastSkipReason", None), ) - protocol, output = self._resolveOutputForVolumes(protocolId, outputName) + protocol, output = self._resolveOutputForVolumes( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) op = OutputsPreview(self.currentProject, protocol, output) return op.listOutputVolumes() @@ -7762,7 +7796,12 @@ def getVolumeInfoService( getattr(pgReader, "lastSkipReason", None), ) - protocol, output = self._resolveOutputForVolumes(protocolId, outputName) + protocol, output = self._resolveOutputForVolumes( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) op = OutputsPreview(self.currentProject, protocol, output) return op.getVolumeInfo(volumeId) @@ -7799,7 +7838,12 @@ def getVolumeHistogramService( getattr(pgReader, "lastSkipReason", None), ) - protocol, output = self._resolveOutputForVolumes(protocolId, outputName) + protocol, output = self._resolveOutputForVolumes( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) op = OutputsPreview(self.currentProject, protocol, output) if isinstance(output, SetOfVolumes): @@ -7881,7 +7925,12 @@ def renderVolumeSliceService( getattr(pgReader, "lastSkipReason", None), ) - protocol, output = self._resolveOutputForVolumes(protocolId, outputName) + protocol, output = self._resolveOutputForVolumes( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) op = OutputsPreview(self.currentProject, protocol, output) return op.renderVolumeSlice( volumeId=volumeId, @@ -7943,7 +7992,12 @@ def getVolumeData3dService( getattr(pgReader, "lastSkipReason", None), ) - protocol, output = self._resolveOutputForVolumes(protocolId, outputName) + protocol, output = self._resolveOutputForVolumes( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) volumePath = self._getVolumePathFromOutput(output, volumeId) try: @@ -8207,7 +8261,12 @@ def getVolumeSurfaceMesh( getattr(pgReader, "lastSkipReason", None), ) - _protocol, output = self._resolveOutputForVolumes(protocolId, outputName) + protocol, output = self._resolveOutputForVolumes( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) volumePath = self._getVolumePathFromOutput(output, volumeId) volume, _props = readVolumeArray3d(volumePath) From 252985ece7f783020d3782ce3b27fe8e35b077f5 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 17:23:24 +0200 Subject: [PATCH 107/278] Resolve coords and metadata fallbacks through runtime helper --- app/backend/api/routers/project_router.py | 1 + app/backend/api/services/project_service.py | 130 ++++++++++++-------- 2 files changed, 79 insertions(+), 52 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index f4645839..c11bb092 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2546,6 +2546,7 @@ def getFscRows( projectId=projectId, protocolId=protocolId, outputName=outputName, + mapper=mapper, ) resp = JSONResponse(payload or {"curves": [], "threshold": 0.143}) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 533b5286..1c8e2afe 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -9290,6 +9290,28 @@ def _getPostgresqlCoords3dReaderIfAvailable( return None + def _resolveOutputForCoordinates3d( + self, + protocolId: int, + outputName: str, + mapper=None, + projectId: Optional[int] = None, + ): + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + output = getattr(protocol, outputName, None) + if output is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Output '{outputName}' not found in protocol", + ) + + return protocol, output + def listCoordinates3dTomogramsService( self, projectId: int, @@ -9333,17 +9355,12 @@ def listCoordinates3dTomogramsService( detail="TiltSeries frame not found in PostgreSQL metadata", ) - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") - - setOfCoordinates3D = getattr(protocol, outputName, None) - if setOfCoordinates3D is None: - raise HTTPException( - status_code=404, - detail=f"Output '{outputName}' not found in protocol", - ) + protocol, setOfCoordinates3D = self._resolveOutputForCoordinates3d( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) self.tomoList = {} tomogramList: List[Dict[str, Any]] = [] tomosIter = None @@ -9475,17 +9492,12 @@ def getCoordinates3dPointsService( getattr(pgReader, "lastSkipReason", None), ) - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") - - setOfCoordinates3D = getattr(protocol, outputName, None) - if setOfCoordinates3D is None: - raise HTTPException( - status_code=404, - detail=f"Output '{outputName}' not found in protocol", - ) + protocol, setOfCoordinates3D = self._resolveOutputForCoordinates3d( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) try: boxSize = float(setOfCoordinates3D.getBoxSize()) @@ -9758,17 +9770,12 @@ def renderCoords3dTomogramSliceService( getattr(pgReader, "lastSkipReason", None), ) - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") - - setOfCoordinates3D = getattr(protocol, outputName, None) - if setOfCoordinates3D is None: - raise HTTPException( - status_code=404, - detail=f"Output '{outputName}' not found in protocol", - ) + protocol, setOfCoordinates3D = self._resolveOutputForCoordinates3d( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) try: if self.tomoList: @@ -10208,14 +10215,14 @@ def createCoords3dOutputFromPointsService( # --------------------------------- # 1. Obtaining protocol and origin # --------------------------------- - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(404, "Protocol not found") + protocol, srcSet = self._resolveOutputForCoordinates3d( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) - srcSet = getattr(protocol, outputName, None) - if srcSet is None: - raise HTTPException(404, f"Output '{outputName}' not found in protocol") + runtimeProtocolId = getattr(protocol, "getObjId", lambda: protocolId)() # ------------------------------- # 2. Ensure tomograms @@ -10223,8 +10230,9 @@ def createCoords3dOutputFromPointsService( if not self.tomoList: self.listCoordinates3dTomogramsService( projectId=projectId, - protocolId=protocolId, + protocolId=runtimeProtocolId, outputName=outputName, + mapper=None, ) # ----------------------------------- @@ -10297,7 +10305,11 @@ def createCoords3dOutputFromPointsService( setMapper = ScipionSetPostgresqlMapper(mapper.db) setMapper.storeSet( projectId=projectId, - protocolDbId=protocolId, + protocolDbId=self._resolvePostgresqlProtocolDbId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) or protocolId, outputName=outName, scipionSet=dstSet, ) @@ -10332,6 +10344,7 @@ def getFscRowsService( projectId: int, protocolId: int, outputName: str, + mapper=None, ) -> Dict[str, Any]: """ Return FSC curves for a SetOfFSCs-like output. @@ -10350,10 +10363,11 @@ def getFscRowsService( ], } """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) output = getattr(protocol, outputName, None) if output is None: @@ -10479,7 +10493,13 @@ def getXY(fsc): # Internal helpers for metadata tables (STAR / SQLITE / etc.) # ====================================================================== - def _resolveOutputForMetadata(self, protocolId: int, outputName: str): + def _resolveOutputForMetadata( + self, + protocolId: int, + outputName: str, + mapper=None, + projectId: Optional[int] = None, + ): """ Resolve protocol and output object for metadata operations. @@ -10488,10 +10508,11 @@ def _resolveOutputForMetadata(self, protocolId: int, outputName: str): project-relative. - Raises 404 if the final path does not exist on disk. """ - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) if not hasattr(protocol, outputName): raise HTTPException( @@ -10619,7 +10640,12 @@ def _getMetadataObjectManagerForOutput( objMgr.getTables() return objMgr - _, _, metaPath = self._resolveOutputForMetadata(protocolId, outputName) + _, _, metaPath = self._resolveOutputForMetadata( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) return self._getMetadataObjectManager(metaPath) def _openMetadataTable( From 0cdbba78896d0a20c20d8ad6424882d5bebc8583 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 17:31:44 +0200 Subject: [PATCH 108/278] Resolve external viewers through runtime helper --- app/backend/api/routers/project_router.py | 30 ++-------- app/backend/api/services/project_service.py | 66 ++++++++++----------- 2 files changed, 36 insertions(+), 60 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index c11bb092..bcfec7d1 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2999,32 +2999,6 @@ def getMetadataTableWindow( # ====================================================================== # ANALYZE RESULTS: EXTERNAL VIEWERS # ====================================================================== - -@router.get( - "/{projectId}/protocols/{protocolId}/outputs/{outputName}/external-viewers", - response_model=Any, - status_code=status.HTTP_200_OK, -) -def listExternalViewers( - projectId: int, - protocolId: int, - outputName: str, - objectId: Optional[str] = Query(None), - objectKind: Optional[str] = Query(None), - currentUser=Depends(getCurrentUser), - mapper: PostgresqlFlatMapper = Depends(getMapper), - service: ProjectService = Depends(getProjectService), -): - project = service.getProjectById( - mapper, - projectId, - currentUser, - refresh=False, - checkPid=False, - ) - if not project: - raise - @router.get( "/{projectId}/protocols/{protocolId}/outputs/{outputName}/external-viewers", response_model=Any, @@ -3056,6 +3030,8 @@ def listExternalViewers( outputName=outputName, objectId=objectId, objectKind=objectKind, + mapper=mapper, + projectId=projectId, ) return {"viewers": viewers or []} @@ -3105,6 +3081,8 @@ def launchExternalViewer( objectId=payload.objectId, objectKind=payload.objectKind, params=payload.params or {}, + mapper=mapper, + projectId=projectId, ) except HTTPException: diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 1c8e2afe..73232116 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -12000,29 +12000,17 @@ def _unwrapScipionObject(self, obj: Any) -> Any: return obj def _getProtocolOutputObject( - self, - protocolId: int, - outputName: str, + self, + protocolId: int, + outputName: str, + mapper=None, + projectId: Optional[int] = None, ) -> Tuple[Any, Any]: - if self.currentProject is None: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="No current project loaded", - ) - - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception as e: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}. {e}", - ) - - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol not found: {protocolId}", - ) + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) outputObj = None @@ -12198,15 +12186,20 @@ def _resolveExternalViewerSetItemByPublicId( return None def listExternalViewers( - self, - protocolId: int, - outputName: str, - objectId: Optional[Union[str, int]] = None, - objectKind: Optional[str] = None, + self, + protocolId: int, + outputName: str, + objectId: Optional[Union[str, int]] = None, + objectKind: Optional[str] = None, + mapper=None, + projectId: Optional[int] = None, ) -> List[Dict[str, Any]]: + protocol, outputObj = self._getProtocolOutputObject( protocolId=protocolId, outputName=outputName, + mapper=mapper, + projectId=projectId, ) targetObj = self._resolveExternalViewerTargetObject( @@ -12339,17 +12332,22 @@ def _runExternalViewer(self, viewerClass: Any, protocol: Any, targetObj: Any): self._showExternalView(view) def launchExternalViewer( - self, - protocolId: int, - outputName: str, - viewerId: str, - objectId: Optional[Union[str, int]] = None, - objectKind: Optional[str] = None, - params: Optional[Dict[str, Any]] = None, + self, + protocolId: int, + outputName: str, + viewerId: str, + objectId: Optional[Union[str, int]] = None, + objectKind: Optional[str] = None, + params: Optional[Dict[str, Any]] = None, + mapper=None, + projectId: Optional[int] = None, ) -> Dict[str, Any]: + protocol, outputObj = self._getProtocolOutputObject( protocolId=protocolId, outputName=outputName, + mapper=mapper, + projectId=projectId, ) targetObj = self._resolveExternalViewerTargetObject( From 6418e4887ed2dbf425b6fae6ec75ff27710e5f4a Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 17:46:00 +0200 Subject: [PATCH 109/278] Resolve auxiliary services through runtime helper --- app/backend/api/routers/project_router.py | 4 ++ app/backend/api/services/coords2d_service.py | 25 +++-------- app/backend/api/services/project_service.py | 10 ++++- .../api/services/protocol_wizard_service.py | 45 ++++++++++++------- 4 files changed, 48 insertions(+), 36 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index bcfec7d1..3421258e 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -3487,6 +3487,8 @@ def buildThumbnail(): protocolId=protocolId, force=False, size=size, + mapper=mapper, + projectId=projectId, ) result = _runThumbnailProjectJob( @@ -3547,6 +3549,8 @@ def buildThumbnail(): protocolId=protocolId, force=True, size=size, + mapper=mapper, + projectId=projectId, ) result = _runThumbnailProjectJob( diff --git a/app/backend/api/services/coords2d_service.py b/app/backend/api/services/coords2d_service.py index be97aea6..720a2350 100644 --- a/app/backend/api/services/coords2d_service.py +++ b/app/backend/api/services/coords2d_service.py @@ -66,26 +66,11 @@ def _loadCoordinatesOutput( detail="Project not found", ) - currentProject = self.projectService.currentProject - if currentProject is None: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Project could not be loaded", - ) - - try: - protocol = currentProject.getProtocol(int(protocolId)) - except Exception as e: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol '{protocolId}' not found: {e}", - ) - - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Protocol '{protocolId}' not found", - ) + protocol = self.projectService._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) if not hasattr(protocol, outputName): raise HTTPException( diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 73232116..85f4c7ce 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6660,10 +6660,18 @@ def buildProtocolThumbnail( force: bool = False, size: int = 320, outputName: Optional[str] = None, + mapper=None, + projectId: Optional[int] = None, ) -> Dict[str, Any]: + runtimeProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + service = ThumbnailService(self.currentProject) return service.buildProtocolThumbnail( - protocolId=protocolId, + protocolId=runtimeProtocolId, force=force, size=size, outputName=outputName, diff --git a/app/backend/api/services/protocol_wizard_service.py b/app/backend/api/services/protocol_wizard_service.py index 45c129e2..946a5006 100644 --- a/app/backend/api/services/protocol_wizard_service.py +++ b/app/backend/api/services/protocol_wizard_service.py @@ -335,9 +335,11 @@ def _classifyWizardKind( return "unknown" def _applyFormValuesToProtocolInstance( - self, - protocol, - params: Dict[str, Any], + self, + protocol, + params: Dict[str, Any], + mapper=None, + projectId: Optional[int] = None, ) -> List[str]: if self.projectService is None: raise RuntimeError("projectService is required to apply wizard form values") @@ -390,22 +392,28 @@ def _applyFormValuesToProtocolInstance( cleaned = re.sub(r"[^A-Za-z0-9\s+\-*/=<>\!&|^%()\[\]{}_,.;:]", "", str(e)) errorList.append("**" + param.label.get() + "** " + cleaned) - errorList += self.projectService.applyParamsToProtocol(protocol, params) + errorList += self.projectService.applyParamsToProtocol( + mapper=mapper, + projectId=projectId, + protocol=protocol, + params=params, + ) return errorList def _buildWizardReadyProtocol( - self, - protocolId: Optional[int], - protocolClassName: str, - formValues: Dict[str, Any], + self, + protocolId: Optional[int], + protocolClassName: str, + formValues: Dict[str, Any], + mapper=None, + projectId: Optional[int] = None, ): if protocolId: - protocol = self.currentProject.getProtocol(int(protocolId)) - if protocol is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Protocol not found", - ) + protocol = self.projectService._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) else: protClass = self.currentProject.getDomain().getProtocols().get(str(protocolClassName)) if protClass is None: @@ -418,7 +426,12 @@ def _buildWizardReadyProtocol( self.currentProject._fixProtParamsConfiguration(protocol) sanitizedFormValues = self._sanitizeWizardFormValues(formValues or {}) - errors = self._applyFormValuesToProtocolInstance(protocol, sanitizedFormValues) + errors = self._applyFormValuesToProtocolInstance( + protocol, + sanitizedFormValues, + mapper=mapper, + projectId=projectId, + ) if errors: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, @@ -495,6 +508,8 @@ def executeProtocolWizard( protocolId=getattr(payload, "protocolId", None), protocolClassName=str(getattr(payload, "protocolClassName", "")).strip(), formValues=getattr(payload, "formValues", {}) or {}, + mapper=mapper, + projectId=projectId, ) paramName = str(getattr(payload, "paramName", "")).strip() From 1e889216d163bfbe800a2ee47a41e6e9bd465198 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 18:31:15 +0200 Subject: [PATCH 110/278] Update tests for runtime protocol resolver --- tests/conftest.py | 45 ++++++++++++++----- tests/integration/api/test_projects_router.py | 1 + .../api/test_projects_router_protocol_ops.py | 44 +++++++++++++++--- .../api/routers/test_protocols_router.py | 8 +++- .../services/test_project_service_coords3d.py | 18 ++++---- .../services/test_project_service_ctftomo.py | 11 +++-- .../test_project_service_io_thumbnails.py | 12 ++++- .../test_project_service_protocol_steps.py | 4 +- .../test_project_service_protocols.py | 44 +++++++++++++++--- .../test_project_service_tiltseries.py | 2 +- .../services/test_project_service_volumes.py | 2 +- 11 files changed, 142 insertions(+), 49 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2db8445a..f1979c83 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -291,16 +291,17 @@ def getProtocols(self, mapper, projectId, currentUser): def listProjectLogChannelsService(self, projectId, protocolId): return self.logChannelsResult - def listProtocolLogChannelsService(self, projectId, protocolId): + def listProtocolLogChannelsService(self, projectId, protocolId, mapper=None): return self.logChannelsResult - def pollProtocolLogsService(self, projectId, protocolId, offsets, maxBytes, maxLines): + def pollProtocolLogsService(self, projectId, protocolId, offsets, maxBytes, maxLines, mapper=None): self.lastPollLogsCall = { "projectId": projectId, "protocolId": protocolId, "offsets": dict(offsets), "maxBytes": maxBytes, "maxLines": maxLines, + "mapper": mapper, } return self.pollLogsResult @@ -533,10 +534,11 @@ def applyWorkflowToProject(self, mapper, projectId, workflowId, currentUser): raise self.applyWorkflowError return self.applyWorkflowResult - def getProtocolParams(self, projectId, protocolId): + def getProtocolParams(self, projectId, protocolId, mapper=None): self.lastGetProtocolParamsCall = { "projectId": projectId, "protocolId": protocolId, + "mapper": mapper, } return self.protocolParamsResult @@ -571,14 +573,21 @@ def saveProtocol(self, mapper, projectId, protocolId, protocolClassName, params) raise self.saveProtocolError return self.saveProtocolResult - def getNextProtocolSuggestions(self, protocolId): - self.lastGetNextProtocolSuggestionsCall = {"protocolId": protocolId} + def getNextProtocolSuggestions(self, protocolId, mapper=None, projectId=None): + self.lastGetNextProtocolSuggestionsCall = { + "protocolId": protocolId, + "mapper": mapper, + "projectId": projectId, + } return self.nextProtocolSuggestionsResult - def renameProtocol(self, protocolId, newName): + def renameProtocol(self, mapper, projectId, protocolId, newName, newComment=""): self.lastRenameProtocolCall = { + "mapper": mapper, + "projectId": projectId, "protocolId": protocolId, "newName": newName, + "newComment": newComment, } if self.renameProtocolError is not None: raise self.renameProtocolError @@ -602,8 +611,12 @@ def deleteProtocol(self, mapper, projectId, protocolIds): if self.deleteProtocolError is not None: raise self.deleteProtocolError - def restartProtocolAll(self, protocolId): - self.lastRestartProtocolAllCall = {"protocolId": protocolId} + def restartProtocolAll(self, mapper, projectId, protocolId): + self.lastRestartProtocolAllCall = { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + } if self.restartProtocolAllError is not None: raise self.restartProtocolAllError return self.restartProtocolAllResult @@ -618,13 +631,21 @@ def continueProtocolAll(self, mapper, projectId, protocolId, currentUser): if self.continueProtocolAllError is not None: raise self.continueProtocolAllError - def resetProtocolFrom(self, protocolId): - self.lastResetProtocolFromCall = {"protocolId": protocolId} + def resetProtocolFrom(self, mapper, projectId, protocolId): + self.lastResetProtocolFromCall = { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + } if self.resetProtocolFromError is not None: raise self.resetProtocolFromError - def stopProtocol(self, protocolIds): - self.lastStopProtocolCall = {"protocolIds": protocolIds} + def stopProtocol(self, mapper, projectId, protocolIds): + self.lastStopProtocolCall = { + "mapper": mapper, + "projectId": projectId, + "protocolIds": protocolIds, + } if self.stopProtocolError is not None: raise self.stopProtocolError diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 95ae7ad9..cf59ae06 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -148,6 +148,7 @@ def test_PollProtocolLogsNormalizesOffsetsAndIncludesDynamicChannels(projectClie }, "maxBytes": 123, "maxLines": 45, + "mapper": fakeProjectService.lastPollLogsCall["mapper"], } diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 0fc7bf84..946ad4c9 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -28,8 +28,10 @@ def patchRenameProtocolFake(fakeProjectService): # patchRenameProtocolFake - def renameProtocol(protocolId, newName, newComment=""): + def renameProtocol(mapper, projectId, protocolId, newName, newComment=""): fakeProjectService.lastRenameProtocolCall = { + "mapper": mapper, + "projectId": projectId, "protocolId": protocolId, "newName": newName, "newComment": newComment, @@ -62,6 +64,7 @@ def test_LoadProtocolReturnsParams(projectClient, fakeProjectService): assert fakeProjectService.lastGetProtocolParamsCall == { "projectId": 1, "protocolId": 10, + "mapper": fakeProjectService.lastGetProtocolParamsCall["mapper"], } @@ -220,7 +223,11 @@ def test_SuggestionProtocolReturnsSuggestions(projectClient, fakeProjectService) assert response.status_code == 200 assert response.json() == [{"id": "next-1", "name": "Next protocol"}] - assert fakeProjectService.lastGetNextProtocolSuggestionsCall == {"protocolId": 10} + assert fakeProjectService.lastGetNextProtocolSuggestionsCall == { + "protocolId": 10, + "mapper": fakeProjectService.lastGetNextProtocolSuggestionsCall["mapper"], + "projectId": 1, + } def test_RenameProtocolRejectsBlankName(projectClient): @@ -256,6 +263,8 @@ def test_RenameProtocolDelegatesToService(projectClient, fakeProjectService): } assert fakeProjectService.lastRenameProtocolCall == { + "mapper": fakeProjectService.lastRenameProtocolCall["mapper"], + "projectId": 1, "protocolId": 10, "newName": "Renamed protocol", "newComment": "Updated comment", @@ -374,7 +383,11 @@ def test_RestartProtocolAllReturnsSuccess(projectClient, fakeProjectService): "workflow": [], } - assert fakeProjectService.lastRestartProtocolAllCall == {"protocolId": 10} + assert fakeProjectService.lastRestartProtocolAllCall == { + "mapper": fakeProjectService.lastRestartProtocolAllCall["mapper"], + "projectId": 1, + "protocolId": 10, + } assert fakeProjectService.lastSyncProjectGraphAfterMutationCall == { "mapper": fakeProjectService.lastSyncProjectGraphAfterMutationCall["mapper"], "projectId": 1, @@ -424,7 +437,11 @@ def test_ResetProtocolFromDelegatesToService(projectClient, fakeProjectService): "workflow": [], } - assert fakeProjectService.lastResetProtocolFromCall == {"protocolId": 10} + assert fakeProjectService.lastResetProtocolFromCall == { + "mapper": fakeProjectService.lastResetProtocolFromCall["mapper"], + "projectId": 1, + "protocolId": 10, + } assert fakeProjectService.lastSyncProjectGraphAfterMutationCall == { "mapper": fakeProjectService.lastSyncProjectGraphAfterMutationCall["mapper"], "projectId": 1, @@ -454,6 +471,8 @@ def test_RenameProtocolReturnsErrorWhenGraphSyncFails(projectClient, fakeProject } assert fakeProjectService.lastRenameProtocolCall == { + "mapper": fakeProjectService.lastRenameProtocolCall["mapper"], + "projectId": 1, "protocolId": 10, "newName": "Renamed protocol", "newComment": "Updated comment", @@ -476,7 +495,11 @@ def test_RestartProtocolAllReturnsErrorWhenGraphSyncFails(projectClient, fakePro "workflow": [], } - assert fakeProjectService.lastRestartProtocolAllCall == {"protocolId": 10} + assert fakeProjectService.lastRestartProtocolAllCall == { + "mapper": fakeProjectService.lastRestartProtocolAllCall["mapper"], + "projectId": 1, + "protocolId": 10, + } def test_ContinueProtocolAllReturnsErrorWhenGraphSyncFails(projectClient, fakeProjectService): @@ -521,7 +544,11 @@ def test_ResetProtocolFromReturnsErrorWhenGraphSyncFails(projectClient, fakeProj "workflow": [], } - assert fakeProjectService.lastResetProtocolFromCall == {"protocolId": 10} + assert fakeProjectService.lastResetProtocolFromCall == { + "mapper": fakeProjectService.lastResetProtocolFromCall["mapper"], + "projectId": 1, + "protocolId": 10, + } def test_StopProtocolReturnsErrorWhenGraphSyncFails(projectClient, fakeProjectService): @@ -543,6 +570,8 @@ def test_StopProtocolReturnsErrorWhenGraphSyncFails(projectClient, fakeProjectSe } assert fakeProjectService.lastStopProtocolCall == { + "mapper": fakeProjectService.lastStopProtocolCall["mapper"], + "projectId": 1, "protocolIds": ["10", "11"], } @@ -575,8 +604,11 @@ def test_StopProtocolDelegatesToService(projectClient, fakeProjectService): } assert fakeProjectService.lastStopProtocolCall == { + "mapper": fakeProjectService.lastStopProtocolCall["mapper"], + "projectId": 1, "protocolIds": ["10", "11"], } + assert fakeProjectService.lastSyncProjectGraphAfterMutationCall == { "mapper": fakeProjectService.lastSyncProjectGraphAfterMutationCall["mapper"], "projectId": 1, diff --git a/tests/unit/backend/api/routers/test_protocols_router.py b/tests/unit/backend/api/routers/test_protocols_router.py index 31b83a4e..4477e127 100644 --- a/tests/unit/backend/api/routers/test_protocols_router.py +++ b/tests/unit/backend/api/routers/test_protocols_router.py @@ -69,10 +69,11 @@ def getProjectById(self, mapper, projectId, currentUser): } return self.projectByIdResult - def getProtocolParams(self, projectId, protocolId): + def getProtocolParams(self, projectId, protocolId, mapper=None): self.lastGetProtocolParamsCall = { "projectId": projectId, "protocolId": protocolId, + "mapper": mapper, } return self.protocolParamsResult @@ -101,13 +102,14 @@ def saveProtocol(self, protocolId, protocolClassName, params): if self.saveError is not None: raise self.saveError - def getProtocolLogs(self, projectId, protocolId, offset, errOffset, scheduleOffset): + def getProtocolLogs(self, projectId, protocolId, offset, errOffset, scheduleOffset, mapper=None): self.lastGetProtocolLogsCall = { "projectId": projectId, "protocolId": protocolId, "offset": offset, "errOffset": errOffset, "scheduleOffset": scheduleOffset, + "mapper": mapper, } return self.protocolLogsResult @@ -172,6 +174,7 @@ def test_LoadProtocolReturnsProtocolParams(protocolClient, fakeProtocolRouterSer assert fakeProtocolRouterService.lastGetProtocolParamsCall == { "projectId": 1, "protocolId": 10, + "mapper": fakeProtocolRouterService.lastGetProtocolParamsCall["mapper"], } @@ -296,4 +299,5 @@ def test_GetProtocolLogsReturnsPayload(protocolClient, fakeProtocolRouterService "offset": 5, "errOffset": 7, "scheduleOffset": 9, + "mapper": fakeProtocolRouterService.lastGetProtocolLogsCall["mapper"], } \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_coords3d.py b/tests/unit/backend/api/services/test_project_service_coords3d.py index c57627d0..b55f311f 100644 --- a/tests/unit/backend/api/services/test_project_service_coords3d.py +++ b/tests/unit/backend/api/services/test_project_service_coords3d.py @@ -482,16 +482,14 @@ def test_CreateCoords3dOutputFromPointsServiceCreatesNewOutput(projectServiceMod payload=payload, ) - assert result == { - "success": True, - "outputName": "outputCoords3d_edited", - "message": "Created new coords3d output 'outputCoords3d_edited'", - "data": { - "sourceOutputName": "outputCoords3d", - "replacedPoints": 2, - "copiedPoints": 0, - }, - } + assert result["success"] is True + assert result["outputName"] == "outputCoords3d_edited" + assert result["message"] == "Created new coords3d output 'outputCoords3d_edited'" + assert result["data"]["sourceOutputName"] == "outputCoords3d" + assert result["data"]["replacedPoints"] == 2 + assert result["data"]["copiedPoints"] == 0 + assert result["data"]["postgresqlStored"] is False + assert result["data"]["postgresqlError"] is None assert "outputCoords3d_edited" in protocol.definedOutputs createdSet = protocol.definedOutputs["outputCoords3d_edited"] diff --git a/tests/unit/backend/api/services/test_project_service_ctftomo.py b/tests/unit/backend/api/services/test_project_service_ctftomo.py index e37f624f..2f0b32e2 100644 --- a/tests/unit/backend/api/services/test_project_service_ctftomo.py +++ b/tests/unit/backend/api/services/test_project_service_ctftomo.py @@ -590,12 +590,11 @@ def test_CreateNewSetOfCtftomoSeriesServiceCreatesFilteredSeries(service, tmp_pa restack=False, ) - assert result == { - "status": 0, - "outputName": "CTFTomoSeries_0", - "createdSeries": 1, - "restack": False, - } + assert result["status"] == 0 + assert result["outputName"] == "CTFTomoSeries_0" + assert result["createdSeries"] == 1 + assert result["restack"] is False + assert result["postgresqlSync"] is None assert "CTFTomoSeries_0" in protocol._definedOutputs createdSet = protocol._definedOutputs["CTFTomoSeries_0"] assert createdSet.isEmpty() is False diff --git a/tests/unit/backend/api/services/test_project_service_io_thumbnails.py b/tests/unit/backend/api/services/test_project_service_io_thumbnails.py index 88d56cad..5eff0b32 100644 --- a/tests/unit/backend/api/services/test_project_service_io_thumbnails.py +++ b/tests/unit/backend/api/services/test_project_service_io_thumbnails.py @@ -341,7 +341,11 @@ def test_ExportProtocolsServiceWritesJsonFile(service, monkeypatch, tmp_path): exportPayload=[{"protocolId": 10}, {"protocolId": 11}], ) - monkeypatch.setattr(service, "_resolveFsRootForWrite", lambda protocolId: rootPath) + monkeypatch.setattr( + service, + "_resolveFsRootForWrite", + lambda protocolId, mapper=None, projectId=None: rootPath, + ) payload = FakePayload( protocolIds=[10, 11], @@ -402,7 +406,11 @@ def test_WriteRemoteFileServiceWritesContent(service, monkeypatch, tmp_path): rootPath = tmp_path / "browser-root" rootPath.mkdir(parents=True, exist_ok=True) - monkeypatch.setattr(service, "_resolveFsRootForWrite", lambda protocolId: rootPath) + monkeypatch.setattr( + service, + "_resolveFsRootForWrite", + lambda protocolId, mapper=None, projectId=None: rootPath, + ) payload = FakePayload( path="exports/result.json", diff --git a/tests/unit/backend/api/services/test_project_service_protocol_steps.py b/tests/unit/backend/api/services/test_project_service_protocol_steps.py index b6fcc2c5..63c16892 100644 --- a/tests/unit/backend/api/services/test_project_service_protocol_steps.py +++ b/tests/unit/backend/api/services/test_project_service_protocol_steps.py @@ -230,7 +230,7 @@ def test_UpdateProtocolStepStatusRaisesWhenCurrentProjectIsMissing(service, mapp ) assert exc.value.status_code == 500 - assert exc.value.detail == "No current project loaded" + assert exc.value.detail == "No current Scipion project loaded" def test_UpdateProtocolStepStatusRaisesWhenProtocolIsMissing(service, mapper): @@ -244,7 +244,7 @@ def test_UpdateProtocolStepStatusRaisesWhenProtocolIsMissing(service, mapper): ) assert exc.value.status_code == 404 - assert exc.value.detail == "Protocol not found: 99" + assert str(exc.value.detail).startswith("Protocol not found in Scipion runtime: 99") def test_UpdateProtocolStepStatusRaisesWhenStepIsMissing(service, mapper): diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index 638a5fe7..59b572f1 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -305,7 +305,11 @@ def test_SaveProtocolCreatesNewProtocolAndPersistsContext(projectServiceModule, monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) - monkeypatch.setattr(service, "applyParamsToProtocol", lambda protocol, params: []) + monkeypatch.setattr( + service, + "applyParamsToProtocol", + lambda mapper=None, projectId=None, protocol=None, params=None: [], + ) monkeypatch.setattr( service, "_buildProtocolContext", @@ -382,7 +386,11 @@ def test_SaveProtocolAggregatesValidationAndPointerErrors(projectServiceModule, service.currentProject.protocols[10] = protocol mapper.dbProtocolsByProtocolId[(10, 1)] = {"id": 500, "protocolId": 10} - monkeypatch.setattr(service, "applyParamsToProtocol", lambda protocolObj, params: ["pointer error"]) + monkeypatch.setattr( + service, + "applyParamsToProtocol", + lambda mapper=None, projectId=None, protocol=None, params=None: ["pointer error"], + ) def fakeSyncProjectProtocolsAndDependencies(mapperObj, projectId, refresh=False, checkPid=False): for protocolObj in service.currentProject.storedProtocols: @@ -428,7 +436,11 @@ def test_LaunchProtocolRejectsUnknownExecuteMode(service, mapper): def test_LaunchProtocolStopDelegatesToStopProtocol(service, mapper, monkeypatch): calls = [] - monkeypatch.setattr(service, "stopProtocol", lambda protocolIds: calls.append(protocolIds)) + monkeypatch.setattr( + service, + "stopProtocol", + lambda mapper, projectId, protocolIds: calls.append(protocolIds), + ) service.launchProtocol( mapper=mapper, @@ -519,7 +531,13 @@ def test_RenameProtocolStoresAnnotation(service): protocol = FakeProtocol(objId=10) service.currentProject.protocols[10] = protocol - result = service.renameProtocol(10, "Renamed protocol", "Updated comment") + result = service.renameProtocol( + mapper=None, + projectId=None, + protocolId=10, + newName="Renamed protocol", + newComment="Updated comment", + ) assertSuccessEnvelope(result) assert protocol._label is None @@ -624,7 +642,11 @@ def test_RestartProtocolAllReturnsCollectedErrors(service): service.currentProject.restartWorkflowInjectedErrors = ["cannot restart", "blocked"] with pytest.raises(HTTPException) as exc: - service.restartProtocolAll(10) + service.restartProtocolAll( + mapper=None, + projectId=None, + protocolId=10, + ) assert exc.value.status_code == 422 assert exc.value.detail == ["cannot restart", "blocked"] @@ -656,7 +678,11 @@ def test_ResetProtocolFromReturnsSuccessWhenWorkflowResets(service): service.currentProject.protocols[10] = protocol service.currentProject.resetWorkflowResult = [] - result = service.resetProtocolFrom(10) + result = service.resetProtocolFrom( + mapper=None, + projectId=None, + protocolId=10, + ) assertSuccessEnvelope(result) @@ -667,7 +693,11 @@ def test_StopProtocolStopsEachProtocol(service): service.currentProject.protocols[10] = protocolA service.currentProject.protocols[11] = protocolB - result = service.stopProtocol(["10", "11"]) + result = service.stopProtocol( + mapper=None, + projectId=None, + protocolIds=["10", "11"], + ) assertSuccessEnvelope(result) assert service.currentProject.stoppedProtocols == [protocolA, protocolB] \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_tiltseries.py b/tests/unit/backend/api/services/test_project_service_tiltseries.py index ef4cd9fa..c5b725a0 100644 --- a/tests/unit/backend/api/services/test_project_service_tiltseries.py +++ b/tests/unit/backend/api/services/test_project_service_tiltseries.py @@ -507,4 +507,4 @@ def getProtocol(self, protocolId): service._resolveOutputForTiltSeries(10, "outputTiltSeries") assert exc.value.status_code == 404 - assert exc.value.detail == "Protocol not found" \ No newline at end of file + assert str(exc.value.detail).startswith("Protocol not found in Scipion runtime:") \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_volumes.py b/tests/unit/backend/api/services/test_project_service_volumes.py index d5d08d98..a52275ce 100644 --- a/tests/unit/backend/api/services/test_project_service_volumes.py +++ b/tests/unit/backend/api/services/test_project_service_volumes.py @@ -193,7 +193,7 @@ def test_ResolveOutputForVolumesReturns404WhenProtocolMissing(service): service._resolveOutputForVolumes(10, "outputVolumes") assert exc.value.status_code == 404 - assert exc.value.detail == "Protocol not found" + assert str(exc.value.detail).startswith("Protocol not found in Scipion runtime: 10") def test_ListOutputVolumesServiceDelegatesToOutputsPreview(projectServiceModule, service, monkeypatch): From f5a1bbecc680019f609d9a02559265d9969ec072 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 21:54:41 +0200 Subject: [PATCH 111/278] Document runtime-only protocol utilities --- app/backend/utils/file_handlers.py | 3 +++ app/backend/utils/thumbnail_service.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/app/backend/utils/file_handlers.py b/app/backend/utils/file_handlers.py index f1c02af7..02ad7eec 100644 --- a/app/backend/utils/file_handlers.py +++ b/app/backend/utils/file_handlers.py @@ -68,6 +68,9 @@ def getProtocolPath(self, protocolId): """ Return the protocol browser paths. + protocolId must be a Scipion runtime protocol id. PostgreSQL protocol + ids must be resolved by ProjectService before calling FileHandlers. + Returns: - rootAbs: absolute root boundary (project folder inferred) - startPath: relative to rootAbs ("" means rootAbs) diff --git a/app/backend/utils/thumbnail_service.py b/app/backend/utils/thumbnail_service.py index 1b803b0b..fa3eca82 100644 --- a/app/backend/utils/thumbnail_service.py +++ b/app/backend/utils/thumbnail_service.py @@ -176,6 +176,12 @@ def buildProtocolThumbnail( size: int = 360, outputName: Optional[str] = None, ) -> Dict[str, Any]: + """ + Build a thumbnail for a Scipion runtime protocol id. + + PostgreSQL protocol ids must be resolved by ProjectService before + calling ThumbnailService. + """ try: protocol = self.currentProject.getProtocol(int(protocolId)) except Exception: From 2b512ec688113bad8f473a9e423142a4dcc72958 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 22:25:26 +0200 Subject: [PATCH 112/278] Sync generated outputs with resolved protocol ids --- app/backend/api/services/project_service.py | 57 ++++++++++++++++++--- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 85f4c7ce..83ed630b 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -8532,7 +8532,12 @@ def createNewSetOfCtftomoSeriesService( ... } """ - protocol, inputSet = self._resolveOutputForCtftomoSeries(protocolId, outputName) + protocol, inputSet = self._resolveOutputForCtftomoSeries( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) # Normalize exclusions keys to strings normalizedExclusions: Dict[str, Dict[str, Any]] = {} @@ -8590,9 +8595,15 @@ def createNewSetOfCtftomoSeriesService( from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper setMapper = ScipionSetPostgresqlMapper(mapper.db) + protocolDbId = self._resolvePostgresqlProtocolDbId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) or protocolId + postgresqlSync = setMapper.storeSet( projectId=projectId, - protocolDbId=protocolId, + protocolDbId=protocolDbId, outputName=newOutputName, scipionSet=outputSet, ) @@ -9228,9 +9239,15 @@ def createNewSetOfTiltSeriesService( from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper setMapper = ScipionSetPostgresqlMapper(mapper.db) + protocolDbId = self._resolvePostgresqlProtocolDbId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) or protocolId + postgresqlSync = setMapper.storeSet( projectId=projectId, - protocolDbId=protocolId, + protocolDbId=protocolDbId, outputName=newOutputName, scipionSet=outputSet, ) @@ -11073,10 +11090,11 @@ def runMetadataTableActionService( ) -> Any: selectionIds = self._normalizeMetadataSelectionIds(ids) - try: - protocol = self.currentProject.getProtocol(int(protocolId)) - except Exception: - raise HTTPException(status_code=404, detail="Protocol not found") + protocol = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) if not hasattr(protocol, outputName): raise HTTPException( @@ -11125,9 +11143,34 @@ def runMetadataTableActionService( self.currentProject.launchProtocol(batchProt) + postgresqlSync = None + try: + postgresqlSync = self.syncProjectProtocolsAndDependencies( + mapper, + projectId, + refresh=True, + checkPid=True, + ) + except Exception as syncError: + logger.exception( + "Subset protocol was launched but PostgreSQL sync failed. projectId=%s protocolId=%s outputName=%s tableName=%s", + projectId, + protocolId, + outputName, + tableName, + ) + return { + "success": True, + "message": "Subset protocol was launched successfully, but PostgreSQL sync failed", + "postgresqlSync": None, + "postgresqlError": str(syncError), + } + return { "success": True, "message": "Subset protocol was launched successfully", + "postgresqlSync": postgresqlSync, + "postgresqlError": None, } except Exception as e: From df378b3fea9f15b6a09dd5ee277d5740eb9f6af7 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 22:52:57 +0200 Subject: [PATCH 113/278] Add coverage for generated output synchronization --- .../services/test_project_service_ctftomo.py | 108 ++++++++++++++ .../services/test_project_service_metadata.py | 49 ++++++- .../test_project_service_tiltseries.py | 136 ++++++++++++++++++ 3 files changed, 292 insertions(+), 1 deletion(-) diff --git a/tests/unit/backend/api/services/test_project_service_ctftomo.py b/tests/unit/backend/api/services/test_project_service_ctftomo.py index 2f0b32e2..4a73dd9f 100644 --- a/tests/unit/backend/api/services/test_project_service_ctftomo.py +++ b/tests/unit/backend/api/services/test_project_service_ctftomo.py @@ -540,6 +540,114 @@ def test_CreateNewSetOfCtftomoSeriesServiceReturnsEmptyWhenEverythingExcluded(se "message": "No output was generated because it cannot be empty", } +def test_CreateNewSetOfCtftomoSeriesServiceStoresSetWithResolvedProtocolDbId( + service, + monkeypatch, +): + storedCalls = [] + + class FakeDb: + # fakeDb + def fetchOne(self, *args, **kwargs): + return None + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakeScipionSetPostgresqlMapper: + # fakeScipionSetPostgresqlMapper + def __init__(self, db): + self.db = db + + def storeSet(self, projectId, protocolDbId, outputName, scipionSet): + storedCalls.append( + { + "projectId": projectId, + "protocolDbId": protocolDbId, + "outputName": outputName, + "scipionSet": scipionSet, + } + ) + return { + "stored": True, + "protocolDbId": protocolDbId, + "outputName": outputName, + } + + scipionSetMapperModule = importlib.import_module( + "app.backend.mapper.scipion_set_mapper" + ) + monkeypatch.setattr( + scipionSetMapperModule, + "ScipionSetPostgresqlMapper", + FakeScipionSetPostgresqlMapper, + ) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 654, + ) + + associatedTs = FakeAssociatedTiltSeries() + ctf1 = FakeCtfMeasurement( + objId=100, + index=1, + defocusU=12000.0, + defocusV=11000.0, + defocusAngle=45.0, + resolution=3.2, + phaseShift=0.15, + acquisitionOrder=1, + psdFile="psd1.mrc", + enabled=True, + ) + inputSeries = FakeCtftomoSeries( + tsId="TS_001", + label="Series 1", + tiltSeries=associatedTs, + items=[ctf1], + ) + inputSet = FakeCtftomoOutputSet( + seriesList=[inputSeries], + associatedTiltSeriesSet=associatedTs, + ) + protocol = FakeProtocol("outputCtftomo", inputSet, "/tmp/fake-protocol") + service.currentProject = FakeCurrentProject(protocol) + + mapper = FakeMapper() + + result = service.createNewSetOfCtftomoSeriesService( + projectId=1, + protocolId=10, + outputName="outputCtftomo", + exclusions={ + "TS_001": { + "excluded": False, + "tiltimages": [], + } + }, + restack=False, + mapper=mapper, + ) + + assert result["status"] == 0 + assert result["outputName"] == "CTFTomoSeries_0" + assert result["postgresqlSync"] == { + "stored": True, + "protocolDbId": 654, + "outputName": "CTFTomoSeries_0", + } + assert storedCalls == [ + { + "projectId": 1, + "protocolDbId": 654, + "outputName": "CTFTomoSeries_0", + "scipionSet": protocol._definedOutputs["CTFTomoSeries_0"], + } + ] + def test_CreateNewSetOfCtftomoSeriesServiceCreatesFilteredSeries(service, tmp_path): associatedTs = FakeAssociatedTiltSeries() diff --git a/tests/unit/backend/api/services/test_project_service_metadata.py b/tests/unit/backend/api/services/test_project_service_metadata.py index 322204a5..7e925709 100644 --- a/tests/unit/backend/api/services/test_project_service_metadata.py +++ b/tests/unit/backend/api/services/test_project_service_metadata.py @@ -798,12 +798,49 @@ def test_RunMetadataTableActionServiceLaunchesSubsetProtocol( rowsByTable={"objects": []}, dao=dao, ) - mapper = object() + + class FakeDb: + # fakeDb + def fetchOne(self, *args, **kwargs): + return None + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + mapper = FakeMapper() calls = [] + syncCalls = [] + syncResult = { + "protocols": 2, + "dependencies": 1, + } + + def syncProjectProtocolsAndDependencies( + mapperArg, + projectIdArg, + refresh=True, + checkPid=True, + ): + syncCalls.append( + { + "mapper": mapperArg, + "projectId": projectIdArg, + "refresh": refresh, + "checkPid": checkPid, + } + ) + return syncResult patchOpenMetadataTable(service, monkeypatch, objMgr, table, calls=calls) monkeypatch.setattr(projectServiceModule, "OBJECT_TABLE", "objects") monkeypatch.setattr(projectServiceModule, "ProtUserSubSet", object()) + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + syncProjectProtocolsAndDependencies, + ) result = service.runMetadataTableActionService( projectId=1, @@ -820,7 +857,17 @@ def test_RunMetadataTableActionServiceLaunchesSubsetProtocol( assert result == { "success": True, "message": "Subset protocol was launched successfully", + "postgresqlSync": syncResult, + "postgresqlError": None, } + assert syncCalls == [ + { + "mapper": mapper, + "projectId": 1, + "refresh": True, + "checkPid": True, + } + ] assert calls == [ { "projectId": 1, diff --git a/tests/unit/backend/api/services/test_project_service_tiltseries.py b/tests/unit/backend/api/services/test_project_service_tiltseries.py index c5b725a0..1fb78941 100644 --- a/tests/unit/backend/api/services/test_project_service_tiltseries.py +++ b/tests/unit/backend/api/services/test_project_service_tiltseries.py @@ -495,6 +495,142 @@ def create(projectPath, suffix): assert createdOutputSet._dim == [128, 128, 40] +def test_CreateNewSetOfTiltSeriesServiceStoresSetWithResolvedProtocolDbId( + projectServiceModule, + service, + monkeypatch, +): + storedCalls = [] + + class FakeDb: + # fakeDb + def fetchOne(self, *args, **kwargs): + return None + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakeScipionSetPostgresqlMapper: + # fakeScipionSetPostgresqlMapper + def __init__(self, db): + self.db = db + + def storeSet(self, projectId, protocolDbId, outputName, scipionSet): + storedCalls.append( + { + "projectId": projectId, + "protocolDbId": protocolDbId, + "outputName": outputName, + "scipionSet": scipionSet, + } + ) + return { + "stored": True, + "protocolDbId": protocolDbId, + "outputName": outputName, + } + + class FakeCreatedTiltSeries: + # fakeCreatedTiltSeries + def __init__(self): + self._items = [] + self._dim = None + self._anglesCount = None + self._written = False + + def copyInfo(self, tiltSeries): + self._copiedInfoFrom = tiltSeries + + def append(self, item): + self._items.append(item) + + def setEnabled(self, value): + self._enabled = value + + def getSize(self): + return len(self._items) + + def setDim(self, dim): + self._dim = dim + + def setAnglesCount(self, count): + self._anglesCount = count + + def write(self): + self._written = True + + class FakeSetOfTiltSeriesFactory: + # fakeSetOfTiltSeriesFactory + @staticmethod + def create(projectPath, suffix): + return createdOutputSet + + createdOutputSet = FakeCreatedTiltSeriesOutputSet() + createdOutputSet.update = lambda item: None + createdOutputSet.remove = lambda item: None + + tiltSeries = FakeTiltSeries( + tsId="TS_001", + size=0, + dims=[128, 128, 40], + samplingRate=1.5, + tiltAxisAngle=90.0, + items=[], + ) + inputSet = FakeTiltSeriesSet( + items=[tiltSeries], + hasOddEven=False, + dims=[128, 128, 40], + ) + protocol = FakeProtocol("outputTiltSeries", inputSet) + service.currentProject = FakeCurrentProject(protocol) + + scipionSetMapperModule = importlib.import_module( + "app.backend.mapper.scipion_set_mapper" + ) + + monkeypatch.setattr( + scipionSetMapperModule, + "ScipionSetPostgresqlMapper", + FakeScipionSetPostgresqlMapper, + ) + monkeypatch.setattr(projectServiceModule, "SetOfTiltSeries", FakeSetOfTiltSeriesFactory) + monkeypatch.setattr(projectServiceModule, "TiltSeries", FakeCreatedTiltSeries) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 321, + ) + + mapper = FakeMapper() + + result = service.createNewSetOfTiltSeriesService( + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + exclusions={}, + restack=False, + mapper=mapper, + ) + + assert result["status"] == 0 + assert result["outputName"] == "TiltSeries_0" + assert result["postgresqlSync"] == { + "stored": True, + "protocolDbId": 321, + "outputName": "TiltSeries_0", + } + assert storedCalls == [ + { + "projectId": 1, + "protocolDbId": 321, + "outputName": "TiltSeries_0", + "scipionSet": createdOutputSet, + } + ] + def test_ResolveOutputForTiltSeriesReturns404WhenProtocolMissing(service): class BrokenCurrentProject: # brokenCurrentProject From d71cbcc6193e50794afe2cfaf0e1cfeebb9b06f9 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 23:09:15 +0200 Subject: [PATCH 114/278] Centralize generated set PostgreSQL persistence --- app/backend/api/services/project_service.py | 163 +++++++++--------- .../services/test_project_service_ctftomo.py | 3 + .../test_project_service_tiltseries.py | 1 + 3 files changed, 87 insertions(+), 80 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 83ed630b..c98887c9 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -860,6 +860,55 @@ def _resolvePostgresqlProtocolDbId( return None + def _storeGeneratedSetInPostgresql( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + outputName: str, + scipionSet, + contextLabel: str, + ) -> Dict[str, Any]: + postgresqlSync = None + postgresqlError = None + + if mapper is None: + return { + "postgresqlSync": postgresqlSync, + "postgresqlError": postgresqlError, + } + + try: + from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + protocolDbId = self._resolvePostgresqlProtocolDbId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) or protocolId + + setMapper = ScipionSetPostgresqlMapper(mapper.db) + postgresqlSync = setMapper.storeSet( + projectId=projectId, + protocolDbId=protocolDbId, + outputName=outputName, + scipionSet=scipionSet, + ) + + except Exception as e: + postgresqlError = str(e) + logger.exception( + "Failed to persist generated %s output to PostgreSQL. projectId=%s protocolId=%s outputName=%s", + contextLabel, + projectId, + protocolId, + outputName, + ) + + return { + "postgresqlSync": postgresqlSync, + "postgresqlError": postgresqlError, + } def _deletePersistedProtocolOutputsFromPostgresql( self, @@ -8585,36 +8634,22 @@ def createNewSetOfCtftomoSeriesService( "message": "No output was generated because it cannot be empty", } postgresqlSync = None + postgresqlError = None try: protocol._defineOutputs(**{newOutputName: outputSet}) protocol._store() - if mapper is not None: - try: - from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper - - setMapper = ScipionSetPostgresqlMapper(mapper.db) - protocolDbId = self._resolvePostgresqlProtocolDbId( - mapper=mapper, - projectId=projectId, - protocolId=protocolId, - ) or protocolId - - postgresqlSync = setMapper.storeSet( - projectId=projectId, - protocolDbId=protocolDbId, - outputName=newOutputName, - scipionSet=outputSet, - ) - - except Exception: - logger.exception( - "Failed to persist created CTFTomo output to PostgreSQL. projectId=%s protocolId=%s outputName=%s", - projectId, - protocolId, - newOutputName, - ) + postgresqlStore = self._storeGeneratedSetInPostgresql( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=newOutputName, + scipionSet=outputSet, + contextLabel="CTFTomo", + ) + postgresqlSync = postgresqlStore["postgresqlSync"] + postgresqlError = postgresqlStore["postgresqlError"] except Exception: logger.exception("Error attaching Ctftomo filtered set '%s' to protocol", newOutputName) @@ -8631,6 +8666,7 @@ def createNewSetOfCtftomoSeriesService( "createdSeries": createdCount, "restack": bool(restack), "postgresqlSync": postgresqlSync, + "postgresqlError": postgresqlError, } def _buildTiltSeriesPreviewCacheKey( @@ -9232,33 +9268,16 @@ def createNewSetOfTiltSeriesService( protocol._defineOutputs(**{newOutputName: outputSet}) protocol._store() - postgresqlSync = None - - if mapper is not None: - try: - from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper - - setMapper = ScipionSetPostgresqlMapper(mapper.db) - protocolDbId = self._resolvePostgresqlProtocolDbId( - mapper=mapper, - projectId=projectId, - protocolId=protocolId, - ) or protocolId - - postgresqlSync = setMapper.storeSet( - projectId=projectId, - protocolDbId=protocolDbId, - outputName=newOutputName, - scipionSet=outputSet, - ) - - except Exception: - logger.exception( - "Failed to persist created TiltSeries output to PostgreSQL. projectId=%s protocolId=%s outputName=%s", - projectId, - protocolId, - newOutputName, - ) + postgresqlStore = self._storeGeneratedSetInPostgresql( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=newOutputName, + scipionSet=outputSet, + contextLabel="TiltSeries", + ) + postgresqlSync = postgresqlStore["postgresqlSync"] + postgresqlError = postgresqlStore["postgresqlError"] logger.info("The new set (%s) has been created successfully", newOutputName) @@ -9269,6 +9288,7 @@ def createNewSetOfTiltSeriesService( "hasOddEven": bool(hasOddEven), "restack": bool(restack), "postgresqlSync": postgresqlSync, + "postgresqlError": postgresqlError, } def _cloneTiltImage(self, ti, included): @@ -10308,7 +10328,7 @@ def createCoords3dOutputFromPointsService( replaced += 1 break # ------------------------------------ - # 7. Saving and registering the output + # 5. Saving and registering the output # ------------------------------------ try: dstSet.write() @@ -10320,33 +10340,16 @@ def createCoords3dOutputFromPointsService( except Exception as e: raise HTTPException(500, f"Failed to attach new coords3d output: {e}") - postgresqlStored = False - postgresqlError = None - - if mapper is not None: - try: - from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper - - setMapper = ScipionSetPostgresqlMapper(mapper.db) - setMapper.storeSet( - projectId=projectId, - protocolDbId=self._resolvePostgresqlProtocolDbId( - mapper=mapper, - projectId=projectId, - protocolId=protocolId, - ) or protocolId, - outputName=outName, - scipionSet=dstSet, - ) - postgresqlStored = True - except Exception as e: - postgresqlError = str(e) - logger.exception( - "Failed to store new Coordinates3D output in PostgreSQL. projectId=%s protocolId=%s outputName=%s", - projectId, - protocolId, - outName, - ) + postgresqlStore = self._storeGeneratedSetInPostgresql( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outName, + scipionSet=dstSet, + contextLabel="Coordinates3D", + ) + postgresqlError = postgresqlStore["postgresqlError"] + postgresqlStored = mapper is not None and postgresqlError is None return { "success": True, diff --git a/tests/unit/backend/api/services/test_project_service_ctftomo.py b/tests/unit/backend/api/services/test_project_service_ctftomo.py index 4a73dd9f..9c13fe90 100644 --- a/tests/unit/backend/api/services/test_project_service_ctftomo.py +++ b/tests/unit/backend/api/services/test_project_service_ctftomo.py @@ -540,6 +540,7 @@ def test_CreateNewSetOfCtftomoSeriesServiceReturnsEmptyWhenEverythingExcluded(se "message": "No output was generated because it cannot be empty", } + def test_CreateNewSetOfCtftomoSeriesServiceStoresSetWithResolvedProtocolDbId( service, monkeypatch, @@ -639,6 +640,7 @@ def storeSet(self, projectId, protocolDbId, outputName, scipionSet): "protocolDbId": 654, "outputName": "CTFTomoSeries_0", } + assert result["postgresqlError"] is None assert storedCalls == [ { "projectId": 1, @@ -703,6 +705,7 @@ def test_CreateNewSetOfCtftomoSeriesServiceCreatesFilteredSeries(service, tmp_pa assert result["createdSeries"] == 1 assert result["restack"] is False assert result["postgresqlSync"] is None + assert result["postgresqlError"] is None assert "CTFTomoSeries_0" in protocol._definedOutputs createdSet = protocol._definedOutputs["CTFTomoSeries_0"] assert createdSet.isEmpty() is False diff --git a/tests/unit/backend/api/services/test_project_service_tiltseries.py b/tests/unit/backend/api/services/test_project_service_tiltseries.py index 1fb78941..c10ed029 100644 --- a/tests/unit/backend/api/services/test_project_service_tiltseries.py +++ b/tests/unit/backend/api/services/test_project_service_tiltseries.py @@ -622,6 +622,7 @@ def create(projectPath, suffix): "protocolDbId": 321, "outputName": "TiltSeries_0", } + assert result["postgresqlError"] is None assert storedCalls == [ { "projectId": 1, From 4d4e0ebb4ccb979c2bc0d87fffea34096921f200 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 23:18:59 +0200 Subject: [PATCH 115/278] Resolve PostgreSQL reader protocol ids --- app/backend/api/services/project_service.py | 57 ++++++++++++++++++--- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index c98887c9..31c44010 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -860,6 +860,21 @@ def _resolvePostgresqlProtocolDbId( return None + def _resolvePostgresqlReaderProtocolId( + self, + mapper, + projectId: Optional[int], + protocolId: Union[int, str], + ) -> Union[int, str]: + if mapper is None: + return protocolId + + return self._resolvePostgresqlProtocolDbId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) or protocolId + def _storeGeneratedSetInPostgresql( self, mapper, @@ -7179,10 +7194,16 @@ def _getPostgresqlTiltSeriesReaderIfAvailable( try: from app.backend.viewers.postgresql_tiltseries_reader import PostgresqlTiltSeriesReader + readerProtocolId = self._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + reader = PostgresqlTiltSeriesReader( db=mapper.db, projectId=projectId, - protocolId=protocolId, + protocolId=readerProtocolId, outputName=outputName, ) @@ -7672,10 +7693,16 @@ def _getPostgresqlCtftomoReaderIfAvailable( try: from app.backend.viewers.postgresql_ctftomo_reader import PostgresqlCtftomoReader + readerProtocolId = self._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + reader = PostgresqlCtftomoReader( db=mapper.db, projectId=projectId, - protocolId=protocolId, + protocolId=readerProtocolId, outputName=outputName, ) @@ -7742,6 +7769,12 @@ def _getPostgresqlVolumeReaderIfAvailable( if mapper is None: return None + readerProtocolId = self._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + try: from app.backend.viewers.postgresql_coords3d_tomogram_volume_reader import \ PostgresqlCoords3dTomogramVolumeReader @@ -7749,7 +7782,7 @@ def _getPostgresqlVolumeReaderIfAvailable( reader = PostgresqlCoords3dTomogramVolumeReader( db=mapper.db, projectId=projectId, - protocolId=protocolId, + protocolId=readerProtocolId, outputName=outputName, ) @@ -7771,7 +7804,7 @@ def _getPostgresqlVolumeReaderIfAvailable( reader = PostgresqlVolumeReader( db=mapper.db, projectId=projectId, - protocolId=protocolId, + protocolId=readerProtocolId, outputName=outputName, ) @@ -9315,10 +9348,16 @@ def _getPostgresqlCoords3dReaderIfAvailable( try: from app.backend.viewers.postgresql_coords3d_reader import PostgresqlCoords3dReader + readerProtocolId = self._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + reader = PostgresqlCoords3dReader( db=mapper.db, projectId=projectId, - protocolId=protocolId, + protocolId=readerProtocolId, outputName=outputName, ) @@ -10630,10 +10669,16 @@ def _getPostgresqlDAOIfAvailable( if mapper is None: return None + readerProtocolId = self._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + dao = PostgresqlDAO( db=mapper.db, projectId=projectId, - protocolId=protocolId, + protocolId=readerProtocolId, outputName=outputName, ) From 2f07fcd7fcf77f03c54806bf11fc1035361135a4 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 23 Jun 2026 23:25:25 +0200 Subject: [PATCH 116/278] Test PostgreSQL reader protocol id resolution --- .../services/test_project_service_coords3d.py | 57 +++++++++++++ .../services/test_project_service_ctftomo.py | 57 +++++++++++++ .../services/test_project_service_metadata.py | 57 +++++++++++++ .../test_project_service_tiltseries.py | 56 +++++++++++++ .../services/test_project_service_volumes.py | 84 +++++++++++++++++++ 5 files changed, 311 insertions(+) diff --git a/tests/unit/backend/api/services/test_project_service_coords3d.py b/tests/unit/backend/api/services/test_project_service_coords3d.py index b55f311f..2d142903 100644 --- a/tests/unit/backend/api/services/test_project_service_coords3d.py +++ b/tests/unit/backend/api/services/test_project_service_coords3d.py @@ -299,6 +299,63 @@ def test_ListCoordinates3dTomogramsServiceBuildsTomogramList(service, tmp_path): assert service.tomoList["TS_001"] is tomo1 assert service.tomoList["TS_002"] is tomo2 +def test_GetPostgresqlCoords3dReaderIfAvailableUsesResolvedProtocolDbId( + service, + monkeypatch, +): + createdReaders = [] + + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakePostgresqlCoords3dReader: + # fakePostgresqlCoords3dReader + def __init__(self, db, projectId, protocolId, outputName): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + createdReaders.append(self) + + def hasOutput(self): + return True + + readerModule = importlib.import_module( + "app.backend.viewers.postgresql_coords3d_reader" + ) + + monkeypatch.setattr( + readerModule, + "PostgresqlCoords3dReader", + FakePostgresqlCoords3dReader, + ) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 987, + ) + + mapper = FakeMapper() + + reader = service._getPostgresqlCoords3dReaderIfAvailable( + mapper=mapper, + projectId=1, + protocolId=10, + outputName="outputCoords3d", + ) + + assert reader is createdReaders[0] + assert createdReaders[0].db is mapper.db + assert createdReaders[0].projectId == 1 + assert createdReaders[0].protocolId == 987 + assert createdReaders[0].outputName == "outputCoords3d" + def test_GetCoordinates3dPointsServiceBuildsPointPayload(service, tmp_path): tomoPath = tmp_path / "tomo1.mrc" diff --git a/tests/unit/backend/api/services/test_project_service_ctftomo.py b/tests/unit/backend/api/services/test_project_service_ctftomo.py index 9c13fe90..c161e0fb 100644 --- a/tests/unit/backend/api/services/test_project_service_ctftomo.py +++ b/tests/unit/backend/api/services/test_project_service_ctftomo.py @@ -371,6 +371,63 @@ def test_ListOutputCtftomoSeriesServiceBuildsSummaries(service, tmp_path): }, ] +def test_GetPostgresqlCtftomoReaderIfAvailableUsesResolvedProtocolDbId( + service, + monkeypatch, +): + createdReaders = [] + + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakePostgresqlCtftomoReader: + # fakePostgresqlCtftomoReader + def __init__(self, db, projectId, protocolId, outputName): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + createdReaders.append(self) + + def hasOutput(self): + return True + + readerModule = importlib.import_module( + "app.backend.viewers.postgresql_ctftomo_reader" + ) + + monkeypatch.setattr( + readerModule, + "PostgresqlCtftomoReader", + FakePostgresqlCtftomoReader, + ) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 654, + ) + + mapper = FakeMapper() + + reader = service._getPostgresqlCtftomoReaderIfAvailable( + mapper=mapper, + projectId=1, + protocolId=10, + outputName="outputCtftomo", + ) + + assert reader is createdReaders[0] + assert createdReaders[0].db is mapper.db + assert createdReaders[0].projectId == 1 + assert createdReaders[0].protocolId == 654 + assert createdReaders[0].outputName == "outputCtftomo" + def test_GetCtftomoSeriesViewsServiceBuildsFrames(service, tmp_path): associatedTs = FakeAssociatedTiltSeries( diff --git a/tests/unit/backend/api/services/test_project_service_metadata.py b/tests/unit/backend/api/services/test_project_service_metadata.py index 7e925709..527476f1 100644 --- a/tests/unit/backend/api/services/test_project_service_metadata.py +++ b/tests/unit/backend/api/services/test_project_service_metadata.py @@ -399,6 +399,63 @@ def test_ListOutputMetadataTablesServiceReturnsSummaries(service, monkeypatch): }, ] +def test_GetPostgresqlDAOIfAvailableUsesResolvedProtocolDbId( + service, + monkeypatch, +): + createdDaos = [] + + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakePostgresqlDAO: + # fakePostgresqlDAO + def __init__(self, db, projectId, protocolId, outputName): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + createdDaos.append(self) + + def hasOutput(self): + return True + + daoModule = importlib.import_module( + "app.backend.viewers.postgresql_dao" + ) + + monkeypatch.setattr( + daoModule, + "PostgresqlDAO", + FakePostgresqlDAO, + ) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 852, + ) + + mapper = FakeMapper() + + dao = service._getPostgresqlDAOIfAvailable( + projectId=1, + protocolId=10, + outputName="outputParticles", + mapper=mapper, + ) + + assert dao is createdDaos[0] + assert createdDaos[0].db is mapper.db + assert createdDaos[0].projectId == 1 + assert createdDaos[0].protocolId == 852 + assert createdDaos[0].outputName == "outputParticles" + def test_GetMetadataTableSchemaServiceBuildsColumns(service, monkeypatch): columns = [ diff --git a/tests/unit/backend/api/services/test_project_service_tiltseries.py b/tests/unit/backend/api/services/test_project_service_tiltseries.py index c10ed029..c6245784 100644 --- a/tests/unit/backend/api/services/test_project_service_tiltseries.py +++ b/tests/unit/backend/api/services/test_project_service_tiltseries.py @@ -280,6 +280,62 @@ def service(projectServiceModule): instance.currentProject = None return instance +def test_GetPostgresqlTiltSeriesReaderIfAvailableUsesResolvedProtocolDbId( + service, + monkeypatch, +): + createdReaders = [] + + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakePostgresqlTiltSeriesReader: + # fakePostgresqlTiltSeriesReader + def __init__(self, db, projectId, protocolId, outputName): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + createdReaders.append(self) + + def hasOutput(self): + return True + + readerModule = importlib.import_module( + "app.backend.viewers.postgresql_tiltseries_reader" + ) + + monkeypatch.setattr( + readerModule, + "PostgresqlTiltSeriesReader", + FakePostgresqlTiltSeriesReader, + ) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 321, + ) + + mapper = FakeMapper() + + reader = service._getPostgresqlTiltSeriesReaderIfAvailable( + mapper=mapper, + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + ) + + assert reader is createdReaders[0] + assert createdReaders[0].db is mapper.db + assert createdReaders[0].projectId == 1 + assert createdReaders[0].protocolId == 321 + assert createdReaders[0].outputName == "outputTiltSeries" def test_ListOutputTiltSeriesServiceBuildsSummaries(service): ts1 = FakeTiltSeries( diff --git a/tests/unit/backend/api/services/test_project_service_volumes.py b/tests/unit/backend/api/services/test_project_service_volumes.py index a52275ce..edbcb491 100644 --- a/tests/unit/backend/api/services/test_project_service_volumes.py +++ b/tests/unit/backend/api/services/test_project_service_volumes.py @@ -174,6 +174,90 @@ def test_ResolveOutputForVolumesReturnsExactOutput(service): assert resolvedProtocol is protocol assert resolvedOutput is volume +def test_GetPostgresqlVolumeReaderIfAvailableUsesResolvedProtocolDbId( + service, + monkeypatch, +): + createdCoordsVolumeReaders = [] + createdVolumeReaders = [] + + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakePostgresqlCoords3dTomogramVolumeReader: + # fakePostgresqlCoords3dTomogramVolumeReader + def __init__(self, db, projectId, protocolId, outputName): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + createdCoordsVolumeReaders.append(self) + + def hasOutput(self): + return False + + class FakePostgresqlVolumeReader: + # fakePostgresqlVolumeReader + def __init__(self, db, projectId, protocolId, outputName): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + createdVolumeReaders.append(self) + + def hasOutput(self): + return True + + coordsVolumeModule = importlib.import_module( + "app.backend.viewers.postgresql_coords3d_tomogram_volume_reader" + ) + volumeModule = importlib.import_module( + "app.backend.viewers.postgresql_volume_reader" + ) + + monkeypatch.setattr( + coordsVolumeModule, + "PostgresqlCoords3dTomogramVolumeReader", + FakePostgresqlCoords3dTomogramVolumeReader, + ) + monkeypatch.setattr( + volumeModule, + "PostgresqlVolumeReader", + FakePostgresqlVolumeReader, + ) + monkeypatch.setattr( + service, + "_resolvePostgresqlProtocolDbId", + lambda mapper, projectId, protocolId: 741, + ) + + mapper = FakeMapper() + + reader = service._getPostgresqlVolumeReaderIfAvailable( + mapper=mapper, + projectId=1, + protocolId=10, + outputName="outputVolumes", + ) + + assert reader is createdVolumeReaders[0] + + assert createdCoordsVolumeReaders[0].db is mapper.db + assert createdCoordsVolumeReaders[0].projectId == 1 + assert createdCoordsVolumeReaders[0].protocolId == 741 + assert createdCoordsVolumeReaders[0].outputName == "outputVolumes" + + assert createdVolumeReaders[0].db is mapper.db + assert createdVolumeReaders[0].projectId == 1 + assert createdVolumeReaders[0].protocolId == 741 + assert createdVolumeReaders[0].outputName == "outputVolumes" + def test_ResolveOutputForVolumesSupportsAliasFallback(service): volume = FakeVolumeOutput("/tmp/volume.mrc") From 254708aed7d01efd1a0ba803e8204b0260384e9e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 10:13:41 +0200 Subject: [PATCH 117/278] Require PostgreSQL readers for web viewers --- app/backend/api/services/project_service.py | 123 +++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 31c44010..600bceb4 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -860,6 +860,40 @@ def _resolvePostgresqlProtocolDbId( return None + def _raisePostgresqlViewerUnavailable( + self, + viewerName: str, + projectId: int, + protocolId: Union[int, str], + outputName: str, + reason: Optional[str] = None, + **extra, + ) -> None: + extraText = " ".join( + "%s=%s" % (key, value) + for key, value in extra.items() + if value is not None + ) + + logger.warning( + "%s output is not available in PostgreSQL metadata. projectId=%s protocolId=%s outputName=%s reason=%s %s", + viewerName, + projectId, + protocolId, + outputName, + reason, + extraText, + ) + + detail = "%s output is not available in PostgreSQL metadata" % viewerName + if reason: + detail = "%s: %s" % (detail, reason) + + raise HTTPException( + status_code=404, + detail=detail, + ) + def _resolvePostgresqlReaderProtocolId( self, mapper, @@ -7848,6 +7882,15 @@ def listOutputVolumesService( getattr(pgReader, "lastSkipReason", None), ) + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Volume", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + ) + protocol, output = self._resolveOutputForVolumes( protocolId=protocolId, outputName=outputName, @@ -7886,6 +7929,16 @@ def getVolumeInfoService( getattr(pgReader, "lastSkipReason", None), ) + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Volume", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + volumeId=volumeId, + ) + protocol, output = self._resolveOutputForVolumes( protocolId=protocolId, outputName=outputName, @@ -7928,6 +7981,16 @@ def getVolumeHistogramService( getattr(pgReader, "lastSkipReason", None), ) + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Volume histogram", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + volumeId=volumeId, + ) + protocol, output = self._resolveOutputForVolumes( protocolId=protocolId, outputName=outputName, @@ -8015,6 +8078,16 @@ def renderVolumeSliceService( getattr(pgReader, "lastSkipReason", None), ) + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Volume slice", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + volumeId=volumeId, + ) + protocol, output = self._resolveOutputForVolumes( protocolId=protocolId, outputName=outputName, @@ -8082,6 +8155,16 @@ def getVolumeData3dService( getattr(pgReader, "lastSkipReason", None), ) + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Volume 3D data", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + volumeId=volumeId, + ) + protocol, output = self._resolveOutputForVolumes( protocolId=protocolId, outputName=outputName, @@ -8351,6 +8434,16 @@ def getVolumeSurfaceMesh( getattr(pgReader, "lastSkipReason", None), ) + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Volume surface mesh", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + volumeId=volumeId, + ) + protocol, output = self._resolveOutputForVolumes( protocolId=protocolId, outputName=outputName, @@ -8428,6 +8521,15 @@ def listOutputTiltSeriesService( if pgReader is not None: return pgReader.listTiltSeries() + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="TiltSeries", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="reader_not_available", + ) + _, setOfTiltSeries = self._resolveOutputForTiltSeries( protocolId=protocolId, outputName=outputName, @@ -8490,6 +8592,16 @@ def getTiltSeriesFramesService( if payload is not None: return payload + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="TiltSeries frames", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="frames_not_available" if pgReader is not None else "reader_not_available", + tiltSeriesId=tiltSeriesId, + ) + protocol, setOfTiltSeries = self._resolveOutputForTiltSeries( protocolId=protocolId, outputName=outputName, @@ -9436,7 +9548,7 @@ def listCoordinates3dTomogramsService( if mapper is not None: raise HTTPException( status_code=404, - detail="TiltSeries frame not found in PostgreSQL metadata", + detail="Coordinates3D output is not available in PostgreSQL metadata", ) protocol, setOfCoordinates3D = self._resolveOutputForCoordinates3d( @@ -10713,6 +10825,15 @@ def _getMetadataObjectManagerForOutput( objMgr.getTables() return objMgr + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Metadata", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="dao_not_available", + ) + _, _, metaPath = self._resolveOutputForMetadata( protocolId=protocolId, outputName=outputName, From 83658bc3b14efa62a782fd72c38b453601f18f26 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 10:25:59 +0200 Subject: [PATCH 118/278] Block CTFTomo viewer runtime fallback --- app/backend/api/services/project_service.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 600bceb4..50ab36d7 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -7442,6 +7442,15 @@ def listOutputCtftomoSeriesService( if pgReader is not None: return pgReader.listCtftomoSeries() + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="CTFTomo", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="reader_not_available", + ) + protocol, output = self._resolveOutputForCtftomoSeries( protocolId=protocolId, outputName=outputName, @@ -7634,6 +7643,16 @@ def renderCtfTomoPsdImageService( psdPath, ) + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="CTFTomo PSD", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="psd_file_not_available", + psdPath=psdPath, + ) + protocol, output = self._resolveOutputForCtftomoSeries( protocolId=protocolId, outputName=outputName, From db4029ab1b97ee2159fdc1e22b0df06294f612f1 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 10:30:00 +0200 Subject: [PATCH 119/278] Test PostgreSQL-only viewer mode --- .../services/test_project_service_ctftomo.py | 55 ++++++++++ .../services/test_project_service_metadata.py | 38 +++++++ .../test_project_service_tiltseries.py | 58 ++++++++++ .../services/test_project_service_volumes.py | 101 ++++++++++++++++++ 4 files changed, 252 insertions(+) diff --git a/tests/unit/backend/api/services/test_project_service_ctftomo.py b/tests/unit/backend/api/services/test_project_service_ctftomo.py index c161e0fb..e3864fdc 100644 --- a/tests/unit/backend/api/services/test_project_service_ctftomo.py +++ b/tests/unit/backend/api/services/test_project_service_ctftomo.py @@ -371,6 +371,61 @@ def test_ListOutputCtftomoSeriesServiceBuildsSummaries(service, tmp_path): }, ] +def test_ListOutputCtftomoSeriesServiceRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + monkeypatch.setattr( + service, + "_getPostgresqlCtftomoReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy CTFTomo fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForCtftomoSeries", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service.listOutputCtftomoSeriesService( + projectId=1, + protocolId=10, + outputName="outputCtftomo", + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "CTFTomo output is not available in PostgreSQL metadata" in exc.value.detail + assert "reader_not_available" in exc.value.detail + + +def test_RenderCtfTomoPsdImageServiceRequiresPostgresqlPathWhenMapperIsPresent( + service, + monkeypatch, +): + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy CTFTomo PSD fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForCtftomoSeries", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service.renderCtfTomoPsdImageService( + projectId=1, + protocolId=10, + outputName="outputCtftomo", + psdPath="relative/psd.mrc", + size=512, + fmt="png", + inline=True, + index=0, + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "CTFTomo PSD output is not available in PostgreSQL metadata" in exc.value.detail + assert "psd_file_not_available" in exc.value.detail + + def test_GetPostgresqlCtftomoReaderIfAvailableUsesResolvedProtocolDbId( service, monkeypatch, diff --git a/tests/unit/backend/api/services/test_project_service_metadata.py b/tests/unit/backend/api/services/test_project_service_metadata.py index 527476f1..21a39a90 100644 --- a/tests/unit/backend/api/services/test_project_service_metadata.py +++ b/tests/unit/backend/api/services/test_project_service_metadata.py @@ -399,6 +399,7 @@ def test_ListOutputMetadataTablesServiceReturnsSummaries(service, monkeypatch): }, ] + def test_GetPostgresqlDAOIfAvailableUsesResolvedProtocolDbId( service, monkeypatch, @@ -457,6 +458,43 @@ def hasOutput(self): assert createdDaos[0].outputName == "outputParticles" +def test_GetMetadataObjectManagerForOutputRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + monkeypatch.setattr( + service, + "_getPostgresqlDAOIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy metadata fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForMetadata", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service._getMetadataObjectManagerForOutput( + projectId=1, + protocolId=10, + outputName="outputParticles", + mapper=FakeMapper(), + ) + + assert exc.value.status_code == 404 + assert "Metadata output is not available in PostgreSQL metadata" in exc.value.detail + assert "dao_not_available" in exc.value.detail + + def test_GetMetadataTableSchemaServiceBuildsColumns(service, monkeypatch): columns = [ FakeColumn("id", "Id", IntRenderer(), sortable=True), diff --git a/tests/unit/backend/api/services/test_project_service_tiltseries.py b/tests/unit/backend/api/services/test_project_service_tiltseries.py index c6245784..06c8ae09 100644 --- a/tests/unit/backend/api/services/test_project_service_tiltseries.py +++ b/tests/unit/backend/api/services/test_project_service_tiltseries.py @@ -280,6 +280,7 @@ def service(projectServiceModule): instance.currentProject = None return instance + def test_GetPostgresqlTiltSeriesReaderIfAvailableUsesResolvedProtocolDbId( service, monkeypatch, @@ -337,6 +338,63 @@ def hasOutput(self): assert createdReaders[0].protocolId == 321 assert createdReaders[0].outputName == "outputTiltSeries" +def test_ListOutputTiltSeriesServiceRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + monkeypatch.setattr( + service, + "_getPostgresqlTiltSeriesReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy TiltSeries fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForTiltSeries", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service.listOutputTiltSeriesService( + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "TiltSeries output is not available in PostgreSQL metadata" in exc.value.detail + assert "reader_not_available" in exc.value.detail + + +def test_GetTiltSeriesFramesServiceRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + monkeypatch.setattr( + service, + "_getPostgresqlTiltSeriesReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy TiltSeries fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForTiltSeries", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service.getTiltSeriesFramesService( + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + tiltSeriesId="TS_001", + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "TiltSeries frames output is not available in PostgreSQL metadata" in exc.value.detail + assert "reader_not_available" in exc.value.detail + + def test_ListOutputTiltSeriesServiceBuildsSummaries(service): ts1 = FakeTiltSeries( tsId="TS_001", diff --git a/tests/unit/backend/api/services/test_project_service_volumes.py b/tests/unit/backend/api/services/test_project_service_volumes.py index edbcb491..b165d2bd 100644 --- a/tests/unit/backend/api/services/test_project_service_volumes.py +++ b/tests/unit/backend/api/services/test_project_service_volumes.py @@ -269,6 +269,107 @@ def test_ResolveOutputForVolumesSupportsAliasFallback(service): assert resolvedProtocol is protocol assert resolvedOutput is volume +@pytest.mark.parametrize( + "serviceCall, expectedDetail", + [ + ( + lambda service, mapper: service.listOutputVolumesService( + projectId=1, + protocolId=10, + outputName="outputVolumes", + mapper=mapper, + ), + "Volume output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.getVolumeInfoService( + projectId=1, + protocolId=10, + outputName="outputVolumes", + volumeId=0, + mapper=mapper, + ), + "Volume output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.getVolumeHistogramService( + projectId=1, + protocolId=10, + outputName="outputVolumes", + volumeId=0, + bins=32, + mapper=mapper, + ), + "Volume histogram output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.renderVolumeSliceService( + projectId=1, + protocolId=10, + outputName="outputVolumes", + volumeId=0, + sliceIndex=0, + axis="z", + colormap=None, + normalize="minmax", + scale=1.0, + inline=True, + mapper=mapper, + ), + "Volume slice output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.getVolumeData3dService( + projectId=1, + protocolId=10, + outputName="outputVolumes", + volumeId=0, + maxDim=32, + method="binning", + mapper=mapper, + ), + "Volume 3D data output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.getVolumeSurfaceMesh( + projectId=1, + protocolId=10, + outputName="outputVolumes", + volumeId=0, + level=0.1, + maxDim=32, + method="binning", + maxTriangles=1000, + currentUser={"id": 1}, + mapper=mapper, + ), + "Volume surface mesh output is not available in PostgreSQL metadata", + ), + ], +) +def test_VolumeServicesRequirePostgresqlWhenMapperIsPresent( + service, + monkeypatch, + serviceCall, + expectedDetail, +): + monkeypatch.setattr( + service, + "_getPostgresqlVolumeReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy volume fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForVolumes", failRuntimeFallback) + + with pytest.raises(HTTPException) as exc: + serviceCall(service, object()) + + assert exc.value.status_code == 404 + assert expectedDetail in exc.value.detail + assert "reader_not_available" in exc.value.detail def test_ResolveOutputForVolumesReturns404WhenProtocolMissing(service): service.currentProject = FakeCurrentProject(protocolError=RuntimeError("missing")) From 3834947c6245b854f50e5f7f4800e9bb1c3746da Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 11:15:39 +0200 Subject: [PATCH 120/278] Report persisted outputs during project sync --- app/backend/api/services/project_service.py | 39 +++++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 50ab36d7..54f6f34d 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -1227,6 +1227,9 @@ def syncProjectProtocolsAndDependencies( currentProtocolIds: Set[str] = set() protocolsByScipionId: Dict[str, Any] = {} + outputSyncResults: List[Dict[str, Any]] = [] + outputSyncErrors: List[Dict[str, Any]] = [] + # 1) Save all protocol nodes that are currently present in the real Scipion graph for nodeId, nodeObj in nodesDict.items(): nodeIdText = str(nodeId) @@ -1244,7 +1247,23 @@ def syncProjectProtocolsAndDependencies( protocolDbId = mapper.saveProtocol(protocolContext) if self._shouldRegisterProtocolOutputs(protocol): - self.registerOutput(projectId, protocol) + try: + outputResults = self.registerOutput( + projectId=projectId, + protocol=protocol, + mapper=mapper, + ) + outputSyncResults.extend(outputResults or []) + except Exception as exc: + outputSyncErrors.append({ + "protocolId": nodeIdText, + "error": str(exc), + }) + logger.exception( + "Failed to sync protocol outputs. projectId=%s protocolId=%s", + projectId, + nodeIdText, + ) currentProtocolIds.add(nodeIdText) protocolDbIdByScipionId[nodeIdText] = int(protocolDbId) @@ -1293,10 +1312,18 @@ def syncProjectProtocolsAndDependencies( if callable(replaceInputRefs): savedInputRefs = replaceInputRefs(projectId, inputRefs) + outputResultsByKind: Dict[str, int] = {} + for item in outputSyncResults: + mapperKind = str(item.get("mapperKind") or "unknown") + outputResultsByKind[mapperKind] = outputResultsByKind.get(mapperKind, 0) + 1 + return { "protocols": len(protocolDbIdByScipionId), "dependencies": int(savedEdges), "inputRefs": int(savedInputRefs), + "outputs": len(outputSyncResults), + "outputsByKind": outputResultsByKind, + "outputErrors": outputSyncErrors, } def syncProjectGraphAfterMutation( @@ -3756,6 +3783,7 @@ def registerOutput( projectId: int, protocol: Any, raiseOnError: bool = False, + mapper=None, ) -> List[Dict[str, Any]]: """ Persist all Scipion protocol outputs in PostgreSQL. @@ -3773,7 +3801,10 @@ def registerOutput( results: List[Dict[str, Any]] = [] - mapper = getMapper() + closeMapper = mapper is None + if mapper is None: + mapper = getMapper() + try: protocolDbId = self._resolveProtocolDbIdForOutputPersistence( mapper.db, @@ -3829,8 +3860,10 @@ def registerOutput( exc_info=True, ) + finally: - mapper.db.close() + if closeMapper: + mapper.db.close() return results From 5e273346ed76e136cc84afd3c82683c0ab929022 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 11:23:43 +0200 Subject: [PATCH 121/278] Report skipped and failed output sync --- app/backend/api/services/project_service.py | 46 +++++++++++++++++---- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 54f6f34d..6ad44af7 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -1213,7 +1213,7 @@ def syncProjectProtocolsAndDependencies( projectId: int, refresh: bool = False, checkPid: bool = False, - ) -> Dict[str, int]: + ) -> Dict[str, Any]: if self.currentProject is None: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -1248,12 +1248,30 @@ def syncProjectProtocolsAndDependencies( if self._shouldRegisterProtocolOutputs(protocol): try: - outputResults = self.registerOutput( + outputReport = self.registerOutput( projectId=projectId, protocol=protocol, mapper=mapper, + returnReport=True, ) - outputSyncResults.extend(outputResults or []) + + outputSyncResults.extend(outputReport.get("persisted") or []) + + for skippedOutput in outputReport.get("skipped") or []: + outputSyncErrors.append({ + "protocolId": nodeIdText, + "outputName": skippedOutput.get("outputName"), + "outputClassName": skippedOutput.get("outputClassName"), + "reason": skippedOutput.get("reason"), + }) + + for outputError in outputReport.get("errors") or []: + outputSyncErrors.append({ + "protocolId": nodeIdText, + "outputName": outputError.get("outputName"), + "outputClassName": outputError.get("outputClassName"), + "error": outputError.get("error"), + }) except Exception as exc: outputSyncErrors.append({ "protocolId": nodeIdText, @@ -1333,7 +1351,7 @@ def syncProjectGraphAfterMutation( actionLabel: str, refresh: bool = True, checkPid: bool = True, - ) -> Dict[str, int]: + ) -> Dict[str, Any]: try: return self.syncProjectProtocolsAndDependencies( mapper, @@ -3784,7 +3802,8 @@ def registerOutput( protocol: Any, raiseOnError: bool = False, mapper=None, - ) -> List[Dict[str, Any]]: + returnReport: bool = False, + ) -> Union[List[Dict[str, Any]], Dict[str, Any]]: """ Persist all Scipion protocol outputs in PostgreSQL. @@ -3800,6 +3819,8 @@ def registerOutput( from app.backend.mapper import ScipionObjectPostgresqlMapper, ScipionSetPostgresqlMapper results: List[Dict[str, Any]] = [] + skippedOutputs: List[Dict[str, Any]] = [] + outputErrors: List[Dict[str, Any]] = [] closeMapper = mapper is None if mapper is None: @@ -3839,7 +3860,15 @@ def registerOutput( ) result["mapperKind"] = "tree" + else: + + skippedOutputs.append({ + "outputName": outputName, + "outputClassName": self._getOutputClassName(outputObj), + "reason": "unsupported_output_type", + }) + continue result["outputName"] = outputName @@ -3850,6 +3879,11 @@ def registerOutput( if raiseOnError: raise + outputErrors.append({ + "outputName": outputName, + "outputClassName": self._getOutputClassName(outputObj), + "error": str(exc), + }) logger.warning( "Could not persist Scipion output. projectId=%s protocolId=%s outputName=%s outputClass=%s error=%s", projectId, @@ -3859,8 +3893,6 @@ def registerOutput( exc, exc_info=True, ) - - finally: if closeMapper: mapper.db.close() From 70783ca32743c7fc943e150b49dc84b94a5d072e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 11:27:52 +0200 Subject: [PATCH 122/278] Return output sync report from registerOutput --- app/backend/api/services/project_service.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 6ad44af7..252f202f 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -3897,6 +3897,13 @@ def registerOutput( if closeMapper: mapper.db.close() + if returnReport: + return { + "persisted": results, + "skipped": skippedOutputs, + "errors": outputErrors, + } + return results def _resolveProtocolDbIdForOutputPersistence(self, db, projectId: int, protocol: Any) -> int: From 3479e8feb8819a7c0314c097402c66b4daf34726 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 11:31:00 +0200 Subject: [PATCH 123/278] Test project sync output report --- .../test_project_service_protocols.py | 310 ++++++++++++++++++ 1 file changed, 310 insertions(+) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index 59b572f1..70188206 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -272,6 +272,316 @@ def assertSuccessEnvelope(result): assert result["errors"] == [] +def test_RegisterOutputReturnsPersistenceReport( + projectServiceModule, + service, + monkeypatch, +): + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + class FakeSetOutput: + # fakeSetOutput + def getClassName(self): + return "SetOfParticles" + + class FakeObjectOutput: + # fakeObjectOutput + def getClassName(self): + return "Volume" + + class FakeBadSetOutput: + # fakeBadSetOutput + def getClassName(self): + return "SetOfBadThings" + + class FakeUnsupportedOutput: + # fakeUnsupportedOutput + pass + + class FakeProtocolWithOutputs: + # fakeProtocolWithOutputs + def getObjId(self): + return 10 + + def iterOutputAttributes(self): + return [ + ("outputParticles", FakeSetOutput()), + ("outputVolume", FakeObjectOutput()), + ("badSet", FakeBadSetOutput()), + ("unsupportedOutput", FakeUnsupportedOutput()), + ] + + class FakeScipionSetPostgresqlMapper: + # fakeScipionSetPostgresqlMapper + def __init__(self, db): + self.db = db + + def storeSet(self, projectId, protocolDbId, outputName, scipionSet): + if outputName == "badSet": + raise RuntimeError("broken set output") + + return { + "projectId": projectId, + "protocolDbId": protocolDbId, + "stored": True, + } + + class FakeScipionObjectPostgresqlMapper: + # fakeScipionObjectPostgresqlMapper + def __init__(self, db): + self.db = db + + def storeObjectTree( + self, + projectId, + protocolDbId, + outputName, + scipionObj, + includeNestedProperties, + ): + return { + "projectId": projectId, + "protocolDbId": protocolDbId, + "stored": True, + "includeNestedProperties": includeNestedProperties, + } + + mapperPackage = importlib.import_module("app.backend.mapper") + + monkeypatch.setattr( + mapperPackage, + "ScipionSetPostgresqlMapper", + FakeScipionSetPostgresqlMapper, + ) + monkeypatch.setattr( + mapperPackage, + "ScipionObjectPostgresqlMapper", + FakeScipionObjectPostgresqlMapper, + ) + monkeypatch.setattr( + service, + "_resolveProtocolDbIdForOutputPersistence", + lambda db, projectId, protocol: 500, + ) + + report = service.registerOutput( + projectId=1, + protocol=FakeProtocolWithOutputs(), + mapper=FakeMapper(), + returnReport=True, + ) + + assert report["persisted"] == [ + { + "projectId": 1, + "protocolDbId": 500, + "stored": True, + "mapperKind": "flat_set", + "outputName": "outputParticles", + "outputClassName": "SetOfParticles", + }, + { + "projectId": 1, + "protocolDbId": 500, + "stored": True, + "includeNestedProperties": True, + "mapperKind": "tree", + "outputName": "outputVolume", + "outputClassName": "Volume", + }, + ] + + assert report["skipped"] == [ + { + "outputName": "unsupportedOutput", + "outputClassName": "FakeUnsupportedOutput", + "reason": "unsupported_output_type", + } + ] + + assert report["errors"] == [ + { + "outputName": "badSet", + "outputClassName": "SetOfBadThings", + "error": "broken set output", + } + ] + + +def test_SyncProjectProtocolsAndDependenciesReportsOutputPersistence( + projectServiceModule, + service, + monkeypatch, +): + service.syncProjectProtocolsAndDependencies = ( + projectServiceModule.ProjectService.syncProjectProtocolsAndDependencies.__get__( + service, + projectServiceModule.ProjectService, + ) + ) + + class FakeProtocolNode: + # fakeProtocolNode + def __init__(self, protocol): + self.run = protocol + self._parents = [] + + class FakeRunsGraph: + # fakeRunsGraph + def __init__(self, protocol): + self._nodesDict = { + "PROJECT": object(), + "10": FakeProtocolNode(protocol), + } + + class FakeCurrentProjectForSync: + # fakeCurrentProjectForSync + def __init__(self, protocol): + self.protocol = protocol + + def getRunsGraph(self, refresh=False, checkPids=False): + return FakeRunsGraph(self.protocol) + + class FakeProtocolForSync: + # fakeProtocolForSync + def getObjId(self): + return 10 + + def iterInputAttributes(self): + return [] + + class FakeSyncMapper: + # fakeSyncMapper + def __init__(self): + self.savedProtocolContexts = [] + self.deletedProtocolIds = None + self.savedEdges = None + self.savedInputRefs = None + + def saveProtocol(self, protocolContext): + self.savedProtocolContexts.append(protocolContext) + return 500 + + def deleteProjectProtocolsNotInProtocolIds(self, projectId, protocolIds): + self.deletedProtocolIds = { + "projectId": projectId, + "protocolIds": protocolIds, + } + + def replaceProjectProtocolDependencies(self, projectId, edges): + self.savedEdges = { + "projectId": projectId, + "edges": edges, + } + return len(edges) + + def replaceProjectProtocolInputRefs(self, projectId, inputRefs): + self.savedInputRefs = { + "projectId": projectId, + "inputRefs": inputRefs, + } + return len(inputRefs) + + protocol = FakeProtocolForSync() + service.currentProject = FakeCurrentProjectForSync(protocol) + + monkeypatch.setattr( + service, + "_buildProtocolContext", + lambda projectId, protocol: { + "projectId": projectId, + "protocolId": protocol.getObjId(), + }, + ) + monkeypatch.setattr( + service, + "_shouldRegisterProtocolOutputs", + lambda protocol: True, + ) + monkeypatch.setattr( + service, + "registerOutput", + lambda projectId, protocol, mapper, returnReport=False: { + "persisted": [ + { + "mapperKind": "flat_set", + "outputName": "outputParticles", + }, + { + "mapperKind": "tree", + "outputName": "outputVolume", + }, + ], + "skipped": [ + { + "outputName": "unsupportedOutput", + "outputClassName": "UnsupportedOutput", + "reason": "unsupported_output_type", + } + ], + "errors": [ + { + "outputName": "badOutput", + "outputClassName": "SetOfBad", + "error": "boom", + } + ], + }, + ) + + mapper = FakeSyncMapper() + + result = service.syncProjectProtocolsAndDependencies( + mapper=mapper, + projectId=1, + refresh=True, + checkPid=True, + ) + + assert result == { + "protocols": 1, + "dependencies": 0, + "inputRefs": 0, + "outputs": 2, + "outputsByKind": { + "flat_set": 1, + "tree": 1, + }, + "outputErrors": [ + { + "protocolId": "10", + "outputName": "unsupportedOutput", + "outputClassName": "UnsupportedOutput", + "reason": "unsupported_output_type", + }, + { + "protocolId": "10", + "outputName": "badOutput", + "outputClassName": "SetOfBad", + "error": "boom", + }, + ], + } + + assert mapper.savedProtocolContexts == [ + { + "projectId": 1, + "protocolId": 10, + } + ] + assert mapper.deletedProtocolIds == { + "projectId": 1, + "protocolIds": ["10"], + } + + def test_CastParamValueSupportsEnumLookup(projectServiceModule, service, monkeypatch): monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) From 6cffca555966b5d14d0268b209933b00010954a6 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 11:43:33 +0200 Subject: [PATCH 124/278] Compare declared and persisted outputs during sync --- app/backend/api/services/project_service.py | 94 +++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 252f202f..f039d5fc 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -1229,6 +1229,8 @@ def syncProjectProtocolsAndDependencies( outputSyncResults: List[Dict[str, Any]] = [] outputSyncErrors: List[Dict[str, Any]] = [] + outputSyncDeclared: List[Dict[str, Any]] = [] + outputSyncMissing: List[Dict[str, Any]] = [] # 1) Save all protocol nodes that are currently present in the real Scipion graph for nodeId, nodeObj in nodesDict.items(): @@ -1256,6 +1258,27 @@ def syncProjectProtocolsAndDependencies( ) outputSyncResults.extend(outputReport.get("persisted") or []) + declaredOutputs = outputReport.get("declared") or [] + persistedOutputs = outputReport.get("persisted") or [] + skippedOutputs = outputReport.get("skipped") or [] + erroredOutputs = outputReport.get("errors") or [] + + for declaredOutput in declaredOutputs: + outputSyncDeclared.append({ + "protocolId": nodeIdText, + "outputName": declaredOutput.get("outputName"), + "outputClassName": declaredOutput.get("outputClassName"), + }) + + outputSyncMissing.extend( + self._buildMissingOutputSyncItems( + protocolId=nodeIdText, + declaredOutputs=declaredOutputs, + persistedOutputs=persistedOutputs, + skippedOutputs=skippedOutputs, + outputErrors=erroredOutputs, + ) + ) for skippedOutput in outputReport.get("skipped") or []: outputSyncErrors.append({ @@ -1339,8 +1362,11 @@ def syncProjectProtocolsAndDependencies( "protocols": len(protocolDbIdByScipionId), "dependencies": int(savedEdges), "inputRefs": int(savedInputRefs), + "outputsDeclared": len(outputSyncDeclared), "outputs": len(outputSyncResults), + "outputsMissing": len(outputSyncMissing), "outputsByKind": outputResultsByKind, + "outputMissing": outputSyncMissing, "outputErrors": outputSyncErrors, } @@ -3796,6 +3822,60 @@ def listProjectThumbnailItems( inlineImages=inlineImages, ) + def _buildMissingOutputSyncItems( + self, + protocolId: Union[int, str], + declaredOutputs: List[Dict[str, Any]], + persistedOutputs: List[Dict[str, Any]], + skippedOutputs: List[Dict[str, Any]], + outputErrors: List[Dict[str, Any]], + ) -> List[Dict[str, Any]]: + persistedByName = { + str(item.get("outputName") or ""): item + for item in persistedOutputs or [] + if item.get("outputName") is not None + } + + skippedByName = { + str(item.get("outputName") or ""): item + for item in skippedOutputs or [] + if item.get("outputName") is not None + } + + errorsByName = { + str(item.get("outputName") or ""): item + for item in outputErrors or [] + if item.get("outputName") is not None + } + + missingOutputs: List[Dict[str, Any]] = [] + + for declaredOutput in declaredOutputs or []: + outputName = str(declaredOutput.get("outputName") or "") + if not outputName or outputName in persistedByName: + continue + + missingItem = { + "protocolId": str(protocolId), + "outputName": outputName, + "outputClassName": declaredOutput.get("outputClassName"), + } + + skippedOutput = skippedByName.get(outputName) + outputError = errorsByName.get(outputName) + + if skippedOutput is not None: + missingItem["reason"] = skippedOutput.get("reason") or "skipped" + elif outputError is not None: + missingItem["reason"] = "persistence_error" + missingItem["error"] = outputError.get("error") + else: + missingItem["reason"] = "not_persisted" + + missingOutputs.append(missingItem) + + return missingOutputs + def registerOutput( self, projectId: int, @@ -3821,6 +3901,7 @@ def registerOutput( results: List[Dict[str, Any]] = [] skippedOutputs: List[Dict[str, Any]] = [] outputErrors: List[Dict[str, Any]] = [] + declaredOutputs: List[Dict[str, Any]] = [] closeMapper = mapper is None if mapper is None: @@ -3837,7 +3918,19 @@ def registerOutput( setMapper = ScipionSetPostgresqlMapper(mapper.db) for outputName, outputObj in protocol.iterOutputAttributes(): + outputClassName = self._getOutputClassName(outputObj) + + declaredOutputs.append({ + "outputName": outputName, + "outputClassName": outputClassName, + }) + if outputObj is None: + skippedOutputs.append({ + "outputName": outputName, + "outputClassName": outputClassName, + "reason": "empty_output", + }) continue try: @@ -3899,6 +3992,7 @@ def registerOutput( if returnReport: return { + "declared": declaredOutputs, "persisted": results, "skipped": skippedOutputs, "errors": outputErrors, From 11bdea134f6c96c820a1df07084c6a7b9db01762 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 11:51:50 +0200 Subject: [PATCH 125/278] Test declared output sync comparison --- .../test_project_service_protocols.py | 146 +++++++++++++++++- 1 file changed, 145 insertions(+), 1 deletion(-) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index 70188206..ff70715c 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -272,6 +272,76 @@ def assertSuccessEnvelope(result): assert result["errors"] == [] +def test_BuildMissingOutputSyncItemsClassifiesMissingOutputs(service): + declaredOutputs = [ + { + "outputName": "persistedOutput", + "outputClassName": "SetOfParticles", + }, + { + "outputName": "skippedOutput", + "outputClassName": "UnsupportedOutput", + }, + { + "outputName": "erroredOutput", + "outputClassName": "SetOfBad", + }, + { + "outputName": "missingOutput", + "outputClassName": "SetOfMissing", + }, + ] + persistedOutputs = [ + { + "outputName": "persistedOutput", + "outputClassName": "SetOfParticles", + } + ] + skippedOutputs = [ + { + "outputName": "skippedOutput", + "outputClassName": "UnsupportedOutput", + "reason": "unsupported_output_type", + } + ] + outputErrors = [ + { + "outputName": "erroredOutput", + "outputClassName": "SetOfBad", + "error": "broken output", + } + ] + + result = service._buildMissingOutputSyncItems( + protocolId=10, + declaredOutputs=declaredOutputs, + persistedOutputs=persistedOutputs, + skippedOutputs=skippedOutputs, + outputErrors=outputErrors, + ) + + assert result == [ + { + "protocolId": "10", + "outputName": "skippedOutput", + "outputClassName": "UnsupportedOutput", + "reason": "unsupported_output_type", + }, + { + "protocolId": "10", + "outputName": "erroredOutput", + "outputClassName": "SetOfBad", + "reason": "persistence_error", + "error": "broken output", + }, + { + "protocolId": "10", + "outputName": "missingOutput", + "outputClassName": "SetOfMissing", + "reason": "not_persisted", + }, + ] + def test_RegisterOutputReturnsPersistenceReport( projectServiceModule, service, @@ -315,6 +385,7 @@ def iterOutputAttributes(self): ("outputParticles", FakeSetOutput()), ("outputVolume", FakeObjectOutput()), ("badSet", FakeBadSetOutput()), + ("emptyOutput", None), ("unsupportedOutput", FakeUnsupportedOutput()), ] @@ -378,6 +449,29 @@ def storeObjectTree( returnReport=True, ) + assert report["declared"] == [ + { + "outputName": "outputParticles", + "outputClassName": "SetOfParticles", + }, + { + "outputName": "outputVolume", + "outputClassName": "Volume", + }, + { + "outputName": "badSet", + "outputClassName": "SetOfBadThings", + }, + { + "outputName": "emptyOutput", + "outputClassName": "", + }, + { + "outputName": "unsupportedOutput", + "outputClassName": "FakeUnsupportedOutput", + }, + ] + assert report["persisted"] == [ { "projectId": 1, @@ -399,11 +493,16 @@ def storeObjectTree( ] assert report["skipped"] == [ + { + "outputName": "emptyOutput", + "outputClassName": "", + "reason": "empty_output", + }, { "outputName": "unsupportedOutput", "outputClassName": "FakeUnsupportedOutput", "reason": "unsupported_output_type", - } + }, ] assert report["errors"] == [ @@ -509,6 +608,28 @@ def replaceProjectProtocolInputRefs(self, projectId, inputRefs): service, "registerOutput", lambda projectId, protocol, mapper, returnReport=False: { + "declared": [ + { + "outputName": "outputParticles", + "outputClassName": "SetOfParticles", + }, + { + "outputName": "outputVolume", + "outputClassName": "Volume", + }, + { + "outputName": "unsupportedOutput", + "outputClassName": "UnsupportedOutput", + }, + { + "outputName": "badOutput", + "outputClassName": "SetOfBad", + }, + { + "outputName": "orphanOutput", + "outputClassName": "SetOfOrphan", + }, + ], "persisted": [ { "mapperKind": "flat_set", @@ -549,11 +670,34 @@ def replaceProjectProtocolInputRefs(self, projectId, inputRefs): "protocols": 1, "dependencies": 0, "inputRefs": 0, + "outputsDeclared": 5, "outputs": 2, + "outputsMissing": 3, "outputsByKind": { "flat_set": 1, "tree": 1, }, + "outputMissing": [ + { + "protocolId": "10", + "outputName": "unsupportedOutput", + "outputClassName": "UnsupportedOutput", + "reason": "unsupported_output_type", + }, + { + "protocolId": "10", + "outputName": "badOutput", + "outputClassName": "SetOfBad", + "reason": "persistence_error", + "error": "boom", + }, + { + "protocolId": "10", + "outputName": "orphanOutput", + "outputClassName": "SetOfOrphan", + "reason": "not_persisted", + }, + ], "outputErrors": [ { "protocolId": "10", From cddc32f5a4c7e4c1847da6b239a06e79896edbae Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 11:58:31 +0200 Subject: [PATCH 126/278] Use PostgreSQL-only integrated analyze context --- app/backend/api/services/project_service.py | 29 ++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index f039d5fc..7ed433c8 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -3285,12 +3285,19 @@ def _getPostgresqlIntegratedAnalyzeContextIfAvailable( try: from app.backend.viewers.postgresql_integrated_context_reader import PostgresqlIntegratedContextReader + readerProtocolId = self._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + reader = PostgresqlIntegratedContextReader( db=mapper.db, projectId=projectId, - protocolId=protocolId, + protocolId=readerProtocolId, outputName=outputName, ) + return reader.getContext() except Exception: logger.debug( @@ -3309,11 +3316,6 @@ def getIntegratedAnalyzeContextService( outputName: str, mapper=None, ) -> Dict[str, Any]: - if self.currentProject is None: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="No current project loaded", - ) pgContext = self._getPostgresqlIntegratedAnalyzeContextIfAvailable( mapper=mapper, @@ -3325,6 +3327,21 @@ def getIntegratedAnalyzeContextService( if pgContext is not None: return pgContext + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Integrated Analyze Context", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="context_not_available", + ) + + if self.currentProject is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="No current project loaded", + ) + try: protocol = self.currentProject.getProtocol(int(protocolId)) except Exception as e: From 22071f1bcd1d4e2e8d1fb084fa55b0cf2fc72fb8 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 12:04:32 +0200 Subject: [PATCH 127/278] Test PostgreSQL-only integrated analyze context --- .../test_project_service_protocols.py | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index ff70715c..d53625f1 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -726,6 +726,210 @@ def replaceProjectProtocolInputRefs(self, projectId, inputRefs): } +def test_GetPostgresqlIntegratedAnalyzeContextUsesResolvedProtocolId( + service, + monkeypatch, +): + readerCalls = [] + resolverCalls = [] + + class FakeDb: + pass + + class FakeMapper: + def __init__(self): + self.db = FakeDb() + + class FakePostgresqlIntegratedContextReader: + def __init__(self, db, projectId, protocolId, outputName): + readerCalls.append({ + "db": db, + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + }) + + def getContext(self): + return { + "root": { + "projectId": 1, + "protocolId": 321, + "outputName": "outputTiltSeries", + } + } + + readerModule = importlib.import_module( + "app.backend.viewers.postgresql_integrated_context_reader" + ) + + monkeypatch.setattr( + readerModule, + "PostgresqlIntegratedContextReader", + FakePostgresqlIntegratedContextReader, + ) + + def fakeResolvePostgresqlReaderProtocolId(mapper, projectId, protocolId): + resolverCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + }) + return 321 + + monkeypatch.setattr( + service, + "_resolvePostgresqlReaderProtocolId", + fakeResolvePostgresqlReaderProtocolId, + ) + + mapper = FakeMapper() + + result = service._getPostgresqlIntegratedAnalyzeContextIfAvailable( + mapper=mapper, + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + ) + + assert result == { + "root": { + "projectId": 1, + "protocolId": 321, + "outputName": "outputTiltSeries", + } + } + + assert resolverCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 10, + } + ] + + assert readerCalls == [ + { + "db": mapper.db, + "projectId": 1, + "protocolId": 321, + "outputName": "outputTiltSeries", + } + ] + + +def test_GetIntegratedAnalyzeContextRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + class FakeMapper: + pass + + class RuntimeShouldNotBeUsed: + def getProtocol(self, protocolId): + raise AssertionError("Runtime fallback should not be used") + + service.currentProject = RuntimeShouldNotBeUsed() + + monkeypatch.setattr( + service, + "_getPostgresqlIntegratedAnalyzeContextIfAvailable", + lambda mapper, projectId, protocolId, outputName: None, + ) + + with pytest.raises(HTTPException) as exc: + service.getIntegratedAnalyzeContextService( + mapper=FakeMapper(), + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == ( + "Integrated Analyze Context output is not available in PostgreSQL metadata: " + "context_not_available" + ) + + +def test_GetIntegratedAnalyzeContextKeepsLegacyRuntimeFallbackWithoutMapper( + service, + monkeypatch, +): + class FakeTiltSeriesOutput: + def getClassName(self): + return "SetOfTiltSeries" + + def getObjId(self): + return 22 + + def getSize(self): + return 1 + + def getTSIds(self): + return ["TS_001"] + + def getSamplingRate(self): + return 1.5 + + def getDimensions(self): + return (100, 100, 40) + + def iterItems(self): + return iter([]) + + class FakeProtocol: + def __init__(self): + self.outputTiltSeries = FakeTiltSeriesOutput() + + def iterInputAttributes(self): + return [] + + def iterOutputAttributes(self): + return [("outputTiltSeries", self.outputTiltSeries)] + + class FakeCurrentProjectForIntegratedContext: + def __init__(self): + self.protocol = FakeProtocol() + + def getProtocol(self, protocolId): + assert protocolId == 10 + return self.protocol + + service.currentProject = FakeCurrentProjectForIntegratedContext() + + monkeypatch.setattr( + service, + "_getPostgresqlIntegratedAnalyzeContextIfAvailable", + lambda mapper, projectId, protocolId, outputName: None, + ) + + result = service.getIntegratedAnalyzeContextService( + mapper=None, + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + ) + + assert result["root"] == { + "projectId": 1, + "protocolId": 10, + "outputName": "outputTiltSeries", + "outputClass": "SetOfTiltSeries", + } + + assert result["links"]["tiltSeries"] == { + "protocolId": 10, + "outputName": "outputTiltSeries", + "itemId": 22, + "label": "outputTiltSeries", + "status": "available", + } + + assert result["summaries"]["tiltSeries"]["objectClass"] == "SetOfTiltSeries" + assert result["summaries"]["tiltSeries"]["objectId"] == 22 + assert result["summaries"]["tiltSeries"]["size"] == 1 + + def test_CastParamValueSupportsEnumLookup(projectServiceModule, service, monkeypatch): monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) From 889f56866ba57cec6f817ff03840b40874443eae Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 12:24:12 +0200 Subject: [PATCH 128/278] Resolve protocol ids for thumbnail rendering --- app/backend/api/routers/project_router.py | 8 +++++++ app/backend/api/services/project_service.py | 24 ++++++++++++++------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 3421258e..273ba00a 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -3481,6 +3481,8 @@ def buildThumbnail(): outputName=outputName, force=False, size=size, + mapper=mapper, + projectId=projectId, ) return service.buildProtocolThumbnail( @@ -3543,6 +3545,8 @@ def buildThumbnail(): outputName=outputName, force=True, size=size, + mapper=mapper, + projectId=projectId, ) return service.buildProtocolThumbnail( @@ -3707,6 +3711,8 @@ def getProtocolOutputThumbnail( outputName=outputName, force=False, size=size, + mapper=mapper, + projectId=projectId, ) thumbPath = result.get("absolutePath") @@ -3830,6 +3836,8 @@ def getProtocolOutputThumbnailsBatch( outputName=outputName, force=False, size=payload.size, + mapper=mapper, + projectId=projectId, ) except Exception as exc: logger.debug( diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 7ed433c8..fcba6db2 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -3807,14 +3807,22 @@ def addSetRelations(kind: str, obj: Any) -> None: def buildProtocolOutputThumbnail( self, - protocolId: int, + protocolId: Union[int, str], outputName: str, force: bool = False, size: int = 320, + mapper=None, + projectId: Optional[int] = None, ) -> Dict[str, Any]: - service = ThumbnailService(self.currentProject) - return service.buildProtocolOutputThumbnail( + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, protocolId=protocolId, + ) + + thumbnailService = ThumbnailService(self.currentProject) + return thumbnailService.buildProtocolOutputThumbnail( + protocolId=scipionProtocolId, outputName=outputName, force=force, size=size, @@ -6937,22 +6945,22 @@ def outputPreview( def buildProtocolThumbnail( self, - protocolId: int, + protocolId: Union[int, str], force: bool = False, size: int = 320, outputName: Optional[str] = None, mapper=None, projectId: Optional[int] = None, ) -> Dict[str, Any]: - runtimeProtocolId = self._resolveScipionProtocolId( + scipionProtocolId = self._resolveScipionProtocolId( mapper=mapper, projectId=projectId, protocolId=protocolId, ) - service = ThumbnailService(self.currentProject) - return service.buildProtocolThumbnail( - protocolId=runtimeProtocolId, + thumbnailService = ThumbnailService(self.currentProject) + return thumbnailService.buildProtocolThumbnail( + protocolId=scipionProtocolId, force=force, size=size, outputName=outputName, From 0eccf4eb14cc71a3bcc4d14fda2c347369691780 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 12:31:52 +0200 Subject: [PATCH 129/278] Resolve protocol ids in batch output thumbnails --- app/backend/api/routers/project_router.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 273ba00a..ae04e849 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -3787,25 +3787,25 @@ def getProtocolOutputThumbnailsBatch( seen = set() for requestedOutput in requestedOutputs: - protocolId = int(requestedOutput.protocolId) + requestedProtocolId = int(requestedOutput.protocolId) outputName = str(requestedOutput.outputName or "").strip() if not outputName: continue - requestKey = (protocolId, outputName) + requestKey = (requestedProtocolId, outputName) if requestKey in seen: continue seen.add(requestKey) item = { - "protocolId": protocolId, + "protocolId": requestedProtocolId, "outputName": outputName, "outputClassName": None, "exists": False, "cached": False, "thumbnailUrl": ( - f"/projects/{int(projectId)}/protocols/{protocolId}" + f"/projects/{int(projectId)}/protocols/{requestedProtocolId}" f"/outputs/{outputName}/thumbnail" ), "thumbnailDataUrl": None, @@ -3813,17 +3813,17 @@ def getProtocolOutputThumbnailsBatch( } try: - protocol = service.currentProject.getProtocol(protocolId) + scipionProtocolId = service._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=requestedProtocolId, + ) + protocol = service.currentProject.getProtocol(int(scipionProtocolId)) except Exception: item["error"] = "Protocol not found" items.append(item) continue - if not hasattr(protocol, outputName): - item["error"] = "Output not found" - items.append(item) - continue - try: outputObject = getattr(protocol, outputName) item["outputClassName"] = outputObject.__class__.__name__ @@ -3832,7 +3832,7 @@ def getProtocolOutputThumbnailsBatch( try: result = service.buildProtocolOutputThumbnail( - protocolId=protocolId, + protocolId=requestedProtocolId, outputName=outputName, force=False, size=payload.size, From c37b55cca4c873033bc987e5a8b6c36c193a1ceb Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 12:33:59 +0200 Subject: [PATCH 130/278] Fix batch thumbnail logging ids --- app/backend/api/routers/project_router.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index ae04e849..785e6e5d 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -3843,7 +3843,7 @@ def getProtocolOutputThumbnailsBatch( logger.debug( "Failed building batch protocol output thumbnail. projectId=%s protocolId=%s outputName=%s", projectId, - protocolId, + requestedProtocolId, outputName, exc_info=True, ) @@ -3870,7 +3870,7 @@ def getProtocolOutputThumbnailsBatch( logger.debug( "Could not inline batch thumbnail image. projectId=%s protocolId=%s outputName=%s", projectId, - protocolId, + requestedProtocolId, outputName, exc_info=True, ) From ce1cee9a2b7b2e965254babe295e659b762529c7 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 12:49:38 +0200 Subject: [PATCH 131/278] Test PostgreSQL id resolution for thumbnails --- app/backend/api/services/project_service.py | 35 +-- .../test_project_service_io_thumbnails.py | 251 ++++++++++++++++++ 2 files changed, 263 insertions(+), 23 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index fcba6db2..04b07f42 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6897,28 +6897,31 @@ def previewProtocolImageFile( def outputPreview( self, - protocolId: int, - outputName: str, - requestHeaders: dict = None, - colormap: str = None, + protocolId, + outputName, + requestHeaders=None, + colormap=None, mapper=None, - projectId: Optional[int] = None, + projectId=None, ): - protocol = self._getScipionProtocolForRuntime( + scipionProtocolId = self._resolveScipionProtocolId( mapper=mapper, projectId=projectId, protocolId=protocolId, ) + protocol = self._getScipionProtocolByRuntimeId(scipionProtocolId) + if not hasattr(protocol, outputName): raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail=f"Output not found: {outputName}", + detail=f"Output '{outputName}' not found in protocol", ) output = getattr(protocol, outputName) + outputPath = output.getFileName() - outputPreview = OutputsPreview( + preview = OutputsPreview( self.currentProject, protocol, output, @@ -6926,22 +6929,8 @@ def outputPreview( colormapOverride=colormap, ) - getFileName = getattr(output, "getFileName", None) - if not callable(getFileName): - return outputPreview.renderOutputFallbackPreview() - - try: - outputPath = getFileName() - except Exception: - return outputPreview.renderOutputFallbackPreview() - - if not outputPath: - return outputPreview.renderOutputFallbackPreview() - - runtimeProtocolId = getattr(protocol, "getObjId", lambda: protocolId)() - objMgr = self._createObjectManager() - return outputPreview.preview(runtimeProtocolId, outputPath, objMgr) + return preview.preview(scipionProtocolId, outputPath, objMgr) def buildProtocolThumbnail( self, diff --git a/tests/unit/backend/api/services/test_project_service_io_thumbnails.py b/tests/unit/backend/api/services/test_project_service_io_thumbnails.py index 5eff0b32..ef836849 100644 --- a/tests/unit/backend/api/services/test_project_service_io_thumbnails.py +++ b/tests/unit/backend/api/services/test_project_service_io_thumbnails.py @@ -48,6 +48,33 @@ def __init__(self, protocolId, outputName=None, output=None): if outputName is not None: setattr(self, outputName, output) +class FakeDb: + def __init__(self, runtimeProtocolIdByDbId=None): + self.runtimeProtocolIdByDbId = runtimeProtocolIdByDbId or {} + self.fetchCalls = [] + + def fetchOne(self, query, params): + self.fetchCalls.append({ + "query": query, + "params": params, + }) + + if len(params) < 3: + return None + + protocolDbId = params[1] + runtimeProtocolId = self.runtimeProtocolIdByDbId.get(int(protocolDbId)) + if runtimeProtocolId is None: + return None + + return { + "protocolId": runtimeProtocolId, + } + + +class FakeMapper: + def __init__(self, runtimeProtocolIdByDbId=None): + self.db = FakeDb(runtimeProtocolIdByDbId=runtimeProtocolIdByDbId) class FakeCurrentProject: # fakeCurrentProject @@ -212,6 +239,49 @@ def test_OutputPreviewDelegatesToOutputsPreview(projectServiceModule, service, m } +def test_OutputPreviewResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, + tmp_path, +): + FakeOutputsPreview.instances = [] + + outputFile = tmp_path / "output.sqlite" + outputFile.write_text("placeholder", encoding="utf-8") + + output = FakeOutput(str(outputFile)) + protocol = FakeProtocol(protocolId=10, outputName="outputMetadata", output=output) + service.currentProject = FakeCurrentProject(protocols={10: protocol}) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + monkeypatch.setattr(projectServiceModule, "OutputsPreview", FakeOutputsPreview) + monkeypatch.setattr(service, "_createObjectManager", lambda: {"manager": "fresh"}) + + result = service.outputPreview( + protocolId=500, + outputName="outputMetadata", + requestHeaders={"x-preview-colormap": "viridis"}, + colormap="plasma", + mapper=mapper, + projectId=1, + ) + + assert result == { + "preview": True, + "protocolId": 10, + "outputPath": str(outputFile), + "colormap": "plasma", + } + assert FakeOutputsPreview.instances[0].lastPreviewCall == { + "protocolId": 10, + "outputPath": str(outputFile), + "objMgr": {"manager": "fresh"}, + } + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + def test_BuildProtocolThumbnailDelegatesToThumbnailService(projectServiceModule, service, monkeypatch): FakeThumbnailService.instances = [] monkeypatch.setattr(projectServiceModule, "ThumbnailService", FakeThumbnailService) @@ -230,6 +300,42 @@ def test_BuildProtocolThumbnailDelegatesToThumbnailService(projectServiceModule, ] +def test_BuildProtocolThumbnailResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, +): + FakeThumbnailService.instances = [] + monkeypatch.setattr(projectServiceModule, "ThumbnailService", FakeThumbnailService) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.buildProtocolThumbnail( + protocolId=500, + force=True, + size=400, + outputName="outputA", + mapper=mapper, + projectId=1, + ) + + assert result == { + "kind": "protocol", + "protocolId": 10, + "outputName": "outputA", + } + assert FakeThumbnailService.instances[0].calls == [ + { + "method": "buildProtocolThumbnail", + "protocolId": 10, + "force": True, + "size": 400, + "outputName": "outputA", + } + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + def test_BuildProjectThumbnailDelegatesToThumbnailService(projectServiceModule, service, monkeypatch): FakeThumbnailService.instances = [] monkeypatch.setattr(projectServiceModule, "ThumbnailService", FakeThumbnailService) @@ -265,6 +371,42 @@ def test_BuildProtocolOutputThumbnailDelegatesToThumbnailService(projectServiceM ] +def test_BuildProtocolOutputThumbnailResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, +): + FakeThumbnailService.instances = [] + monkeypatch.setattr(projectServiceModule, "ThumbnailService", FakeThumbnailService) + + mapper = FakeMapper(runtimeProtocolIdByDbId={501: 11}) + + result = service.buildProtocolOutputThumbnail( + protocolId=501, + outputName="outputVol", + force=False, + size=256, + mapper=mapper, + projectId=1, + ) + + assert result == { + "kind": "output", + "protocolId": 11, + "outputName": "outputVol", + } + assert FakeThumbnailService.instances[0].calls == [ + { + "method": "buildProtocolOutputThumbnail", + "protocolId": 11, + "outputName": "outputVol", + "force": False, + "size": 256, + } + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 501, "501") + + def test_ListProjectThumbnailItemsDelegatesToThumbnailService(projectServiceModule, service, monkeypatch): FakeThumbnailService.instances = [] monkeypatch.setattr(projectServiceModule, "ThumbnailService", FakeThumbnailService) @@ -429,3 +571,112 @@ def test_WriteRemoteFileServiceWritesContent(service, monkeypatch, tmp_path): "size": targetPath.stat().st_size, "mimeType": "application/json", } + + +def test_GetProtocolOutputThumbnailsBatchResolvesPostgresqlProtocolId( + service, + monkeypatch, + tmp_path, +): + projectRouterModule = importlib.import_module("app.backend.api.routers.project_router") + + outputFile = tmp_path / "output.sqlite" + outputFile.write_text("placeholder", encoding="utf-8") + + thumbnailFile = tmp_path / "thumbnail.png" + thumbnailFile.write_bytes(b"fake-thumbnail") + + output = FakeOutput(str(outputFile)) + protocol = FakeProtocol(protocolId=10, outputName="outputVol", output=output) + service.currentProject = FakeCurrentProject(protocols={10: protocol}) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + monkeypatch.setattr( + service, + "getProjectDbRow", + lambda mapper, projectId, currentUser: {"id": projectId, "name": str(tmp_path)}, + ) + monkeypatch.setattr( + service, + "loadProjectForThumbnails", + lambda dbProj: service.currentProject, + ) + + buildCalls = [] + + def fakeBuildProtocolOutputThumbnail( + protocolId, + outputName, + force=False, + size=320, + mapper=None, + projectId=None, + ): + buildCalls.append({ + "protocolId": protocolId, + "outputName": outputName, + "force": force, + "size": size, + "mapper": mapper, + "projectId": projectId, + }) + return { + "absolutePath": str(thumbnailFile), + "exists": True, + "cached": False, + } + + monkeypatch.setattr( + service, + "buildProtocolOutputThumbnail", + fakeBuildProtocolOutputThumbnail, + ) + + payload = FakePayload( + outputs=[ + FakePayload(protocolId=500, outputName="outputVol"), + ], + size=256, + inlineImages=False, + ) + + response = projectRouterModule.getProtocolOutputThumbnailsBatch( + projectId=1, + payload=payload, + currentUser={"id": 1}, + mapper=mapper, + service=service, + ) + + payloadJson = json.loads(response.body.decode("utf-8")) + + assert payloadJson == { + "projectId": 1, + "size": 256, + "items": [ + { + "protocolId": 500, + "outputName": "outputVol", + "outputClassName": "FakeOutput", + "exists": True, + "cached": False, + "thumbnailUrl": "/projects/1/protocols/500/outputs/outputVol/thumbnail", + "thumbnailDataUrl": None, + "error": None, + } + ], + } + + assert buildCalls == [ + { + "protocolId": 500, + "outputName": "outputVol", + "force": False, + "size": 256, + "mapper": mapper, + "projectId": 1, + } + ] + + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") \ No newline at end of file From b253f821058ba5f91038380f0c779c5aa352383c Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 13:00:58 +0200 Subject: [PATCH 132/278] Resolve runtime ids for protocol logs --- app/backend/api/services/project_service.py | 16 +- .../services/test_project_service_logs_fs.py | 143 ++++++++++++++++++ 2 files changed, 154 insertions(+), 5 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 04b07f42..3dbe4b92 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -5398,12 +5398,14 @@ def listProtocolLogChannelsService( """ Return available log channels for a protocol, including paths and basic file stats. """ - protocol = self._getScipionProtocolForRuntime( + scipionProtocolId = self._resolveScipionProtocolId( mapper=mapper, projectId=projectId, protocolId=protocolId, ) + protocol = self._getScipionProtocolByRuntimeId(scipionProtocolId) + # Resolve log paths from Scipion protocol object stdoutPath = protocol.getStdoutLog() if hasattr(protocol, "getStdoutLog") else None stderrPath = protocol.getStderrLog() if hasattr(protocol, "getStderrLog") else None @@ -5445,7 +5447,7 @@ def buildChannel(channelId: str, label: str, order, filePath: Optional[str]) -> # Keep a consistent list but allow UI to filter by exists==True return { "projectId": projectId, - "protocolId": int(protocolId), + "protocolId": int(scipionProtocolId), "channels": channels, } @@ -5461,12 +5463,14 @@ def pollProtocolLogsService( """ Incrementally read protocol logs from the given offsets, applying maxBytes and maxLines limits. """ - protocol = self._getScipionProtocolForRuntime( + scipionProtocolId = self._resolveScipionProtocolId( mapper=mapper, projectId=projectId, protocolId=protocolId, ) + protocol = self._getScipionProtocolByRuntimeId(scipionProtocolId) + stdoutPath = protocol.getStdoutLog() if hasattr(protocol, "getStdoutLog") else None stderrPath = protocol.getStderrLog() if hasattr(protocol, "getStderrLog") else None schedulePath = protocol.getScheduleLog() if hasattr(protocol, "getScheduleLog") else None @@ -5616,7 +5620,7 @@ def readChunk(filePath: Optional[str], startOffset: int) -> Dict[str, Any]: return { "projectId": projectId, - "protocolId": int(protocolId), + "protocolId": int(scipionProtocolId), "channels": { "stdout": stdoutRes, "stderr": stderrRes, @@ -5633,11 +5637,13 @@ def getProtocolLogs( scheduleOffset: int = 0, mapper=None, ): - protocol = self._getScipionProtocolForRuntime( + scipionProtocolId = self._resolveScipionProtocolId( mapper=mapper, projectId=projectId, protocolId=protocolId, ) + + protocol = self._getScipionProtocolByRuntimeId(scipionProtocolId) logPath = protocol.getStdoutLog() errLogPath = protocol.getStderrLog() scheduleLogPath = protocol.getScheduleLog() diff --git a/tests/unit/backend/api/services/test_project_service_logs_fs.py b/tests/unit/backend/api/services/test_project_service_logs_fs.py index 95f1d19f..b6acf838 100644 --- a/tests/unit/backend/api/services/test_project_service_logs_fs.py +++ b/tests/unit/backend/api/services/test_project_service_logs_fs.py @@ -51,6 +51,34 @@ def getStderrLog(self): def getScheduleLog(self): return self._scheduleLog +class FakeDb: + def __init__(self, runtimeProtocolIdByDbId=None): + self.runtimeProtocolIdByDbId = runtimeProtocolIdByDbId or {} + self.fetchCalls = [] + + def fetchOne(self, query, params): + self.fetchCalls.append({ + "query": query, + "params": params, + }) + + if len(params) < 3: + return None + + protocolDbId = params[1] + runtimeProtocolId = self.runtimeProtocolIdByDbId.get(int(protocolDbId)) + if runtimeProtocolId is None: + return None + + return { + "protocolId": runtimeProtocolId, + } + + +class FakeMapper: + def __init__(self, runtimeProtocolIdByDbId=None): + self.db = FakeDb(runtimeProtocolIdByDbId=runtimeProtocolIdByDbId) + class FakeCurrentProject: # fakeCurrentProject @@ -255,6 +283,121 @@ def test_GetProtocolLogsReadsAllChannelsFromOffsets(service, tmp_path): } +def test_ListProtocolLogChannelsServiceResolvesPostgresqlProtocolId(service, tmp_path): + stdoutLog = tmp_path / "stdout.log" + stderrLog = tmp_path / "stderr.log" + scheduleLog = tmp_path / "schedule.log" + + stdoutLog.write_text("hello\n", encoding="utf-8") + stderrLog.write_text("error\n", encoding="utf-8") + scheduleLog.write_text("schedule\n", encoding="utf-8") + + service.currentProject.protocols[10] = FakeProtocol( + stdoutLog=str(stdoutLog), + stderrLog=str(stderrLog), + scheduleLog=str(scheduleLog), + ) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.listProtocolLogChannelsService( + projectId=1, + protocolId=500, + mapper=mapper, + ) + + assert result["projectId"] == 1 + assert result["protocolId"] == 10 + assert result["channels"] == [ + {"id": "stdout", "label": "Output", "order": 1}, + {"id": "stderr", "label": "Errors", "order": 2}, + {"id": "schedule", "label": "Schedule", "order": 3}, + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_PollProtocolLogsServiceResolvesPostgresqlProtocolId(service, tmp_path): + stdoutLog = tmp_path / "stdout.log" + scheduleLog = tmp_path / "schedule.log" + + stdoutLog.write_text("line1\nline2\nline3\n", encoding="utf-8") + scheduleLog.write_text("sched1\nsched2\n", encoding="utf-8") + + service.currentProject.protocols[10] = FakeProtocol( + stdoutLog=str(stdoutLog), + stderrLog=str(tmp_path / "missing-stderr.log"), + scheduleLog=str(scheduleLog), + ) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.pollProtocolLogsService( + projectId=1, + protocolId=500, + offsets={ + "stdoutLog": 6, + "err": 0, + "schedule": 7, + }, + maxBytes=64, + maxLines=1, + mapper=mapper, + ) + + assert result["projectId"] == 1 + assert result["protocolId"] == 10 + assert result["channels"]["stdout"] == { + "content": "line2\n", + "offset": 12, + } + assert result["channels"]["stderr"] == { + "content": "", + "offset": 0, + } + assert result["channels"]["schedule"] == { + "content": "sched2\n", + "offset": 14, + } + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_GetProtocolLogsResolvesPostgresqlProtocolId(service, tmp_path): + stdoutLog = tmp_path / "stdout.log" + stderrLog = tmp_path / "stderr.log" + scheduleLog = tmp_path / "schedule.log" + + stdoutLog.write_text("abc\ndef\n", encoding="utf-8") + stderrLog.write_text("ERR1\nERR2\n", encoding="utf-8") + scheduleLog.write_text("SCH1\nSCH2\n", encoding="utf-8") + + service.currentProject.protocols[10] = FakeProtocol( + stdoutLog=str(stdoutLog), + stderrLog=str(stderrLog), + scheduleLog=str(scheduleLog), + ) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.getProtocolLogs( + projectId=1, + protocolId=500, + offset=4, + errOffset=5, + scheduleOffset=5, + mapper=mapper, + ) + + assert result == { + "stdoutLog": "def\n", + "stderrLog": "ERR2\n", + "stdoutOffset": 8, + "stderrOffset": 10, + "scheduleLog": "SCH2\n", + "scheduleOffset": 10, + } + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + def test_GetProtocolPathReturnsGlobalBrowserPayload(service, monkeypatch, tmp_path): browserRoot = tmp_path / "browser-root" browserRoot.mkdir(parents=True, exist_ok=True) From 375e1d130a829497cb669d2e7ffa91020860cb00 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 13:31:28 +0200 Subject: [PATCH 133/278] ormalize PostgreSQL metadata values before rendering --- app/backend/viewers/postgresql_dao.py | 218 +++++++++++++++++- .../backend/viewers/test_postgresql_dao.py | 84 ++++++- 2 files changed, 289 insertions(+), 13 deletions(-) diff --git a/app/backend/viewers/postgresql_dao.py b/app/backend/viewers/postgresql_dao.py index c8c06063..5d07f420 100644 --- a/app/backend/viewers/postgresql_dao.py +++ b/app/backend/viewers/postgresql_dao.py @@ -32,8 +32,10 @@ # * # ****************************************************************************** +import ast import json import logging +import re from typing import Any, Dict, Iterable, List, Optional @@ -47,6 +49,7 @@ FloatRenderer, ImageRenderer, StrRenderer, + MatrixRender, ) from pwem.convert.transformations import euler_from_matrix @@ -259,9 +262,23 @@ def fillTable(self, table, objectManager): for index, colName in enumerate(labels): value = firstRow.get(colName) isFileNameCol = imgRenderer is None and colName.endswith("_filename") + columnDef = next( + ( + column + for column in columns + if str(column.get("labelProperty") or "").strip() == str(colName) + ), + None, + ) + columnClassName = self._getColumnClassName(columnDef) + columnClassNameLower = columnClassName.lower() - if colName == ENABLED_COLUMN: + if colName == ENABLED_COLUMN or columnClassNameLower == "boolean": renderer = BoolRenderer() + elif columnClassNameLower == "matrix" or colName.endswith("_matrix") or self._looksLikeMatrix(value): + renderer = MatrixRender() + elif columnClassNameLower == "float": + renderer = FloatRenderer() elif isFileNameCol: renderer = StrRenderer() else: @@ -840,7 +857,7 @@ def _itemToRow( row: Dict[str, Any] = { "id": itemId, - "enabled": bool(item.get("enabled", True)), + "enabled": self._toBool(item.get("enabled", True), default=True), } for column in columns: @@ -852,7 +869,7 @@ def _itemToRow( if value is None: value = values.get(str(label).strip()) - row[str(label)] = self._normalizeValue(str(label), value) + row[str(label)] = self._normalizeValue(str(label), value, column) return row @@ -1012,47 +1029,224 @@ def _getStoredSetItemsCount(self) -> int: return 0 - def _normalizeValue(self, label: str, value: Any) -> Any: + @staticmethod + def _getColumnClassName(column: Optional[Dict[str, Any]]) -> str: + if not isinstance(column, dict): + return "" + + for key in ("className", "type", "columnType"): + value = column.get(key) + if value is not None: + return str(value).strip() + + return "" + + @staticmethod + def _isBooleanString(value: Any) -> bool: + if not isinstance(value, str): + return False + + return value.strip().lower() in { + "true", + "false", + "t", + "f", + "yes", + "no", + "y", + "n", + "1", + "0", + "on", + "off", + } + + @staticmethod + def _toBool(value: Any, default: bool = False) -> bool: + if isinstance(value, bool): + return value + if value is None: + return default + + if isinstance(value, (int, float)): + return bool(value) + + if isinstance(value, str): + text = value.strip().lower() + if text in {"true", "t", "yes", "y", "1", "on"}: + return True + if text in {"false", "f", "no", "n", "0", "off"}: + return False + + return default + + @staticmethod + def _toFloat(value: Any) -> Optional[float]: + if value is None or value == "": return None - if label.endswith("_matrix"): + try: + return float(value) + except Exception: + return None + + @staticmethod + def _looksLikeMatrix(value: Any) -> bool: + if isinstance(value, numpy.ndarray): + return value.ndim == 2 + + if isinstance(value, (list, tuple)): + try: + return numpy.asarray(value).ndim == 2 + except Exception: + return False + + if not isinstance(value, str): + return False + + text = value.strip() + return text.startswith("[[") and text.endswith("]]") + + def _normalizeValue( + self, + label: str, + value: Any, + column: Optional[Dict[str, Any]] = None, + ) -> Any: + if value is None: + return None + + className = self._getColumnClassName(column) + classNameLower = className.lower() + + if classNameLower == "boolean" or label == ENABLED_COLUMN: + return self._toBool(value) + + if classNameLower == "integer": + parsed = self._toInt(value) + return parsed if parsed is not None else value + + if classNameLower == "float": + parsed = self._toFloat(value) + return parsed if parsed is not None else value + + if classNameLower == "matrix" or label.endswith("_matrix") or self._looksLikeMatrix(value): return self._toNumpyMatrix(value) - if isinstance(value, (str, int, float, bool)): + if classNameLower == "csvlist": + return self._toListValue(value) + + if isinstance(value, str): + if self._isBooleanString(value): + return self._toBool(value) + + return value + + if isinstance(value, (int, float, bool)): return value if isinstance(value, list): return [ - self._normalizeValue(label, item) + self._normalizeValue(label, item, column) for item in value ] if isinstance(value, tuple): return [ - self._normalizeValue(label, item) + self._normalizeValue(label, item, column) for item in value ] if isinstance(value, dict): return { - str(key): self._normalizeValue(label, item) + str(key): self._normalizeValue(label, item, column) for key, item in value.items() } return str(value) + def _toListValue(self, value: Any) -> List[Any]: + if value is None: + return [] + + if isinstance(value, numpy.ndarray): + return value.tolist() + + if isinstance(value, list): + return value + + if isinstance(value, tuple): + return list(value) + + if isinstance(value, str): + text = value.strip() + if not text: + return [] + + try: + parsed = json.loads(text) + if isinstance(parsed, list): + return parsed + except Exception: + pass + + try: + parsed = ast.literal_eval(text) + if isinstance(parsed, (list, tuple)): + return list(parsed) + except Exception: + pass + + return [item.strip() for item in text.split(",") if item.strip()] + + return [value] + def _toNumpyMatrix(self, value: Any): if isinstance(value, numpy.ndarray): return value if isinstance(value, str): + text = value.strip() + if not text: + return numpy.array([]) + + parsed = None + try: - value = eval(value) + parsed = json.loads(text) except Exception: - value = [] + parsed = None + + if parsed is None: + try: + parsed = ast.literal_eval(text) + except Exception: + parsed = None + + if parsed is None: + rowTexts = re.findall(r"\[([^\[\]]+)\]", text) + rows = [] + + for rowText in rowTexts: + numbers = re.findall( + r"[-+]?(?:\d+\.\d*|\.\d+|\d+)(?:[eE][-+]?\d+)?", + rowText, + ) + if numbers: + rows.append([float(number) for number in numbers]) - return numpy.array(value) + parsed = rows + + try: + return numpy.asarray(parsed, dtype=float) + except Exception: + return numpy.array([]) + + try: + return numpy.asarray(value, dtype=float) + except Exception: + return numpy.array([]) def _sortValue(self, value: Any): if value is None: diff --git a/tests/unit/backend/viewers/test_postgresql_dao.py b/tests/unit/backend/viewers/test_postgresql_dao.py index 998307e9..b1cf6918 100644 --- a/tests/unit/backend/viewers/test_postgresql_dao.py +++ b/tests/unit/backend/viewers/test_postgresql_dao.py @@ -25,6 +25,7 @@ # ****************************************************************************** import importlib +import numpy as np import pytest @@ -371,4 +372,85 @@ def test_GetTableWithAdditionalInfoReturnsCompatibilityTuple(postgresqlDaoModule assert table is None assert displayColumns == postgresqlDaoModule.ADITIONAL_INFO_DISPLAY_COLUMN_LIST - assert displayColumns == ["_size", "id"] \ No newline at end of file + assert displayColumns == ["_size", "id"] + + +def test_PostgresqlDaoNormalizesTypedStringsFromDatabase(postgresqlDaoModule): + dao = object.__new__(postgresqlDaoModule.PostgresqlDAO) + + matrixText = ( + "[[ 1. 0. 0. -2275.2 ]\n" + " [ 0. 1. 0. -1616.34]\n" + " [ 0. 0. 1. 0. ]\n" + " [ 0. 0. 0. 1. ]]" + ) + + item = { + "id": 1, + "enabled": "False", + "values": { + "flag": "False", + "score": "3.5", + "count": "7", + "transform_matrix": matrixText, + }, + } + + columns = [ + { + "labelProperty": "flag", + "position": 0, + "className": "Boolean", + }, + { + "labelProperty": "score", + "position": 1, + "className": "Float", + }, + { + "labelProperty": "count", + "position": 2, + "className": "Integer", + }, + { + "labelProperty": "transform_matrix", + "position": 3, + "className": "Matrix", + }, + ] + + row = dao._itemToRow(item, columns) + + assert row["enabled"] is False + assert row["flag"] is False + assert row["score"] == pytest.approx(3.5) + assert row["count"] == 7 + + assert isinstance(row["transform_matrix"], np.ndarray) + assert row["transform_matrix"].shape == (4, 4) + assert row["transform_matrix"][0, 3] == pytest.approx(-2275.2) + assert row["transform_matrix"][1, 3] == pytest.approx(-1616.34) + + +def test_PostgresqlDaoParsesJsonMatrixString(postgresqlDaoModule): + dao = object.__new__(postgresqlDaoModule.PostgresqlDAO) + + matrix = dao._normalizeValue( + "transform_matrix", + "[[1, 0, 0, -1.5], [0, 1, 0, 2.5], [0, 0, 1, 0], [0, 0, 0, 1]]", + {"className": "Matrix"}, + ) + + assert isinstance(matrix, np.ndarray) + assert matrix.shape == (4, 4) + assert matrix[0, 3] == pytest.approx(-1.5) + assert matrix[1, 3] == pytest.approx(2.5) + + +def test_PostgresqlDaoDoesNotUseEvalForMatrixStrings(postgresqlDaoModule): + dao = object.__new__(postgresqlDaoModule.PostgresqlDAO) + + matrix = dao._toNumpyMatrix("__import__('os').system('echo unsafe')") + + assert isinstance(matrix, np.ndarray) + assert matrix.size == 0 \ No newline at end of file From ccdd314b4e9e229081eb6afdfe6a95987f25c013 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 13:54:07 +0200 Subject: [PATCH 134/278] Apply sorting to metadata table pages --- app/backend/api/services/project_service.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 3dbe4b92..fc76a790 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -11587,6 +11587,8 @@ def getMetadataTablePageService( mapper=mapper, ) columns = list(table.getColumns()) + table.setSortingColumn(sortBy) + table.setSortingAsc(asc) if selectionOnly: rows: list = [] From bb7e439f5513aee1357e533d0573695cb89b59a6 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 14:04:53 +0200 Subject: [PATCH 135/278] Resolve runtime ids for protocol mutations --- app/backend/api/services/project_service.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index fc76a790..06a1bd96 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -4902,12 +4902,14 @@ def updateProtocolStepStatusService( targetStatus = statusMap[normalizedStatus] - protocol = self._getScipionProtocolForRuntime( + scipionProtocolId = self._resolveScipionProtocolId( mapper=mapper, projectId=projectId, protocolId=protocolId, ) + protocol = self._getScipionProtocolByRuntimeId(scipionProtocolId) + try: steps = protocol.loadSteps() or [] except Exception as e: @@ -4965,7 +4967,7 @@ def updateProtocolStepStatusService( row = mapper.updateProtocolStepStatus( projectId=projectId, - protocolId=protocolId, + protocolId=scipionProtocolId, stepIndex=stepIndex, stepStatus=targetStatus, ) From 658350b9e66b808da9af9e5936b584f4b83fe80d Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 14:08:46 +0200 Subject: [PATCH 136/278] Resolve runtime id when updating protocol steps --- .../test_project_service_protocol_steps.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/unit/backend/api/services/test_project_service_protocol_steps.py b/tests/unit/backend/api/services/test_project_service_protocol_steps.py index 63c16892..265a2584 100644 --- a/tests/unit/backend/api/services/test_project_service_protocol_steps.py +++ b/tests/unit/backend/api/services/test_project_service_protocol_steps.py @@ -37,8 +37,33 @@ def getProtocol(self, protocolId): return self.protocols[int(protocolId)] +class FakeDb: + def __init__(self): + self.runtimeProtocolIdByDbId = {} + self.fetchOneCalls = [] + + def fetchOne(self, query, params): + self.fetchOneCalls.append({ + "query": query, + "params": params, + }) + + if len(params) < 3: + return None + + protocolDbId = params[1] + runtimeProtocolId = self.runtimeProtocolIdByDbId.get(int(protocolDbId)) + if runtimeProtocolId is None: + return None + + return { + "protocolId": runtimeProtocolId, + } + + class FakeMapper: def __init__(self): + self.db = FakeDb() self.listProtocolStepsResult = [] self.updateProtocolStepStatusCalls = [] self.updateProtocolStepStatusResult = None @@ -173,6 +198,48 @@ def test_UpdateProtocolStepStatusUpdatesScipionAndPostgres( ] +def test_UpdateProtocolStepStatusResolvesPostgresqlProtocolId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "STATUS_FINISHED", "finished") + + step = FakeStep(index=2, objId=102) + protocol = FakeProtocolWithSteps([step]) + + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.updateProtocolStepStatusResult = { + "index": 2, + "name": "processStep", + "status": "finished", + } + + result = service.updateProtocolStepStatusService( + mapper=mapper, + projectId=1, + protocolId=500, + stepIndex=2, + stepStatus="finished", + ) + + assert result == mapper.updateProtocolStepStatusResult + assert protocol.updateStepsCalls == [{"where": "id='102'"}] + assert step.status == "finished" + + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + assert mapper.updateProtocolStepStatusCalls == [ + { + "projectId": 1, + "protocolId": 10, + "stepIndex": 2, + "stepStatus": "finished", + }, + ] + + def test_UpdateProtocolStepStatusAcceptsObjIdHolderFallback( projectServiceModule, service, From ba9c968e67fec6f224b96adc90d4b8cc2ac988fb Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 14:18:16 +0200 Subject: [PATCH 137/278] Test runtime id resolution for protocol mutations --- .../test_project_service_protocols.py | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index d53625f1..2abe3b93 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -212,9 +212,34 @@ def resetWorkFlow(self, workflowProtocolList): return self.resetWorkflowResult +class FakeDb: + def __init__(self): + self.runtimeProtocolIdByDbId = {} + self.fetchOneCalls = [] + + def fetchOne(self, query, params): + self.fetchOneCalls.append({ + "query": query, + "params": params, + }) + + if len(params) < 3: + return None + + protocolDbId = params[1] + runtimeProtocolId = self.runtimeProtocolIdByDbId.get(int(protocolDbId)) + if runtimeProtocolId is None: + return None + + return { + "protocolId": runtimeProtocolId, + } + + class FakeMapper: # fakeMapper def __init__(self): + self.db = FakeDb() self.dbProtocolsByProtocolId = {} self.savedProtocolContexts = [] self.deleteProtocolCalls = [] @@ -1357,5 +1382,119 @@ def test_StopProtocolStopsEachProtocol(service): protocolIds=["10", "11"], ) + assertSuccessEnvelope(result) + assert service.currentProject.stoppedProtocols == [protocolA, protocolB] + + +def test_RenameProtocolResolvesPostgresqlProtocolId(service, mapper): + protocol = FakeProtocol(objId=10) + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + result = service.renameProtocol( + mapper=mapper, + projectId=1, + protocolId=500, + newName="Renamed protocol", + newComment="Updated comment", + ) + + assertSuccessEnvelope(result) + assert protocol.runName.get() == "Renamed protocol" + assert protocol._objComment == "Updated comment" + assert service.currentProject.storedProtocols == [protocol] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_DuplicateProtocolResolvesPostgresqlProtocolIds(service, mapper, monkeypatch): + protocolA = FakeProtocol(objId=10) + protocolB = FakeProtocol(objId=11) + copiedA = FakeProtocol(objId=110) + copiedB = FakeProtocol(objId=111) + + service.currentProject.protocols[10] = protocolA + service.currentProject.protocols[11] = protocolB + service.currentProject.copiedProtocolOutputs = [copiedA, copiedB] + + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.db.runtimeProtocolIdByDbId[501] = 11 + + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + lambda mapper, projectId, refresh=False, checkPid=False: { + "protocols": 2, + "dependencies": 0, + }, + ) + + class DuplicateItem: + def __init__(self, itemId): + self.id = itemId + + result = service.duplicateProtocol( + mapper=mapper, + projectId=1, + protocols=[DuplicateItem("500"), DuplicateItem("501")], + ) + + assertSuccessEnvelope(result) + assert service.currentProject.copiedProtocolInputs == [[protocolA, protocolB]] + assert result["duplicated"] == [ + {"sourceId": "500", "newId": "110"}, + {"sourceId": "501", "newId": "111"}, + ] + + +def test_DeleteProtocolResolvesPostgresqlProtocolIds(service, mapper, monkeypatch): + protocolA = FakeProtocol(objId=10) + protocolB = FakeProtocol(objId=11) + + service.currentProject.protocols[10] = protocolA + service.currentProject.protocols[11] = protocolB + + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.db.runtimeProtocolIdByDbId[501] = 11 + + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + lambda mapper, projectId, refresh=False, checkPid=False: { + "protocols": 0, + "dependencies": 0, + }, + ) + + result = service.deleteProtocol( + mapper=mapper, + projectId=1, + protocols=["500", "501"], + ) + + assert result["status"] == 0 + assert service.currentProject.deleteProtocolCalls == [[protocolA, protocolB]] + assert mapper.deleteProtocolCalls == [ + { + "projectId": 1, + "protocolList": [protocolA, protocolB], + } + ] + +def test_StopProtocolResolvesPostgresqlProtocolIds(service, mapper): + protocolA = FakeProtocol(objId=10) + protocolB = FakeProtocol(objId=11) + + service.currentProject.protocols[10] = protocolA + service.currentProject.protocols[11] = protocolB + + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.db.runtimeProtocolIdByDbId[501] = 11 + + result = service.stopProtocol( + mapper=mapper, + projectId=1, + protocolIds=["500", "501"], + ) + assertSuccessEnvelope(result) assert service.currentProject.stoppedProtocols == [protocolA, protocolB] \ No newline at end of file From 78b2d92019640b50608161cf42fe674dfc5b2e1e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 14:27:28 +0200 Subject: [PATCH 138/278] Test runtime id resolution for subtree mutations --- .../test_project_service_protocols.py | 180 +++++++++++++++++- 1 file changed, 179 insertions(+), 1 deletion(-) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index 2abe3b93..602420c7 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -1497,4 +1497,182 @@ def test_StopProtocolResolvesPostgresqlProtocolIds(service, mapper): ) assertSuccessEnvelope(result) - assert service.currentProject.stoppedProtocols == [protocolA, protocolB] \ No newline at end of file + assert service.currentProject.stoppedProtocols == [protocolA, protocolB] + + +def test_RestartProtocolAllResolvesPostgresqlProtocolId(service, mapper, monkeypatch): + protocol = FakeProtocol(objId=10) + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + subworkflowCalls = [] + + def fakeGetSubworkflow(protocolObj): + subworkflowCalls.append(protocolObj) + return [protocol], [] + + service.currentProject._getSubworkflow = fakeGetSubworkflow + + cleanupCalls = [] + + monkeypatch.setattr( + service, + "_deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql", + lambda mapper, projectId, protocols: cleanupCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocols": protocols, + }) or { + "protocolsCount": len(protocols), + "setsDeleted": 0, + "objectsDeleted": 0, + "items": [], + }, + ) + + result = service.restartProtocolAll( + mapper=mapper, + projectId=1, + protocolId=500, + ) + + assertSuccessEnvelope(result) + assert subworkflowCalls == [protocol] + assert cleanupCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocols": [protocol], + } + ] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_ContinueProtocolAllResolvesPostgresqlProtocolId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "MODE_RESUME", "resume-mode") + + protocol = FakeProtocol(objId=10) + activeProtocol = FakeProtocol(objId=20) + + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + subworkflowCalls = [] + + def fakeGetSubworkflow(protocolObj): + subworkflowCalls.append(protocolObj) + return [protocol], [activeProtocol] + + service.currentProject._getSubworkflow = fakeGetSubworkflow + + result = service.continueProtocolAll( + mapper=mapper, + projectId=1, + protocolId=500, + currentUser={"id": 1}, + ) + + assertSuccessEnvelope(result) + assert subworkflowCalls == [protocol] + assert activeProtocol.runMode.get() == "resume-mode" + assert service.currentProject.launchedProtocols == [activeProtocol] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_ResetProtocolFromResolvesPostgresqlProtocolId(service, mapper, monkeypatch): + protocol = FakeProtocol(objId=10) + + service.currentProject.protocols[10] = protocol + service.currentProject.resetWorkflowResult = [] + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + subworkflowCalls = [] + + def fakeGetSubworkflow(protocolObj): + subworkflowCalls.append(protocolObj) + return [protocol], [] + + service.currentProject._getSubworkflow = fakeGetSubworkflow + + cleanupCalls = [] + + monkeypatch.setattr( + service, + "_deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql", + lambda mapper, projectId, protocols: cleanupCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocols": protocols, + }) or { + "protocolsCount": len(protocols), + "setsDeleted": 0, + "objectsDeleted": 0, + "items": [], + }, + ) + + result = service.resetProtocolFrom( + mapper=mapper, + projectId=1, + protocolId=500, + ) + + assertSuccessEnvelope(result) + assert subworkflowCalls == [protocol] + assert cleanupCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocols": [protocol], + } + ] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_SaveProtocolResolvesPostgresqlProtocolIdForExistingProtocol( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) + monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) + monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) + monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) + + protocol = FakeProtocol(objId=10, className="ProtClass") + protocol.addParam("runName", FakeStringParam(label="Run name")) + protocol.addParam("iterations", FakeIntParam(label="Iterations")) + + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + monkeypatch.setattr( + service, + "applyParamsToProtocol", + lambda mapper=None, projectId=None, protocol=None, params=None: [], + ) + + savedProtocol, errors = service.saveProtocol( + mapper=mapper, + projectId=1, + protocolId=500, + protocolClassName="ProtClass", + params={ + "runName": "Edited protocol", + "iterations": "7", + }, + setToSave=False, + ) + + assert errors == [] + assert savedProtocol is protocol + assert protocol.runName.get() == "Edited protocol" + assert protocol.attributeValues["iterations"] == 7 + assert service.currentProject.storedProtocols == [protocol] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") \ No newline at end of file From ca986cb836aed188b211262b503471d29ef1ec8e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 14:40:35 +0200 Subject: [PATCH 139/278] Test runtime id resolution for protocol launch --- .../test_project_service_protocols.py | 161 +++++++++++++++++- 1 file changed, 160 insertions(+), 1 deletion(-) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index 602420c7..0075d103 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -1675,4 +1675,163 @@ def test_SaveProtocolResolvesPostgresqlProtocolIdForExistingProtocol( assert protocol.runName.get() == "Edited protocol" assert protocol.attributeValues["iterations"] == 7 assert service.currentProject.storedProtocols == [protocol] - assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") \ No newline at end of file + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_LaunchProtocolLaunchResolvesPostgresqlProtocolId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) + monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) + monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) + monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) + monkeypatch.setattr(projectServiceModule, "MODE_RESUME", "resume-mode") + + protocol = FakeProtocol(objId=10, className="ProtClass", validateErrors=[]) + protocol.addParam("runName", FakeStringParam(label="Run name")) + protocol.addParam("iterations", FakeIntParam(label="Iterations")) + + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + monkeypatch.setattr( + service, + "applyParamsToProtocol", + lambda mapper=None, projectId=None, protocol=None, params=None: [], + ) + + service.launchProtocol( + mapper=mapper, + projectId=1, + protocolId=500, + protocolClassName="ProtClass", + params={ + "runName": "Launch protocol", + "iterations": "7", + }, + executeMode="launch", + ) + + assert protocol.runName.get() == "Launch protocol" + assert protocol.attributeValues["iterations"] == 7 + assert protocol.runMode.get() == "resume-mode" + assert service.currentProject.storedProtocols == [protocol] + assert service.currentProject.launchedProtocols == [protocol] + assert service.currentProject.scheduledProtocols == [] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + def test_LaunchProtocolRestartResolvesPostgresqlProtocolId( + projectServiceModule, + service, + mapper, + monkeypatch, + ): + monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) + monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) + monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) + monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) + monkeypatch.setattr(projectServiceModule, "MODE_RESTART", "restart-mode") + + protocol = FakeProtocol(objId=10, className="ProtClass", validateErrors=[]) + protocol.addParam("runName", FakeStringParam(label="Run name")) + protocol.addParam("iterations", FakeIntParam(label="Iterations")) + + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + monkeypatch.setattr( + service, + "applyParamsToProtocol", + lambda mapper=None, projectId=None, protocol=None, params=None: [], + ) + + cleanupCalls = [] + + monkeypatch.setattr( + service, + "_deletePersistedProtocolOutputsForRuntimeProtocolsFromPostgresql", + lambda mapper, projectId, protocols: cleanupCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocols": protocols, + }) or { + "protocolsCount": len(protocols), + "setsDeleted": 0, + "objectsDeleted": 0, + "items": [], + }, + ) + + service.launchProtocol( + mapper=mapper, + projectId=1, + protocolId=500, + protocolClassName="ProtClass", + params={ + "runName": "Restart protocol", + "iterations": "9", + }, + executeMode="restart", + ) + + assert protocol.runName.get() == "Restart protocol" + assert protocol.attributeValues["iterations"] == 9 + assert protocol.runMode.get() == "restart-mode" + assert service.currentProject.storedProtocols == [protocol] + assert service.currentProject.launchedProtocols == [protocol] + assert service.currentProject.scheduledProtocols == [] + assert cleanupCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocols": [protocol], + } + ] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + +def test_LaunchProtocolScheduleResolvesPostgresqlProtocolId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "IntParam", FakeIntParam) + monkeypatch.setattr(projectServiceModule, "StringParam", FakeStringParam) + monkeypatch.setattr(projectServiceModule, "EnumParam", FakeEnumParam) + monkeypatch.setattr(projectServiceModule, "CsvList", FakeCsvList) + + protocol = FakeProtocol(objId=10, className="ProtClass", validateErrors=[]) + protocol.addParam("runName", FakeStringParam(label="Run name")) + protocol.addParam("iterations", FakeIntParam(label="Iterations")) + + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + monkeypatch.setattr( + service, + "applyParamsToProtocol", + lambda mapper=None, projectId=None, protocol=None, params=None: [], + ) + + service.launchProtocol( + mapper=mapper, + projectId=1, + protocolId=500, + protocolClassName="ProtClass", + params={ + "runName": "Schedule protocol", + "iterations": "11", + }, + executeMode="schedule", + ) + + assert protocol.runName.get() == "Schedule protocol" + assert protocol.attributeValues["iterations"] == 11 + assert protocol.runMode.get() is None + assert service.currentProject.storedProtocols == [protocol] + assert service.currentProject.scheduledProtocols == [protocol] + assert service.currentProject.launchedProtocols == [] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") From ac975012cef426b4652552055caa619e26fefcd9 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 14:44:37 +0200 Subject: [PATCH 140/278] Fixing indentation errors --- .../backend/api/services/test_project_service_protocols.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index 0075d103..b5142ddc 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -1723,7 +1723,8 @@ def test_LaunchProtocolLaunchResolvesPostgresqlProtocolId( assert service.currentProject.scheduledProtocols == [] assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") - def test_LaunchProtocolRestartResolvesPostgresqlProtocolId( + +def test_LaunchProtocolRestartResolvesPostgresqlProtocolId( projectServiceModule, service, mapper, From 1453ba04626c6d4c795bfdae39168b6808a6b78f Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 15:27:31 +0200 Subject: [PATCH 141/278] Test runtime id resolution for workflow export --- .../test_project_service_protocols.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index b5142ddc..a7539ef6 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -1836,3 +1836,84 @@ def test_LaunchProtocolScheduleResolvesPostgresqlProtocolId( assert service.currentProject.scheduledProtocols == [protocol] assert service.currentProject.launchedProtocols == [] assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_ExportWorkflowProtocolsResolvesPostgresqlProtocolIds(service, mapper): + protocolA = FakeProtocol(objId=10, className="ProtA") + protocolB = FakeProtocol(objId=11, className="ProtB") + + service.currentProject.protocols[10] = protocolA + service.currentProject.protocols[11] = protocolB + + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.db.runtimeProtocolIdByDbId[501] = 11 + + exportedProtocolLists = [] + + def fakeGetProtocolsJson(protocolList): + exportedProtocolLists.append(protocolList) + return [ + { + "protocol": "exported-a", + }, + { + "protocol": "exported-b", + }, + ] + + service.currentProject.getProtocolsJson = fakeGetProtocolsJson + + class FakeExportPayload: + includeUpstream = False + protocolIds = ["500", "501"] + + result = service.exportWorkflowProtocolsService( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=FakeExportPayload(), + ) + + assert exportedProtocolLists == [[protocolA, protocolB]] + assert result["sourceProjectId"] == 1 + assert result["protocolIds"] == ["500", "501"] + assert result["workflow"] == [ + { + "protocol": "exported-a", + }, + { + "protocol": "exported-b", + }, + ] + + assert result["scipionWeb"]["sourceProjectId"] == 1 + assert result["scipionWeb"]["sourceProtocolIds"] == ["500", "501"] + assert result["scipionWeb"]["protocolPlugins"][0]["protocolId"] == "10" + assert result["scipionWeb"]["protocolPlugins"][1]["protocolId"] == "11" + + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + assert mapper.db.fetchOneCalls[1]["params"] == (1, 501, "501") + + +def test_ExportWorkflowProtocolsRaisesWhenPostgresqlProtocolIdCannotBeResolved(service, mapper): + protocol = FakeProtocol(objId=10, className="ProtA") + service.currentProject.protocols[10] = protocol + + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + class FakeExportPayload: + includeUpstream = False + protocolIds = ["500", "999"] + + with pytest.raises(HTTPException) as exc: + service.exportWorkflowProtocolsService( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=FakeExportPayload(), + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Protocol(s) not found: 999" + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + assert mapper.db.fetchOneCalls[1]["params"] == (1, 999, "999") \ No newline at end of file From 48c6daaa10a6bdbd41c2f80065427db969aa1290 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 15:32:21 +0200 Subject: [PATCH 142/278] Test workflow import reference handling --- .../test_project_service_protocols.py | 156 +++++++++++++++++- 1 file changed, 155 insertions(+), 1 deletion(-) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index a7539ef6..39d9f5d3 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -25,6 +25,7 @@ # ****************************************************************************** import importlib +import json import pytest from fastapi import HTTPException @@ -1916,4 +1917,157 @@ class FakeExportPayload: assert exc.value.status_code == 404 assert exc.value.detail == "Protocol(s) not found: 999" assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") - assert mapper.db.fetchOneCalls[1]["params"] == (1, 999, "999") \ No newline at end of file + assert mapper.db.fetchOneCalls[1]["params"] == (1, 999, "999") + + +def test_ImportWorkflowProtocolsSanitizesExternalReferences(service, mapper, monkeypatch): + loadedJsonPayloads = [] + + def fakeLoadProtocols(jsonStr): + loadedJsonPayloads.append(json.loads(jsonStr)) + return None + + service.currentProject.loadProtocols = fakeLoadProtocols + + workflowIds = [ + {"10"}, + {"10", "20", "21"}, + ] + + monkeypatch.setattr( + service, + "_getCurrentWorkflowProtocolIds", + lambda: workflowIds.pop(0), + ) + + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + lambda mapper, projectId, refresh=False, checkPid=False: { + "protocols": 3, + "dependencies": 2, + }, + ) + + class FakeImportPayload: + mode = "append" + sourceProjectId = 999 + workflow = [ + { + "object.id": "1", + "inputFromCopiedProtocol": "1.outputParticles", + "inputFromExternalProtocol": "99.outputParticles", + "params": { + "validPointer": "2.outputVolume", + "invalidPointer": "100.outputCoordinates", + }, + }, + { + "object.id": "2", + "inputFromFirstProtocol": "1.outputParticles", + }, + ] + + result = service.importWorkflowProtocolsService( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=FakeImportPayload(), + ) + + assert result == { + "status": 0, + "errors": [], + "workflow": [], + "created": [ + {"newId": "20"}, + {"newId": "21"}, + ], + "protocolsCount": 3, + "dependenciesCount": 2, + } + + assert loadedJsonPayloads == [ + [ + { + "object.id": "1", + "inputFromCopiedProtocol": "1.outputParticles", + "params": { + "validPointer": "2.outputVolume", + }, + }, + { + "object.id": "2", + "inputFromFirstProtocol": "1.outputParticles", + }, + ] + ] + + +def test_ImportWorkflowProtocolsKeepsReferencesForSameProject(service, mapper, monkeypatch): + loadedJsonPayloads = [] + + def fakeLoadProtocols(jsonStr): + loadedJsonPayloads.append(json.loads(jsonStr)) + return None + + service.currentProject.loadProtocols = fakeLoadProtocols + + workflowIds = [ + {"10"}, + {"10", "20"}, + ] + + monkeypatch.setattr( + service, + "_getCurrentWorkflowProtocolIds", + lambda: workflowIds.pop(0), + ) + + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + lambda mapper, projectId, refresh=False, checkPid=False: { + "protocols": 2, + "dependencies": 1, + }, + ) + + class FakeImportPayload: + mode = "append" + sourceProjectId = 1 + workflow = [ + { + "object.id": "1", + "inputFromExistingSameProjectProtocol": "99.outputParticles", + "inputFromCopiedProtocol": "1.outputParticles", + }, + ] + + result = service.importWorkflowProtocolsService( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=FakeImportPayload(), + ) + + assert result == { + "status": 0, + "errors": [], + "workflow": [], + "created": [ + {"newId": "20"}, + ], + "protocolsCount": 2, + "dependenciesCount": 1, + } + + assert loadedJsonPayloads == [ + [ + { + "object.id": "1", + "inputFromExistingSameProjectProtocol": "99.outputParticles", + "inputFromCopiedProtocol": "1.outputParticles", + }, + ] + ] \ No newline at end of file From 85f14517536d4db462f71c5fdfc6a1be8feca60b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 15:36:47 +0200 Subject: [PATCH 143/278] Resolve runtime id when listing protocol steps --- app/backend/api/services/project_service.py | 8 ++++- .../test_project_service_protocol_steps.py | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 06a1bd96..b70e8a4e 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -4878,7 +4878,13 @@ def saveProtocol(self, mapper, projectId, protocolId, protocolClassName, params, return protocol, errorList def listProtocolStepsService(self, mapper, projectId: int, protocolId: int): - return mapper.listProtocolSteps(projectId, protocolId) + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + return mapper.listProtocolSteps(projectId, scipionProtocolId) def updateProtocolStepStatusService( self, diff --git a/tests/unit/backend/api/services/test_project_service_protocol_steps.py b/tests/unit/backend/api/services/test_project_service_protocol_steps.py index 265a2584..78d4ced6 100644 --- a/tests/unit/backend/api/services/test_project_service_protocol_steps.py +++ b/tests/unit/backend/api/services/test_project_service_protocol_steps.py @@ -65,10 +65,15 @@ class FakeMapper: def __init__(self): self.db = FakeDb() self.listProtocolStepsResult = [] + self.listProtocolStepsCalls = [] self.updateProtocolStepStatusCalls = [] self.updateProtocolStepStatusResult = None def listProtocolSteps(self, projectId, protocolId): + self.listProtocolStepsCalls.append({ + "projectId": projectId, + "protocolId": protocolId, + }) return self.listProtocolStepsResult def updateProtocolStepStatus(self, **kwargs): @@ -158,6 +163,30 @@ def test_ListProtocolStepsDelegatesToMapper(service, mapper): assert result == [{"index": 1, "name": "resumeStep", "status": "finished"}] +def test_ListProtocolStepsResolvesPostgresqlProtocolId(service, mapper): + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.listProtocolStepsResult = [ + {"index": 1, "name": "resumeStep", "status": "finished"}, + ] + + result = service.listProtocolStepsService( + mapper=mapper, + projectId=1, + protocolId=500, + ) + + assert result == [ + {"index": 1, "name": "resumeStep", "status": "finished"}, + ] + assert mapper.listProtocolStepsCalls == [ + { + "projectId": 1, + "protocolId": 10, + } + ] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + def test_UpdateProtocolStepStatusUpdatesScipionAndPostgres( projectServiceModule, service, From 5a3da1ef9dc3937dcd9692fa6be7599a33474e0a Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 15:42:52 +0200 Subject: [PATCH 144/278] Test runtime id resolution for file workflow export --- .../test_project_service_io_thumbnails.py | 88 ++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/tests/unit/backend/api/services/test_project_service_io_thumbnails.py b/tests/unit/backend/api/services/test_project_service_io_thumbnails.py index ef836849..23cfbbf4 100644 --- a/tests/unit/backend/api/services/test_project_service_io_thumbnails.py +++ b/tests/unit/backend/api/services/test_project_service_io_thumbnails.py @@ -679,4 +679,90 @@ def fakeBuildProtocolOutputThumbnail( } ] - assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") \ No newline at end of file + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_ExportProtocolsServiceResolvesPostgresqlProtocolIdsAndWritesJsonFile( + service, + monkeypatch, + tmp_path, +): + rootPath = tmp_path / "browser-root" + rootPath.mkdir(parents=True, exist_ok=True) + + protocol10 = FakeProtocol(protocolId=10) + protocol11 = FakeProtocol(protocolId=11) + + service.currentProject = FakeCurrentProject( + protocols={ + 10: protocol10, + 11: protocol11, + }, + exportPayload=[ + {"protocolId": 10}, + {"protocolId": 11}, + ], + ) + + exportedProtocolLists = [] + + def fakeGetProtocolsJson(protocolList): + exportedProtocolLists.append(protocolList) + return [ + {"protocolId": 10}, + {"protocolId": 11}, + ] + + service.currentProject.getProtocolsJson = fakeGetProtocolsJson + + mapper = FakeMapper(runtimeProtocolIdByDbId={ + 500: 10, + 501: 11, + }) + + monkeypatch.setattr( + service, + "_resolveFsRootForWrite", + lambda protocolId, mapper=None, projectId=None: rootPath, + ) + + payload = FakePayload( + protocolIds=["500", "501"], + directoryPath="exports", + filename="workflow-export", + ) + + result = service.exportProtocolsService( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=payload, + ) + + exportedPath = rootPath / "exports" / "workflow-export.json" + + assert exportedProtocolLists == [[protocol10, protocol11]] + assert exportedPath.exists() is True + + exportedText = exportedPath.read_text(encoding="utf-8") + assert exportedText.startswith("ScipionWeb metadata format: scipionweb.workflow.metadata") + assert "ScipionWeb metadata version: 1" in exportedText + assert "Scipion required plugins:" in exportedText + + exportedJsonText = service._extractWorkflowJsonText(exportedText) + assert json.loads(exportedJsonText) == [ + {"protocolId": 10}, + {"protocolId": 11}, + ] + + assert result == { + "success": True, + "path": str(exportedPath.resolve()), + "filename": "workflow-export.json", + "size": exportedPath.stat().st_size, + "mimeType": "application/json", + "protocolIds": ["500", "501"], + } + + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + assert mapper.db.fetchCalls[1]["params"] == (1, 501, "501") \ No newline at end of file From 21e1e739c675394f47f6168603a6a844700b0d81 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 16:06:14 +0200 Subject: [PATCH 145/278] Test runtime id resolution for file browser operations --- .../services/test_project_service_logs_fs.py | 166 +++++++++++++++++- 1 file changed, 165 insertions(+), 1 deletion(-) diff --git a/tests/unit/backend/api/services/test_project_service_logs_fs.py b/tests/unit/backend/api/services/test_project_service_logs_fs.py index b6acf838..77e28472 100644 --- a/tests/unit/backend/api/services/test_project_service_logs_fs.py +++ b/tests/unit/backend/api/services/test_project_service_logs_fs.py @@ -570,4 +570,168 @@ def test_PreviewProtocolImageFileDelegatesToProtocolBrowser(projectServiceModule } assert FakeFileHandlers.lastInstance.calls == [ ("previewProtocolImageFile", "10", "preview.webp", False), - ] \ No newline at end of file + ] + + +def test_GetProtocolPathResolvesPostgresqlProtocolId(service, tmp_path): + protocolPath = tmp_path / "DemoProject" / "Runs" / "000010_ProtImport" + protocolPath.mkdir(parents=True, exist_ok=True) + + service.currentProject.protocols[10] = FakeProtocol(protocolPath=str(protocolPath)) + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.getProtocolPath( + protocolId=500, + mapper=mapper, + projectId=1, + ) + + assert result == { + "rootAbs": str((tmp_path / "DemoProject").resolve()), + "startPath": "Runs/000010_ProtImport", + "protocolRoot": "Runs/000010_ProtImport", + } + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_ListProtocolDirResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "FileHandlers", FakeFileHandlers) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.listProtocolDir( + protocolId="500", + path="extra", + mapper=mapper, + projectId=1, + ) + + assert result == { + "mode": "protocol", + "protocolId": "10", + "path": "extra", + } + assert FakeFileHandlers.lastInstance.calls == [ + ("listProtocolDir", "10", "extra"), + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_PreviewProtocolTextFileResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "FileHandlers", FakeFileHandlers) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.previewProtocolTextFile( + protocolId="500", + path="notes.txt", + mapper=mapper, + projectId=1, + ) + + assert result == { + "mode": "protocol-text", + "protocolId": "10", + "path": "notes.txt", + } + assert FakeFileHandlers.lastInstance.calls == [ + ("previewProtocolTextFile", "10", "notes.txt"), + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_PreviewRemoteEntryResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "FileHandlers", FakeFileHandlers) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.previewRemoteEntry( + protocolId="500", + path="image.png", + mapper=mapper, + projectId=1, + ) + + assert result == { + "mode": "protocol-preview", + "protocolId": "10", + "path": "image.png", + } + assert FakeFileHandlers.lastInstance.calls == [ + ("previewProtocolRemoteEntry", "10", "image.png"), + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_PreviewProtocolImageFileResolvesPostgresqlProtocolId( + projectServiceModule, + service, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "FileHandlers", FakeFileHandlers) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + result = service.previewProtocolImageFile( + protocolId="500", + path="preview.webp", + inline=False, + mapper=mapper, + projectId=1, + ) + + assert result == { + "mode": "protocol-image", + "protocolId": "10", + "path": "preview.webp", + "inline": False, + } + assert FakeFileHandlers.lastInstance.calls == [ + ("previewProtocolImageFile", "10", "preview.webp", False), + ] + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + + +def test_WriteRemoteFileServiceResolvesPostgresqlProtocolId(service, tmp_path): + protocolPath = tmp_path / "DemoProject" / "Runs" / "000010_ProtImport" + protocolPath.mkdir(parents=True, exist_ok=True) + + service.currentProject.protocols[10] = FakeProtocol(protocolPath=str(protocolPath)) + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + class FakePayload: + path = "exports/result.json" + content = '{"ok": true}' + mimeType = "application/json" + + result = service.writeRemoteFileService( + protocolId=500, + payload=FakePayload(), + mapper=mapper, + projectId=1, + ) + + targetPath = tmp_path / "DemoProject" / "exports" / "result.json" + + assert targetPath.exists() is True + assert targetPath.read_text(encoding="utf-8") == '{"ok": true}' + assert result == { + "success": True, + "path": str(targetPath.resolve()), + "size": targetPath.stat().st_size, + "mimeType": "application/json", + } + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + From 962f18e6858e6cc584f0a2ea1dd1e8b179a5518b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 16:30:53 +0200 Subject: [PATCH 146/278] Test runtime id resolution for protocol params --- .../test_project_service_protocols.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index 39d9f5d3..59fa082d 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -113,6 +113,9 @@ def addParam(self, name, param): def getParam(self, name): return self._params.get(name) + def getPlugin(self): + return None + def setAttributeValue(self, name, value): self.attributeValues[name] = value @@ -175,6 +178,10 @@ def newProtocol(self, protClass): def getProtocol(self, protocolId): return self.protocols[int(protocolId)] + def _fixProtParamsConfiguration(self, protocol): + self.fixedProtocolParams = getattr(self, "fixedProtocolParams", []) + self.fixedProtocolParams.append(protocol) + def _storeProtocol(self, protocol): self.storedProtocols.append(protocol) @@ -1634,6 +1641,58 @@ def fakeGetSubworkflow(protocolObj): ] assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") +def test_GetProtocolParamsResolvesPostgresqlProtocolId(service, mapper, monkeypatch): + protocol = FakeProtocol(objId=10, className="ProtClass") + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + buildContextCalls = [] + + def fakeBuildProtocolContext(projectId, protocolObj): + buildContextCalls.append({ + "projectId": projectId, + "protocol": protocolObj, + }) + return { + "info": { + "projectId": projectId, + "protocolId": protocolObj.getObjId(), + "protocolClassName": "ProtClass", + }, + "form": { + "sections": [], + }, + "values": {}, + } + + monkeypatch.setattr(service, "_buildProtocolContext", fakeBuildProtocolContext) + + result = service.getProtocolParams( + mapper=mapper, + projectId=1, + protocolId=500, + ) + + assert result == { + "info": { + "projectId": 1, + "protocolId": 10, + "protocolClassName": "ProtClass", + }, + "form": { + "sections": [], + }, + "values": {}, + } + + assert service.currentProject.fixedProtocolParams == [protocol] + assert buildContextCalls == [ + { + "projectId": 1, + "protocol": protocol, + } + ] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") def test_SaveProtocolResolvesPostgresqlProtocolIdForExistingProtocol( projectServiceModule, From 42a0a0fff2786fbd55930ab28b3b2fbbba3998ab Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 17:06:29 +0200 Subject: [PATCH 147/278] Test runtime id resolution for next protocol suggestions --- .../test_project_service_protocols.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index 59fa082d..c51554a8 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -116,6 +116,9 @@ def getParam(self, name): def getPlugin(self): return None + def getClassName(self): + return self._className + def setAttributeValue(self, name, value): self.attributeValues[name] = value @@ -1641,6 +1644,80 @@ def fakeGetSubworkflow(protocolObj): ] assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + +def test_GetNextProtocolSuggestionsResolvesPostgresqlProtocolId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + protocol = FakeProtocol(objId=10, className="ProtImportMovies") + service.currentProject.protocols[10] = protocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + calledUrls = [] + + class FakeResponse: + def read(self): + return json.dumps([ + [ + "ProtLowerScore", + 3, + "Lower score protocol", + "scipion-em-lower", + "Lower score help", + ], + [ + "ProtHigherScore", + 9, + "Higher score protocol", + "scipion-em-higher", + "Higher score help", + ], + ]).encode("utf-8") + + def fakeUrlopen(url): + calledUrls.append(url) + return FakeResponse() + + class FakeConfig: + SCIPION_STATS_SUGGESTION = "https://example.test/suggestions/%s" + + @staticmethod + def getDomain(): + return FakeDomain({}) + + monkeypatch.setattr(projectServiceModule, "Config", FakeConfig) + monkeypatch.setattr(projectServiceModule, "urlopen", fakeUrlopen) + + result = service.getNextProtocolSuggestions( + mapper=mapper, + projectId=1, + protocolId=500, + ) + + assert calledUrls == [ + "https://example.test/suggestions/ProtImportMovies", + ] + + assert result == [ + { + "protocolName": "Higher score protocol", + "protocolClass": "ProtHigherScore", + "help": "Higher score help", + "installed": "Missing. Available in scipion-em-higher plugin.", + }, + { + "protocolName": "Lower score protocol", + "protocolClass": "ProtLowerScore", + "help": "Lower score help", + "installed": "Missing. Available in scipion-em-lower plugin.", + }, + ] + + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + def test_GetProtocolParamsResolvesPostgresqlProtocolId(service, mapper, monkeypatch): protocol = FakeProtocol(objId=10, className="ProtClass") service.currentProject.protocols[10] = protocol From 2506277b3eabc6aebe94ead6197748c0f04bdbc6 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 17:23:25 +0200 Subject: [PATCH 148/278] Test runtime id resolution for pointer params --- .../test_project_service_protocols.py | 159 +++++++++++++++++- 1 file changed, 158 insertions(+), 1 deletion(-) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index c51554a8..2c594918 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -87,6 +87,44 @@ class FakeCsvList(FakeBaseParam): pass +class FakePointerParam(FakeBaseParam): + def __init__(self, label="Pointer", choices=None, validationErrors=None, allowsNull=True, condition=None): + super().__init__( + label=label, + choices=choices, + validationErrors=validationErrors, + allowsNull=allowsNull, + condition=condition, + ) + self.default = FakeValueHolder(None) + + +class FakeMultiPointerParam(FakeBaseParam): + pass + + +class FakeRelationParam(FakeBaseParam): + pass + + +class FakePointerAttribute: + def __init__(self): + self.extended = None + + def setExtended(self, extended): + self.extended = extended + + +class FakePointerList(list): + def isEmpty(self): + return len(self) == 0 + + +class FakePointer: + def __init__(self, protocol, extended=None): + self.protocol = protocol + self.extended = extended + class FakeProtocol: # fakeProtocol def __init__(self, objId=None, className="ProtClass", useQueueFlag=False, validateErrors=None): @@ -2206,4 +2244,123 @@ class FakeImportPayload: "inputFromCopiedProtocol": "1.outputParticles", }, ] - ] \ No newline at end of file + ] + + +def test_ApplyParamsToProtocolResolvesPostgresqlPointerParentId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "PointerParam", FakePointerParam) + monkeypatch.setattr(projectServiceModule, "MultiPointerParam", FakeMultiPointerParam) + monkeypatch.setattr(projectServiceModule, "RelationParam", FakeRelationParam) + + parentProtocol = FakeProtocol(objId=10, className="ParentProtocol") + parentProtocol.outputParticles = object() + + protocol = FakeProtocol(objId=20, className="ChildProtocol") + pointerParam = FakePointerParam(label="Input particles") + protocol.addParam("inputParticles", pointerParam) + protocol.inputParticles = FakePointerAttribute() + + service.currentProject.protocols[10] = parentProtocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + errors = service.applyParamsToProtocol( + mapper=mapper, + projectId=1, + protocol=protocol, + params={ + "inputParticles": "500.outputParticles", + }, + ) + + assert errors == [] + assert pointerParam.get() == "10.outputParticles" + assert pointerParam.default.get() == "10.outputParticles" + assert protocol.attributeValues["inputParticles"] is parentProtocol + assert protocol.inputParticles.extended == "outputParticles" + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + +def test_SetPointerParamResolvesPostgresqlPointerParentId( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "PointerParam", FakePointerParam) + + parentProtocol = FakeProtocol(objId=10, className="ParentProtocol") + protocol = FakeProtocol(objId=20, className="ChildProtocol") + + pointerParam = FakePointerParam(label="Input volume") + protocol.addParam("inputVolume", pointerParam) + + service.currentProject.protocols[10] = parentProtocol + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + service.setPointerParam( + mapper=mapper, + projectId=1, + protocol=protocol, + key="inputVolume", + value={ + "editableValue": "500.outputVolume", + }, + parentId=500, + ) + + assert pointerParam.get() == "10.outputVolume" + assert pointerParam.default.get() == "10.outputVolume" + assert protocol.attributeValues["inputVolume"] is parentProtocol + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + +def test_ApplyParamsToProtocolResolvesPostgresqlMultiPointerParentIds( + projectServiceModule, + service, + mapper, + monkeypatch, +): + monkeypatch.setattr(projectServiceModule, "PointerParam", FakePointerParam) + monkeypatch.setattr(projectServiceModule, "MultiPointerParam", FakeMultiPointerParam) + monkeypatch.setattr(projectServiceModule, "RelationParam", FakeRelationParam) + monkeypatch.setattr(projectServiceModule, "PointerList", FakePointerList) + monkeypatch.setattr(projectServiceModule, "Pointer", FakePointer) + + parentProtocolA = FakeProtocol(objId=10, className="ParentProtocolA") + parentProtocolB = FakeProtocol(objId=11, className="ParentProtocolB") + + protocol = FakeProtocol(objId=20, className="ChildProtocol") + multiPointerParam = FakeMultiPointerParam(label="Input sets") + protocol.addParam("inputSets", multiPointerParam) + + service.currentProject.protocols[10] = parentProtocolA + service.currentProject.protocols[11] = parentProtocolB + + mapper.db.runtimeProtocolIdByDbId[500] = 10 + mapper.db.runtimeProtocolIdByDbId[501] = 11 + + errors = service.applyParamsToProtocol( + mapper=mapper, + projectId=1, + protocol=protocol, + params={ + "inputSets": [ + "500.outputParticles", + "501.outputClasses", + ], + }, + ) + + assert errors == [] + assert len(protocol.attributeValues["inputSets"]) == 2 + assert protocol.attributeValues["inputSets"][0].protocol is parentProtocolA + assert protocol.attributeValues["inputSets"][0].extended == "outputParticles" + assert protocol.attributeValues["inputSets"][1].protocol is parentProtocolB + assert protocol.attributeValues["inputSets"][1].extended == "outputClasses" + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + assert mapper.db.fetchOneCalls[1]["params"] == (1, 501, "501") + + From b6d030593c15f9ec487ba0a60a145fc7d22a7fa8 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 17:33:28 +0200 Subject: [PATCH 149/278] Test runtime id resolution for protocol wizards --- .../services/test_protocol_wizard_service.py | 501 ++++++++++++++++++ 1 file changed, 501 insertions(+) create mode 100644 tests/unit/backend/api/services/test_protocol_wizard_service.py diff --git a/tests/unit/backend/api/services/test_protocol_wizard_service.py b/tests/unit/backend/api/services/test_protocol_wizard_service.py new file mode 100644 index 00000000..3fd298ec --- /dev/null +++ b/tests/unit/backend/api/services/test_protocol_wizard_service.py @@ -0,0 +1,501 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +import importlib + +import pytest +from fastapi import HTTPException + + +class FakeValueHolder: + def __init__(self, value=None): + self.value = value + + def get(self): + return self.value + + def set(self, value): + self.value = value + + +class FakeParam: + def __init__(self, label="Param", validationErrors=None): + self.label = FakeValueHolder(label) + self.validationErrors = validationErrors or [] + self.value = None + + def get(self): + return self.value + + def set(self, value): + self.value = value + + def validate(self, value): + return list(self.validationErrors) + + +class FakeProtocol: + def __init__(self, objId=None, className="ProtClass"): + self._objId = objId + self._className = className + self._params = {} + self.attributeValues = {} + self.label = None + + def addParam(self, name, param): + self._params[name] = param + + def getParam(self, name): + return self._params.get(name) + + def getObjId(self): + return self._objId + + def getClassName(self): + return self._className + + def setObjLabel(self, value): + self.label = value + + def setAttributeValue(self, name, value): + self.attributeValues[name] = value + + +class FakeProtocolFactory: + def __init__(self, protocol): + self.protocol = protocol + + def __call__(self): + return self.protocol + + +class FakeDomain: + def __init__(self, protocols=None, wizards=None): + self.protocols = protocols or {} + self.wizards = wizards or {} + + def getProtocols(self): + return self.protocols + + def getWizards(self): + return self.wizards + + +class FakeCurrentProject: + def __init__(self): + self.protocols = {} + self.protocolFactories = {} + self.fixedProtocolParams = [] + self.createdProtocols = [] + + def getDomain(self): + return FakeDomain(protocols=self.protocolFactories) + + def getProtocol(self, protocolId): + return self.protocols[int(protocolId)] + + def newProtocol(self, protClass): + protocol = protClass() + self.createdProtocols.append(protocol) + return protocol + + def _fixProtParamsConfiguration(self, protocol): + self.fixedProtocolParams.append(protocol) + + +class FakeMapper: + pass + + +class FakeProjectService: + def __init__(self, currentProject): + self.currentProject = currentProject + self.runtimeProtocolIdByDbId = {} + self.runtimeCalls = [] + self.castParamValueCalls = [] + self.applyParamsToProtocolCalls = [] + self.getProjectByIdCalls = [] + self.projectRow = {"id": 1} + + def _getScipionProtocolForRuntime(self, mapper, projectId, protocolId): + self.runtimeCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + }) + + runtimeProtocolId = self.runtimeProtocolIdByDbId.get(int(protocolId), int(protocolId)) + return self.currentProject.protocols[int(runtimeProtocolId)] + + def castParamValue(self, param, value): + self.castParamValueCalls.append({ + "param": param, + "value": value, + }) + + if isinstance(value, str) and value.isdigit(): + return int(value) + + return value + + def applyParamsToProtocol(self, mapper, projectId, protocol, params): + self.applyParamsToProtocolCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocol": protocol, + "params": dict(params or {}), + }) + return [] + + def getProjectById(self, mapper, projectId, currentUser, refresh=False, checkPid=False): + self.getProjectByIdCalls.append({ + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + "refresh": refresh, + "checkPid": checkPid, + }) + return self.projectRow + + +class FakePayload: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + +class FakeWizardClass: + pass + + +@pytest.fixture +def protocolWizardServiceModule(authTestEnv): + return importlib.import_module("app.backend.api.services.protocol_wizard_service") + + +@pytest.fixture +def currentProject(): + return FakeCurrentProject() + + +@pytest.fixture +def projectService(currentProject): + return FakeProjectService(currentProject) + + +@pytest.fixture +def wizardService(protocolWizardServiceModule, currentProject, projectService): + return protocolWizardServiceModule.ProtocolWizardService( + currentProject=currentProject, + projectService=projectService, + ) + + +@pytest.fixture +def mapper(): + return FakeMapper() + + +def test_SanitizeWizardFormValuesDropsEmptyValues(wizardService): + result = wizardService._sanitizeWizardFormValues({ + "noneValue": None, + "blankValue": "", + "spacesValue": " ", + "zeroValue": 0, + "falseValue": False, + "textValue": "value", + }) + + assert result == { + "zeroValue": 0, + "falseValue": False, + "textValue": "value", + } + + +def test_BuildWizardReadyProtocolResolvesPostgresqlProtocolId( + wizardService, + currentProject, + projectService, + mapper, +): + protocol = FakeProtocol(objId=10, className="ProtWizardTarget") + iterationsParam = FakeParam(label="Iterations") + protocol.addParam("iterations", iterationsParam) + + currentProject.protocols[10] = protocol + projectService.runtimeProtocolIdByDbId[500] = 10 + + result = wizardService._buildWizardReadyProtocol( + protocolId=500, + protocolClassName="ProtWizardTarget", + formValues={ + "iterations": "7", + }, + mapper=mapper, + projectId=1, + ) + + assert result is protocol + assert currentProject.fixedProtocolParams == [protocol] + assert iterationsParam.get() == 7 + assert protocol.attributeValues["iterations"] == 7 + + assert projectService.runtimeCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + + assert projectService.applyParamsToProtocolCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocol": protocol, + "params": { + "iterations": "7", + }, + } + ] + + +def test_BuildWizardReadyProtocolCreatesNewProtocolWhenProtocolIdIsMissing( + wizardService, + currentProject, + projectService, + mapper, +): + protocol = FakeProtocol(objId=None, className="ProtNewWizardTarget") + runNameParam = FakeParam(label="Run name") + protocol.addParam("runName", runNameParam) + + currentProject.protocolFactories["ProtNewWizardTarget"] = FakeProtocolFactory(protocol) + + result = wizardService._buildWizardReadyProtocol( + protocolId=None, + protocolClassName="ProtNewWizardTarget", + formValues={ + "runName": "New protocol", + }, + mapper=mapper, + projectId=1, + ) + + assert result is protocol + assert currentProject.createdProtocols == [protocol] + assert currentProject.fixedProtocolParams == [protocol] + assert runNameParam.get() == "New protocol" + assert protocol.attributeValues["runName"] == "New protocol" + assert protocol.label == "New protocol" + assert projectService.runtimeCalls == [] + + +def test_BuildWizardReadyProtocolRaisesWhenNewProtocolClassIsMissing( + wizardService, + mapper, +): + with pytest.raises(HTTPException) as exc: + wizardService._buildWizardReadyProtocol( + protocolId=None, + protocolClassName="MissingProtocol", + formValues={}, + mapper=mapper, + projectId=1, + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Protocol class 'MissingProtocol' not found" + + +def test_ExecuteProtocolWizardResolvesPostgresqlProtocolId( + protocolWizardServiceModule, + wizardService, + currentProject, + projectService, + mapper, + monkeypatch, +): + protocol = FakeProtocol(objId=10, className="ProtWizardTarget") + iterationsParam = FakeParam(label="Iterations") + protocol.addParam("iterations", iterationsParam) + + currentProject.protocols[10] = protocol + projectService.runtimeProtocolIdByDbId[500] = 10 + + descriptor = { + "id": "tests.FakeWizardClass", + "kind": "compute", + "targetParams": ["boxSize"], + } + + monkeypatch.setattr( + wizardService, + "_resolveWizardDescriptorForParam", + lambda protocol, paramName, wizardId: descriptor, + ) + monkeypatch.setattr( + wizardService, + "_getWizardClassById", + lambda wizardId: FakeWizardClass, + ) + + handlerCalls = [] + + def fakeExecuteWizardHandler( + kind, + wizardClass, + protocol, + paramName, + descriptor, + wizardInputs, + currentProject, + projectId, + ): + handlerCalls.append({ + "kind": kind, + "wizardClass": wizardClass, + "protocol": protocol, + "paramName": paramName, + "descriptor": descriptor, + "wizardInputs": wizardInputs, + "currentProject": currentProject, + "projectId": projectId, + }) + return { + "paramUpdates": { + "boxSize": 128, + }, + "message": "Box size computed", + "preview": { + "enabled": True, + }, + } + + monkeypatch.setattr( + protocolWizardServiceModule, + "executeWizardHandler", + fakeExecuteWizardHandler, + ) + + payload = FakePayload( + protocolId=500, + protocolClassName="ProtWizardTarget", + formValues={ + "iterations": "7", + }, + paramName="boxSize", + wizardId="tests.FakeWizardClass", + wizardInputs={ + "diameter": 180, + }, + ) + + result = wizardService.executeProtocolWizard( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=payload, + ) + + assert result == { + "success": True, + "wizardId": "tests.FakeWizardClass", + "kind": "compute", + "paramUpdates": { + "boxSize": 128, + }, + "message": "Box size computed", + "preview": { + "enabled": True, + }, + } + + assert projectService.getProjectByIdCalls == [ + { + "mapper": mapper, + "projectId": 1, + "currentUser": {"id": 1}, + "refresh": False, + "checkPid": False, + } + ] + + assert projectService.runtimeCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + + assert currentProject.fixedProtocolParams == [protocol] + assert iterationsParam.get() == 7 + assert protocol.attributeValues["iterations"] == 7 + + assert handlerCalls == [ + { + "kind": "compute", + "wizardClass": FakeWizardClass, + "protocol": protocol, + "paramName": "boxSize", + "descriptor": descriptor, + "wizardInputs": { + "diameter": 180, + }, + "currentProject": currentProject, + "projectId": 1, + } + ] + + +def test_ExecuteProtocolWizardReturns404WhenProjectDoesNotExist( + wizardService, + projectService, + mapper, +): + projectService.projectRow = None + + payload = FakePayload( + protocolId=500, + protocolClassName="ProtWizardTarget", + formValues={}, + paramName="boxSize", + wizardId="tests.FakeWizardClass", + wizardInputs={}, + ) + + with pytest.raises(HTTPException) as exc: + wizardService.executeProtocolWizard( + mapper=mapper, + projectId=1, + currentUser={"id": 1}, + payload=payload, + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Project not found" \ No newline at end of file From c65bf476a781159fab242ef73f3b15c1437b8a75 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 17:44:06 +0200 Subject: [PATCH 150/278] Test runtime id resolution for coords2d outputs --- .../api/services/test_coords2d_service.py | 580 ++++++++++++++++++ 1 file changed, 580 insertions(+) create mode 100644 tests/unit/backend/api/services/test_coords2d_service.py diff --git a/tests/unit/backend/api/services/test_coords2d_service.py b/tests/unit/backend/api/services/test_coords2d_service.py new file mode 100644 index 00000000..0c028c62 --- /dev/null +++ b/tests/unit/backend/api/services/test_coords2d_service.py @@ -0,0 +1,580 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +import importlib + +import pytest +from fastapi import HTTPException + + +class FakeMapper: + pass + + +class FakeProjectService: + def __init__(self, currentProject): + self.currentProject = currentProject + self.projectRow = {"id": 1} + self.runtimeProtocolIdByDbId = {} + self.getProjectByIdCalls = [] + self.runtimeCalls = [] + + def getProjectById(self, mapper, projectId, currentUser, refresh=False, checkPid=False): + self.getProjectByIdCalls.append({ + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + "refresh": refresh, + "checkPid": checkPid, + }) + return self.projectRow + + def _getScipionProtocolForRuntime(self, mapper, projectId, protocolId): + self.runtimeCalls.append({ + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + }) + + runtimeProtocolId = self.runtimeProtocolIdByDbId.get( + int(protocolId), + int(protocolId), + ) + return self.currentProject.protocols[int(runtimeProtocolId)] + + +class FakeCurrentProject: + def __init__(self): + self.protocols = {} + + +class FakeProtocol: + def __init__(self, objId=10): + self.objId = objId + + def getObjId(self): + return self.objId + + +class FakeMicrograph: + def __init__( + self, + objId, + location=None, + fileName=None, + micName=None, + label=None, + dims=None, + ): + self.objId = objId + self.location = location + self.fileName = fileName + self.micName = micName + self.label = label + self.dims = dims or (4096, 4096) + + def clone(self): + return FakeMicrograph( + objId=self.objId, + location=self.location, + fileName=self.fileName, + micName=self.micName, + label=self.label, + dims=self.dims, + ) + + def getObjId(self): + return self.objId + + def getLocation(self): + return self.location + + def getFileName(self): + return self.fileName + + def getMicName(self): + return self.micName + + def getObjLabel(self): + return self.label + + def getDim(self): + return self.dims + + +class FakeMicrographsSet: + def __init__(self, micrographs): + self.micrographs = list(micrographs) + + def iterItems(self, iterate=False): + return iter(self.micrographs) + + def __getitem__(self, micId): + for micrograph in self.micrographs: + if int(micrograph.getObjId()) == int(micId): + return micrograph + + raise KeyError(micId) + + +class FakeCoordinate: + def __init__( + self, + objId, + micId, + x, + y, + score=None, + classId=None, + ): + self.objId = objId + self.micId = micId + self.x = x + self.y = y + self.score = score + self.classId = classId + + def getObjId(self): + return self.objId + + def getMicId(self): + return self.micId + + def getX(self): + return self.x + + def getY(self): + return self.y + + def getScore(self): + return self.score + + def getClassId(self): + return self.classId + + def clone(self): + return FakeCoordinate( + objId=self.objId, + micId=self.micId, + x=self.x, + y=self.y, + score=self.score, + classId=self.classId, + ) + + +class FakeCoordinatesSet: + def __init__( + self, + micrographs, + coordinates, + boxSize=64, + ): + self.micrographsSet = FakeMicrographsSet(micrographs) + self.coordinates = list(coordinates) + self.boxSize = boxSize + + def getMicrographs(self): + return self.micrographsSet + + def iterItems(self, iterate=False): + return iter(self.coordinates) + + def iterCoordinates(self, micId): + return ( + coordinate + for coordinate in self.coordinates + if str(coordinate.getMicId()) == str(micId) + ) + + def getBoxSize(self): + return self.boxSize + + def getSize(self): + return len(self.coordinates) + + def getFirstItem(self): + return self.coordinates[0] if self.coordinates else None + + +@pytest.fixture +def coords2dServiceModule(authTestEnv): + return importlib.import_module("app.backend.api.services.coords2d_service") + + +@pytest.fixture +def currentProject(): + return FakeCurrentProject() + + +@pytest.fixture +def projectService(currentProject): + return FakeProjectService(currentProject) + + +@pytest.fixture +def service(coords2dServiceModule, projectService): + instance = object.__new__(coords2dServiceModule.Coords2dService) + instance.projectService = projectService + return instance + + +@pytest.fixture +def mapper(): + return FakeMapper() + + +@pytest.fixture +def currentUser(): + return {"id": 1} + + +def buildCoordinatesOutput(): + micrographA = FakeMicrograph( + objId=1, + location=(0, "/data/micrograph-a.mrc"), + micName="Micrograph A", + dims=(4096, 3072), + ) + micrographB = FakeMicrograph( + objId=2, + location="1@/data/micrograph-b.mrc", + label="Micrograph B label", + dims=(2048, 2048), + ) + + coordinateA = FakeCoordinate( + objId=101, + micId=1, + x=10.5, + y=20.5, + score=0.95, + classId="good", + ) + coordinateB = FakeCoordinate( + objId=102, + micId=1, + x=30.0, + y=40.0, + score=0.75, + classId="bad", + ) + coordinateC = FakeCoordinate( + objId=201, + micId=2, + x=15.0, + y=25.0, + score=0.60, + classId="ok", + ) + + return FakeCoordinatesSet( + micrographs=[micrographA, micrographB], + coordinates=[coordinateA, coordinateB, coordinateC], + boxSize=96, + ) + + +def test_LoadCoordinatesOutputResolvesPostgresqlProtocolId( + service, + currentProject, + projectService, + mapper, + currentUser, +): + protocol = FakeProtocol(objId=10) + coordinatesSet = buildCoordinatesOutput() + protocol.outputCoordinates = coordinatesSet + + currentProject.protocols[10] = protocol + projectService.runtimeProtocolIdByDbId[500] = 10 + + loadedProtocol, loadedCoordinatesSet = service._loadCoordinatesOutput( + mapper=mapper, + projectId=1, + currentUser=currentUser, + protocolId=500, + outputName="outputCoordinates", + ) + + assert loadedProtocol is protocol + assert loadedCoordinatesSet is coordinatesSet + + assert projectService.getProjectByIdCalls == [ + { + "mapper": mapper, + "projectId": 1, + "currentUser": currentUser, + "refresh": False, + "checkPid": False, + } + ] + + assert projectService.runtimeCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + + +def test_LoadCoordinatesOutputRaisesWhenProjectDoesNotExist( + service, + projectService, + mapper, + currentUser, +): + projectService.projectRow = None + + with pytest.raises(HTTPException) as exc: + service._loadCoordinatesOutput( + mapper=mapper, + projectId=1, + currentUser=currentUser, + protocolId=500, + outputName="outputCoordinates", + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Project not found" + assert projectService.runtimeCalls == [] + + +def test_LoadCoordinatesOutputRaisesWhenOutputIsMissing( + service, + currentProject, + projectService, + mapper, + currentUser, +): + protocol = FakeProtocol(objId=10) + + currentProject.protocols[10] = protocol + projectService.runtimeProtocolIdByDbId[500] = 10 + + with pytest.raises(HTTPException) as exc: + service._loadCoordinatesOutput( + mapper=mapper, + projectId=1, + currentUser=currentUser, + protocolId=500, + outputName="missingOutput", + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Output 'missingOutput' not found in protocol '500'" + + assert projectService.runtimeCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + + +def test_LoadCoordinatesOutputRaisesWhenOutputIsNotCoordinatesSet( + service, + currentProject, + projectService, + mapper, + currentUser, +): + protocol = FakeProtocol(objId=10) + protocol.outputVolume = object() + + currentProject.protocols[10] = protocol + projectService.runtimeProtocolIdByDbId[500] = 10 + + with pytest.raises(HTTPException) as exc: + service._loadCoordinatesOutput( + mapper=mapper, + projectId=1, + currentUser=currentUser, + protocolId=500, + outputName="outputVolume", + ) + + assert exc.value.status_code == 422 + assert exc.value.detail == "Output 'outputVolume' is not a SetOfCoordinates output" + + assert projectService.runtimeCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + + +def test_ListMicrographsResolvesPostgresqlProtocolId( + service, + currentProject, + projectService, + mapper, + currentUser, +): + protocol = FakeProtocol(objId=10) + protocol.outputCoordinates = buildCoordinatesOutput() + + currentProject.protocols[10] = protocol + projectService.runtimeProtocolIdByDbId[500] = 10 + + result = service.listMicrographs( + mapper=mapper, + projectId=1, + currentUser=currentUser, + protocolId=500, + outputName="outputCoordinates", + ) + + assert result == { + "micrographs": [ + { + "id": "1", + "index": 1, + "fileName": "/data/micrograph-a.mrc", + "label": "Micrograph A", + "particles": 2, + "updated": False, + "width": 4096, + "height": 3072, + "locationIndex": 0, + "thumbnailUrl": None, + }, + { + "id": "2", + "index": 2, + "fileName": "/data/micrograph-b.mrc", + "label": "Micrograph B label", + "particles": 1, + "updated": False, + "width": 2048, + "height": 2048, + "locationIndex": 1, + "thumbnailUrl": None, + }, + ], + "totalMicrographs": 2, + "totalPicks": 3, + "boxSize": 96, + } + + assert projectService.runtimeCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + + +def test_ListCoordinatesForMicrographResolvesPostgresqlProtocolId( + service, + currentProject, + projectService, + mapper, + currentUser, +): + protocol = FakeProtocol(objId=10) + protocol.outputCoordinates = buildCoordinatesOutput() + + currentProject.protocols[10] = protocol + projectService.runtimeProtocolIdByDbId[500] = 10 + + result = service.listCoordinatesForMicrograph( + mapper=mapper, + projectId=1, + currentUser=currentUser, + protocolId=500, + outputName="outputCoordinates", + micId="1", + ) + + assert result == { + "coordinates": [ + { + "id": 101, + "micId": "1", + "x": 10.5, + "y": 20.5, + "score": 0.95, + "classLabel": "good", + }, + { + "id": 102, + "micId": "1", + "x": 30.0, + "y": 40.0, + "score": 0.75, + "classLabel": "bad", + }, + ], + } + + assert projectService.runtimeCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + + +def test_ListCoordinatesForMicrographRaisesWhenMicrographIsMissing( + service, + currentProject, + projectService, + mapper, + currentUser, +): + protocol = FakeProtocol(objId=10) + protocol.outputCoordinates = buildCoordinatesOutput() + + currentProject.protocols[10] = protocol + projectService.runtimeProtocolIdByDbId[500] = 10 + + with pytest.raises(HTTPException) as exc: + service.listCoordinatesForMicrograph( + mapper=mapper, + projectId=1, + currentUser=currentUser, + protocolId=500, + outputName="outputCoordinates", + micId="999", + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Micrograph '999' not found in coordinates output" + + assert projectService.runtimeCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] \ No newline at end of file From de0ec8596c586c2f26c3f1f9d58d483e4d7e9a1e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 19:28:57 +0200 Subject: [PATCH 151/278] Remove legacy protocols router from API --- app/backend/api/routers/protocol_router.py | 40 +++++++++++----------- app/backend/main.py | 4 +-- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/app/backend/api/routers/protocol_router.py b/app/backend/api/routers/protocol_router.py index 68769ffc..b79d261d 100644 --- a/app/backend/api/routers/protocol_router.py +++ b/app/backend/api/routers/protocol_router.py @@ -69,26 +69,26 @@ async def loadNewProtocol( return service.getNewProtocolParams(projectId, protClassName) -@router.post("/launch", response_model=Any) -async def launchProtocol(request: ProtocolRequest): - try: - protocolId = request.getProtocolId() - protocolClassName = request.getProtocolClassName() - params = request.getParams() - service.launchProtocol(protocolId, protocolClassName, params) - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@router.post("/save", response_model=Any) -async def saveProtocol(request: ProtocolRequest): - try: - protocolId = request.getProtocolId() - protocolClassName = request.getProtocolClassName() - params = request.getParams() - service.saveProtocol(protocolId, protocolClassName, params) - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) +# @router.post("/launch", response_model=Any) +# async def launchProtocol(request: ProtocolRequest): +# try: +# protocolId = request.getProtocolId() +# protocolClassName = request.getProtocolClassName() +# params = request.getParams() +# service.launchProtocol(protocolId, protocolClassName, params) +# except Exception as e: +# raise HTTPException(status_code=500, detail=str(e)) +# +# +# @router.post("/save", response_model=Any) +# async def saveProtocol(request: ProtocolRequest): +# try: +# protocolId = request.getProtocolId() +# protocolClassName = request.getProtocolClassName() +# params = request.getParams() +# service.saveProtocol(protocolId, protocolClassName, params) +# except Exception as e: +# raise HTTPException(status_code=500, detail=str(e)) @router.get("/logs/{projectId}/{protocolId}/{offset}/{errOffset}/{scheduleOffset}", response_model=Any) diff --git a/app/backend/main.py b/app/backend/main.py index 7ad0aa20..644087e4 100644 --- a/app/backend/main.py +++ b/app/backend/main.py @@ -39,7 +39,7 @@ from fastapi.middleware.cors import CORSMiddleware from app.backend.api.routers.project_router import router as projects -from app.backend.api.routers.protocol_router import router as protocols +# from app.backend.api.routers.protocol_router import router as protocols from app.backend.api.routers.plugin_router import router as plugins from app.backend.api.routers.auth_router import router as auth from app.backend.api.routers.user_router import router as users @@ -130,7 +130,7 @@ def _buildApiApp() -> FastAPI: # includeRouters apiApp.include_router(projects) - apiApp.include_router(protocols) + # apiApp.include_router(protocols) apiApp.include_router(plugins) apiApp.include_router(auth) apiApp.include_router(users) From dca673991e022bb00241fc8f8c8c1f3164b5a49b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 22:29:41 +0200 Subject: [PATCH 152/278] Resolve protocol tag assignments by database id --- app/backend/api/services/project_service.py | 65 +++++++++++++++---- app/backend/mapper/postgresql.py | 46 +++++++++---- .../api/services/test_project_service_tags.py | 25 +++++++ 3 files changed, 110 insertions(+), 26 deletions(-) create mode 100644 tests/unit/backend/api/services/test_project_service_tags.py diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index b70e8a4e..098af2d3 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -12966,33 +12966,72 @@ def deleteProjectTag( ) def listProtocolTags( - self, - mapper, - projectId: int, - protocolId: int, - currentUser: dict, + self, + mapper, + projectId: int, + protocolId: int, + currentUser: dict, ) -> Dict[str, Any]: # listProtocolTags + protocolDbId = self._resolvePostgresqlProtocolDbId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + if protocolDbId is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol not found in PostgreSQL: {protocolId}", + ) + try: - tagIds = mapper.getProtocolTagIds(projectId=projectId, protocolDbId=int(protocolId)) + tagIds = mapper.getProtocolTagIds( + projectId=projectId, + protocolDbId=protocolDbId, + ) except Exception as e: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to list protocol tags: {e}", ) - return {"tagIds": tagIds} + return { + "protocolId": str(protocolId), + "protocolDbId": protocolDbId, + "tagIds": tagIds, + } def setProtocolTags( - self, - mapper, - projectId: int, - protocolId: int, - tagIds: List[str], - currentUser: dict, + self, + mapper, + projectId: int, + protocolId: int, + tagIds: List[str], + currentUser: dict, ) -> Dict[str, Any]: # setProtocolTags + protocolDbId = self._resolvePostgresqlProtocolDbId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + if protocolDbId is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Protocol not found in PostgreSQL: {protocolId}", + ) + try: + setByDbId = getattr(mapper, "setProtocolTagIdsByProtocolDbId", None) + if callable(setByDbId): + return setByDbId( + projectId=projectId, + protocolDbId=protocolDbId, + tagIds=tagIds or [], + ) + return mapper.setProtocolTagIds( projectId=projectId, protocolId=int(protocolId), diff --git a/app/backend/mapper/postgresql.py b/app/backend/mapper/postgresql.py index 00a66d90..ba03e853 100644 --- a/app/backend/mapper/postgresql.py +++ b/app/backend/mapper/postgresql.py @@ -394,35 +394,31 @@ def getProtocolTagIds(self, projectId: int, protocolDbId: int) -> List[str]: ) return [r["tagId"] for r in rows if r.get("tagId")] - def setProtocolTagIds(self, projectId: int, protocolId: int, tagIds: List[str]) -> dict: - # setProtocolTagIds + def setProtocolTagIdsByProtocolDbId(self, projectId: int, protocolDbId: int, tagIds: List[str]) -> dict: + # setProtocolTagIdsByProtocolDbId clean = sorted({str(t).strip() for t in (tagIds or []) if str(t).strip()}) - # validateProtocolBelongsToProject row = self.db.fetchOne( """ - SELECT "id" - FROM "protocols" - WHERE "protocolId" = %s + SELECT id, "protocolId" + FROM protocols + WHERE id = %s AND "projectId" = %s """, - (str(protocolId), projectId), + (int(protocolDbId), projectId), ) if not row: raise Exception("Protocol not found in project") - # deleteExistingAssignments - protocolDbId = row['id'] self.db.execute( """ DELETE FROM protocol_tag_assignments WHERE "protocolDbId" = %s """, - (protocolDbId,), + (int(protocolDbId),), ) if clean: - # bulkInsertWithUnnest self.db.execute( """ INSERT INTO protocol_tag_assignments ("protocolDbId", "tagId") @@ -430,10 +426,34 @@ def setProtocolTagIds(self, projectId: int, protocolId: int, tagIds: List[str]) FROM unnest(%s::text[]) AS x ON CONFLICT ("protocolDbId", "tagId") DO NOTHING """, - (protocolDbId, clean), + (int(protocolDbId), clean), ) - return {"protocolId": protocolDbId, "tagIds": clean} + return { + "protocolId": str(row["protocolId"]), + "protocolDbId": int(protocolDbId), + "tagIds": clean, + } + + def setProtocolTagIds(self, projectId: int, protocolId: int, tagIds: List[str]) -> dict: + # setProtocolTagIds + row = self.db.fetchOne( + """ + SELECT id + FROM protocols + WHERE "protocolId" = %s + AND "projectId" = %s + """, + (str(protocolId), projectId), + ) + if not row: + raise Exception("Protocol not found in project") + + return self.setProtocolTagIdsByProtocolDbId( + projectId=projectId, + protocolDbId=int(row["id"]), + tagIds=tagIds, + ) def getProjectProtocolTagIdsByProtocolId(self, projectId: int, includeEmpty: bool = False) -> Dict[str, List[str]]: # getProjectProtocolTagIdsByProtocolId diff --git a/tests/unit/backend/api/services/test_project_service_tags.py b/tests/unit/backend/api/services/test_project_service_tags.py new file mode 100644 index 00000000..4cdb527e --- /dev/null +++ b/tests/unit/backend/api/services/test_project_service_tags.py @@ -0,0 +1,25 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** From 3f922b505f05c66a5fd14c70214afc554bcddca1 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 22:43:13 +0200 Subject: [PATCH 153/278] Test protocol tag id resolution --- .../api/services/test_project_service_tags.py | 443 ++++++++++++++++++ 1 file changed, 443 insertions(+) diff --git a/tests/unit/backend/api/services/test_project_service_tags.py b/tests/unit/backend/api/services/test_project_service_tags.py index 4cdb527e..0c4f4869 100644 --- a/tests/unit/backend/api/services/test_project_service_tags.py +++ b/tests/unit/backend/api/services/test_project_service_tags.py @@ -23,3 +23,446 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ****************************************************************************** + +import importlib + +import pytest +from fastapi import HTTPException + + +class FakeDb: + def __init__(self): + self.fetchOneCalls = [] + self.protocolRows = [] + + def addProtocol(self, projectId, protocolDbId, protocolId): + self.protocolRows.append({ + "projectId": int(projectId), + "id": int(protocolDbId), + "protocolId": str(protocolId), + }) + + def fetchOne(self, query, params=None): + self.fetchOneCalls.append({ + "query": query, + "params": params, + }) + + if params is None: + return None + + queryText = " ".join(str(query).split()) + + if "FROM protocols" not in queryText: + return None + + # ProjectService._resolvePostgresqlProtocolDbId: + # SELECT id FROM protocols WHERE "projectId" = %s + # AND (id = %s OR "protocolId" = %s) + if len(params) == 3: + projectId, protocolDbIdCandidate, rawProtocolId = params + projectId = int(projectId) + rawProtocolId = str(rawProtocolId) + + for row in self.protocolRows: + if row["projectId"] != projectId: + continue + + if row["id"] == int(protocolDbIdCandidate): + return {"id": row["id"]} + + if row["protocolId"] == rawProtocolId: + return {"id": row["id"]} + + return None + + # ProjectService._resolvePostgresqlProtocolDbId for non-integer values: + # SELECT id FROM protocols WHERE "projectId" = %s + # AND "protocolId" = %s + if len(params) == 2 and '"projectId" = %s' in queryText and '"protocolId" = %s' in queryText: + projectId, rawProtocolId = params + projectId = int(projectId) + rawProtocolId = str(rawProtocolId) + + for row in self.protocolRows: + if row["projectId"] == projectId and row["protocolId"] == rawProtocolId: + return {"id": row["id"]} + + return None + + def getRuntimeProtocolId(self, projectId, protocolDbId): + for row in self.protocolRows: + if row["projectId"] == int(projectId) and row["id"] == int(protocolDbId): + return row["protocolId"] + + return str(protocolDbId) + + +class FakeMapper: + def __init__(self): + self.db = FakeDb() + self.protocolTagIdsByProtocolDbId = {} + self.getProtocolTagIdsCalls = [] + self.setProtocolTagIdsByProtocolDbIdCalls = [] + self.setProtocolTagIdsCalls = [] + self.raiseOnGetProtocolTagIds = None + self.raiseOnSetProtocolTagIdsByProtocolDbId = None + self.projectTags = [] + + def listProjectTags(self, projectId): + return list(self.projectTags) + + def getProtocolTagIds(self, projectId, protocolDbId): + self.getProtocolTagIdsCalls.append({ + "projectId": projectId, + "protocolDbId": protocolDbId, + }) + + if self.raiseOnGetProtocolTagIds is not None: + raise self.raiseOnGetProtocolTagIds + + return list(self.protocolTagIdsByProtocolDbId.get(int(protocolDbId), [])) + + def setProtocolTagIdsByProtocolDbId(self, projectId, protocolDbId, tagIds): + self.setProtocolTagIdsByProtocolDbIdCalls.append({ + "projectId": projectId, + "protocolDbId": protocolDbId, + "tagIds": list(tagIds or []), + }) + + if self.raiseOnSetProtocolTagIdsByProtocolDbId is not None: + raise self.raiseOnSetProtocolTagIdsByProtocolDbId + + cleanTagIds = sorted({ + str(tagId).strip() + for tagId in (tagIds or []) + if str(tagId).strip() + }) + + self.protocolTagIdsByProtocolDbId[int(protocolDbId)] = cleanTagIds + + return { + "protocolId": self.db.getRuntimeProtocolId(projectId, protocolDbId), + "protocolDbId": int(protocolDbId), + "tagIds": cleanTagIds, + } + + def setProtocolTagIds(self, projectId, protocolId, tagIds): + self.setProtocolTagIdsCalls.append({ + "projectId": projectId, + "protocolId": protocolId, + "tagIds": list(tagIds or []), + }) + + for row in self.db.protocolRows: + if row["projectId"] == int(projectId) and row["protocolId"] == str(protocolId): + return self.setProtocolTagIdsByProtocolDbId( + projectId=projectId, + protocolDbId=row["id"], + tagIds=tagIds, + ) + + raise Exception("Protocol not found in project") + + +class FakeMapperWithoutDbIdSetter(FakeMapper): + setProtocolTagIdsByProtocolDbId = None + + +@pytest.fixture +def projectServiceModule(authTestEnv): + return importlib.import_module("app.backend.api.services.project_service") + + +@pytest.fixture +def service(projectServiceModule): + return projectServiceModule.ProjectService() + + +@pytest.fixture +def mapper(): + return FakeMapper() + + +@pytest.fixture +def currentUser(): + return {"id": 1} + + +def test_ListProjectTagsUsesMapperProjectTags(service, mapper, currentUser): + mapper.projectTags = [ + { + "id": "good", + "title": "Good particles", + "description": None, + "color": "#00ff00", + }, + { + "id": "bad", + "title": "Bad particles", + "description": "Rejected items", + "color": "#ff0000", + }, + ] + + result = service.listProjectTags( + mapper=mapper, + projectId=1, + currentUser=currentUser, + ) + + assert result == mapper.projectTags + + +def test_ListProtocolTagsResolvesPostgresqlProtocolDbId( + service, + mapper, + currentUser, +): + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + mapper.protocolTagIdsByProtocolDbId[500] = ["good", "selected"] + + result = service.listProtocolTags( + mapper=mapper, + projectId=1, + protocolId=500, + currentUser=currentUser, + ) + + assert result == { + "protocolId": "500", + "protocolDbId": 500, + "tagIds": ["good", "selected"], + } + + assert mapper.getProtocolTagIdsCalls == [ + { + "projectId": 1, + "protocolDbId": 500, + } + ] + + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_ListProtocolTagsAlsoAcceptsRuntimeProtocolId( + service, + mapper, + currentUser, +): + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + mapper.protocolTagIdsByProtocolDbId[500] = ["movie-alignment"] + + result = service.listProtocolTags( + mapper=mapper, + projectId=1, + protocolId=10, + currentUser=currentUser, + ) + + assert result == { + "protocolId": "10", + "protocolDbId": 500, + "tagIds": ["movie-alignment"], + } + + assert mapper.getProtocolTagIdsCalls == [ + { + "projectId": 1, + "protocolDbId": 500, + } + ] + + assert mapper.db.fetchOneCalls[0]["params"] == (1, 10, "10") + + +def test_ListProtocolTagsRaises404WhenProtocolCannotBeResolved( + service, + mapper, + currentUser, +): + with pytest.raises(HTTPException) as exc: + service.listProtocolTags( + mapper=mapper, + projectId=1, + protocolId=999, + currentUser=currentUser, + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Protocol not found in PostgreSQL: 999" + assert mapper.getProtocolTagIdsCalls == [] + + +def test_ListProtocolTagsWrapsMapperErrors( + service, + mapper, + currentUser, +): + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + mapper.raiseOnGetProtocolTagIds = RuntimeError("database error") + + with pytest.raises(HTTPException) as exc: + service.listProtocolTags( + mapper=mapper, + projectId=1, + protocolId=500, + currentUser=currentUser, + ) + + assert exc.value.status_code == 500 + assert exc.value.detail == "Failed to list protocol tags: database error" + + assert mapper.getProtocolTagIdsCalls == [ + { + "projectId": 1, + "protocolDbId": 500, + } + ] + + +def test_SetProtocolTagsResolvesPostgresqlProtocolDbId( + service, + mapper, + currentUser, +): + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + + result = service.setProtocolTags( + mapper=mapper, + projectId=1, + protocolId=500, + tagIds=[" selected ", "good", "good", ""], + currentUser=currentUser, + ) + + assert result == { + "protocolId": "10", + "protocolDbId": 500, + "tagIds": ["good", "selected"], + } + + assert mapper.setProtocolTagIdsByProtocolDbIdCalls == [ + { + "projectId": 1, + "protocolDbId": 500, + "tagIds": [" selected ", "good", "good", ""], + } + ] + + assert mapper.setProtocolTagIdsCalls == [] + assert mapper.protocolTagIdsByProtocolDbId[500] == ["good", "selected"] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") + + +def test_SetProtocolTagsAlsoAcceptsRuntimeProtocolId( + service, + mapper, + currentUser, +): + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + + result = service.setProtocolTags( + mapper=mapper, + projectId=1, + protocolId=10, + tagIds=["movie-alignment"], + currentUser=currentUser, + ) + + assert result == { + "protocolId": "10", + "protocolDbId": 500, + "tagIds": ["movie-alignment"], + } + + assert mapper.setProtocolTagIdsByProtocolDbIdCalls == [ + { + "projectId": 1, + "protocolDbId": 500, + "tagIds": ["movie-alignment"], + } + ] + + assert mapper.setProtocolTagIdsCalls == [] + assert mapper.db.fetchOneCalls[0]["params"] == (1, 10, "10") + + +def test_SetProtocolTagsRaises404WhenProtocolCannotBeResolved( + service, + mapper, + currentUser, +): + with pytest.raises(HTTPException) as exc: + service.setProtocolTags( + mapper=mapper, + projectId=1, + protocolId=999, + tagIds=["good"], + currentUser=currentUser, + ) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Protocol not found in PostgreSQL: 999" + assert mapper.setProtocolTagIdsByProtocolDbIdCalls == [] + assert mapper.setProtocolTagIdsCalls == [] + + +def test_SetProtocolTagsWrapsMapperErrors( + service, + mapper, + currentUser, +): + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + mapper.raiseOnSetProtocolTagIdsByProtocolDbId = RuntimeError("write failed") + + with pytest.raises(HTTPException) as exc: + service.setProtocolTags( + mapper=mapper, + projectId=1, + protocolId=500, + tagIds=["good"], + currentUser=currentUser, + ) + + assert exc.value.status_code == 500 + assert exc.value.detail == "Failed to set protocol tags: write failed" + + assert mapper.setProtocolTagIdsByProtocolDbIdCalls == [ + { + "projectId": 1, + "protocolDbId": 500, + "tagIds": ["good"], + } + ] + + +def test_SetProtocolTagsFallsBackToRuntimeSetterWhenDbIdSetterIsMissing( + projectServiceModule, + currentUser, +): + service = projectServiceModule.ProjectService() + mapper = FakeMapperWithoutDbIdSetter() + mapper.db.addProtocol(projectId=1, protocolDbId=500, protocolId=10) + + result = service.setProtocolTags( + mapper=mapper, + projectId=1, + protocolId=10, + tagIds=["good"], + currentUser=currentUser, + ) + + assert result == { + "protocolId": "10", + "protocolDbId": 500, + "tagIds": ["good"], + } + + assert mapper.setProtocolTagIdsCalls == [ + { + "projectId": 1, + "protocolId": 10, + "tagIds": ["good"], + } + ] \ No newline at end of file From 288668750e928e169cd1a14fe828c42e9375e5bc Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Wed, 24 Jun 2026 22:45:36 +0200 Subject: [PATCH 154/278] Test protocol tag database id resolution --- .../api/services/test_project_service_tags.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/unit/backend/api/services/test_project_service_tags.py b/tests/unit/backend/api/services/test_project_service_tags.py index 0c4f4869..2a290233 100644 --- a/tests/unit/backend/api/services/test_project_service_tags.py +++ b/tests/unit/backend/api/services/test_project_service_tags.py @@ -168,6 +168,31 @@ def setProtocolTagIds(self, projectId, protocolId, tagIds): class FakeMapperWithoutDbIdSetter(FakeMapper): setProtocolTagIdsByProtocolDbId = None + def setProtocolTagIds(self, projectId, protocolId, tagIds): + self.setProtocolTagIdsCalls.append({ + "projectId": projectId, + "protocolId": protocolId, + "tagIds": list(tagIds or []), + }) + + for row in self.db.protocolRows: + if row["projectId"] == int(projectId) and row["protocolId"] == str(protocolId): + cleanTagIds = sorted({ + str(tagId).strip() + for tagId in (tagIds or []) + if str(tagId).strip() + }) + + self.protocolTagIdsByProtocolDbId[int(row["id"])] = cleanTagIds + + return { + "protocolId": str(row["protocolId"]), + "protocolDbId": int(row["id"]), + "tagIds": cleanTagIds, + } + + raise Exception("Protocol not found in project") + @pytest.fixture def projectServiceModule(authTestEnv): From 04f9a91f350f4a6905d35d426e81b188ba212319 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 10:37:04 +0200 Subject: [PATCH 155/278] Pass mapper when resolving analyze viewers --- app/backend/api/routers/project_router.py | 2 +- app/backend/api/services/project_service.py | 73 +++++++++- .../test_project_service_analyze_viewer.py | 132 ++++++++++++++++++ 3 files changed, 202 insertions(+), 5 deletions(-) create mode 100644 tests/unit/backend/api/services/test_project_service_analyze_viewer.py diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 785e6e5d..4152d0ff 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1522,7 +1522,7 @@ def resolveAnalyzeViewer( projectId=projectId, protocolId=protocolId, ctx=payload, - + mapper=mapper, ) return decision or {"handled": False} except Exception as e: diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 098af2d3..e9ced296 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -7475,11 +7475,76 @@ def _parseTiltSeriesFramePath( # ====================================================================== # Analyze Results: Resolve viewer # ====================================================================== - def resolveAnalyzeViewerDecision(self, projectId: int, protocolId: int, ctx: Dict[str, Any]) -> Dict[str, Any]: + def resolveAnalyzeViewerDecision( + self, + projectId: int, + protocolId: int, + ctx: Dict[str, Any], + mapper=None, + ) -> Dict[str, Any]: # resolveAnalyzeViewerDecision - return { - "handled": False, - } + ctx = ctx or {} + + outputName = str(ctx.get("outputName") or "").strip() + pointerClass = str(ctx.get("pointerClass") or "").strip() + paramClass = str(ctx.get("paramClass") or "").strip() + info = str(ctx.get("info") or "").strip() + value = str(ctx.get("value") or "").strip() + + classText = " ".join( + item + for item in ( + outputName, + pointerClass, + paramClass, + info, + value, + ) + if item + ).replace(" ", "").lower() + + if not outputName: + return {"handled": False} + + # These are handled by the React AnalyzeOutputDialog, not by opening a URL. + reactViewerClasses = ( + "volume", + "volumemask", + "setofvolumes", + "setoftiltseries", + "setofctftomoseries", + "setofcoordinates3d", + "setofcoordinates", + "setofparticles", + "setofclasses2d", + "setofclasses3d", + "setoffscs", + "metadata", + ) + + if any(token in classText for token in reactViewerClasses): + return {"handled": False} + + try: + self._getProtocolOutputObject( + protocolId=protocolId, + outputName=outputName, + mapper=mapper, + projectId=projectId, + ) + except HTTPException: + return {"handled": False} + except Exception: + logger.debug( + "Could not resolve analyze viewer output. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + exc_info=True, + ) + return {"handled": False} + + return {"handled": False} # ====================================================================== # Analyze Results: CTF Tomography (CTFTomoSeries) diff --git a/tests/unit/backend/api/services/test_project_service_analyze_viewer.py b/tests/unit/backend/api/services/test_project_service_analyze_viewer.py new file mode 100644 index 00000000..ba6fea98 --- /dev/null +++ b/tests/unit/backend/api/services/test_project_service_analyze_viewer.py @@ -0,0 +1,132 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +import importlib + + +class FakeDb: + def __init__(self): + self.fetchOneCalls = [] + self.runtimeProtocolIdByDbId = {} + + def fetchOne(self, query, params=None): + self.fetchOneCalls.append({ + "query": query, + "params": params, + }) + + if params is None: + return None + + if len(params) == 3: + _projectId, protocolDbIdCandidate, rawProtocolId = params + protocolDbIdCandidate = int(protocolDbIdCandidate) + + if protocolDbIdCandidate in self.runtimeProtocolIdByDbId: + return { + "protocolId": self.runtimeProtocolIdByDbId[protocolDbIdCandidate], + } + + for dbId, runtimeId in self.runtimeProtocolIdByDbId.items(): + if str(runtimeId) == str(rawProtocolId): + return { + "protocolId": runtimeId, + } + + return None + + +class FakeMapper: + def __init__(self): + self.db = FakeDb() + + +class FakeCurrentProject: + def __init__(self): + self.protocols = {} + + def getProtocol(self, protocolId): + return self.protocols[int(protocolId)] + + +class FakeProtocol: + def __init__(self, objId=10): + self.objId = objId + + def getObjId(self): + return self.objId + + +class FakeOutput: + pass + + +def test_ResolveAnalyzeViewerDecisionReturnsFalseForReactViewerWithoutRuntimeLookup(authTestEnv): + module = importlib.import_module("app.backend.api.services.project_service") + service = module.ProjectService() + mapper = FakeMapper() + + result = service.resolveAnalyzeViewerDecision( + mapper=mapper, + projectId=1, + protocolId=500, + ctx={ + "outputName": "outputVolume", + "pointerClass": "Volume", + "paramClass": "PointerParam", + }, + ) + + assert result == {"handled": False} + assert mapper.db.fetchOneCalls == [] + + +def test_ResolveAnalyzeViewerDecisionResolvesPostgresqlProtocolIdWhenOutputNeedsRuntime(authTestEnv): + module = importlib.import_module("app.backend.api.services.project_service") + service = module.ProjectService() + service.currentProject = FakeCurrentProject() + + protocol = FakeProtocol(objId=10) + protocol.outputCustom = FakeOutput() + + service.currentProject.protocols[10] = protocol + + mapper = FakeMapper() + mapper.db.runtimeProtocolIdByDbId[500] = 10 + + result = service.resolveAnalyzeViewerDecision( + mapper=mapper, + projectId=1, + protocolId=500, + ctx={ + "outputName": "outputCustom", + "pointerClass": "UnknownExternalObject", + "paramClass": "PointerParam", + }, + ) + + assert result == {"handled": False} + assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") \ No newline at end of file From 4f1de3e85ad50eb5693e77e2e59645b6d601729b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 11:03:12 +0200 Subject: [PATCH 156/278] Align tests with external analyze viewer resolver --- app/backend/api/services/project_service.py | 64 +------------------ tests/conftest.py | 5 +- tests/integration/api/test_projects_router.py | 9 ++- .../api/routers/test_protocols_router.py | 54 ++-------------- .../test_project_service_analyze_viewer.py | 61 +----------------- 5 files changed, 20 insertions(+), 173 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index e9ced296..db0baaca 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -7482,68 +7482,8 @@ def resolveAnalyzeViewerDecision( ctx: Dict[str, Any], mapper=None, ) -> Dict[str, Any]: - # resolveAnalyzeViewerDecision - ctx = ctx or {} - - outputName = str(ctx.get("outputName") or "").strip() - pointerClass = str(ctx.get("pointerClass") or "").strip() - paramClass = str(ctx.get("paramClass") or "").strip() - info = str(ctx.get("info") or "").strip() - value = str(ctx.get("value") or "").strip() - - classText = " ".join( - item - for item in ( - outputName, - pointerClass, - paramClass, - info, - value, - ) - if item - ).replace(" ", "").lower() - - if not outputName: - return {"handled": False} - - # These are handled by the React AnalyzeOutputDialog, not by opening a URL. - reactViewerClasses = ( - "volume", - "volumemask", - "setofvolumes", - "setoftiltseries", - "setofctftomoseries", - "setofcoordinates3d", - "setofcoordinates", - "setofparticles", - "setofclasses2d", - "setofclasses3d", - "setoffscs", - "metadata", - ) - - if any(token in classText for token in reactViewerClasses): - return {"handled": False} - - try: - self._getProtocolOutputObject( - protocolId=protocolId, - outputName=outputName, - mapper=mapper, - projectId=projectId, - ) - except HTTPException: - return {"handled": False} - except Exception: - logger.debug( - "Could not resolve analyze viewer output. projectId=%s protocolId=%s outputName=%s", - projectId, - protocolId, - outputName, - exc_info=True, - ) - return {"handled": False} - + # This resolver is reserved for external developer-provided viewers. + # Built-in React viewers are selected on the frontend side. return {"handled": False} # ====================================================================== diff --git a/tests/conftest.py b/tests/conftest.py index f1979c83..23a27eb8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -136,7 +136,7 @@ def __init__(self): } self.lastPollLogsCall = None - self.resolveViewerResult = {"handled": True, "viewer": "volume"} + self.resolveViewerResult = {"handled": False} self.resolveViewerError = None self.lastResolveViewerCall = None @@ -305,11 +305,12 @@ def pollProtocolLogsService(self, projectId, protocolId, offsets, maxBytes, maxL } return self.pollLogsResult - def resolveAnalyzeViewerDecision(self, projectId, protocolId, ctx): + def resolveAnalyzeViewerDecision(self, projectId, protocolId, ctx, mapper=None): self.lastResolveViewerCall = { "projectId": projectId, "protocolId": protocolId, "ctx": ctx, + "mapper": mapper } if self.resolveViewerError is not None: raise self.resolveViewerError diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index cf59ae06..904a779b 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -159,11 +159,12 @@ def test_ResolveAnalyzeViewerUnwrapsCtx(projectClient, fakeProjectService): ) assert response.status_code == 200 - assert response.json() == {"handled": True, "viewer": "volume"} + assert response.json() == {"handled": False} assert fakeProjectService.lastResolveViewerCall == { "projectId": 1, "protocolId": 22, "ctx": {"outputName": "particles", "outputClass": "SetOfParticles"}, + "mapper": fakeProjectService.lastResolveViewerCall["mapper"], } @@ -180,6 +181,12 @@ def test_ResolveAnalyzeViewerReturnsHandledFalseOnUnexpectedError(projectClient, "handled": False, "message": "viewer failed", } + assert fakeProjectService.lastResolveViewerCall == { + "projectId": 1, + "protocolId": 22, + "ctx": {"outputName": "particles"}, + "mapper": fakeProjectService.lastResolveViewerCall["mapper"], + } def test_ListMetadataTablesDelegatesMapperToService(projectClient, fakeProjectService): diff --git a/tests/unit/backend/api/routers/test_protocols_router.py b/tests/unit/backend/api/routers/test_protocols_router.py index 4477e127..693e3638 100644 --- a/tests/unit/backend/api/routers/test_protocols_router.py +++ b/tests/unit/backend/api/routers/test_protocols_router.py @@ -202,7 +202,7 @@ def test_LoadNewProtocolReturnsProtocolTemplate(protocolClient, fakeProtocolRout } -def test_LaunchProtocolReturnsNullOnSuccess(protocolClient, fakeProtocolRouterService): +def test_LaunchProtocolEndpointIsDisabled(protocolClient): response = protocolClient.post( "/protocols/launch", json={ @@ -212,55 +212,10 @@ def test_LaunchProtocolReturnsNullOnSuccess(protocolClient, fakeProtocolRouterSe }, ) - assert response.status_code == 200 - assert response.json() is None - - assert fakeProtocolRouterService.lastLaunchProtocolCall == { - "protocolId": "10", - "protocolClassName": "ProtClass", - "params": {"a": 1}, - } - - -def test_LaunchProtocolWrapsUnexpectedErrorAs500(protocolClient, fakeProtocolRouterService): - fakeProtocolRouterService.launchError = RuntimeError("launch failed") - - response = protocolClient.post( - "/protocols/launch", - json={ - "protocolId": "10", - "protocolClassName": "ProtClass", - "params": {"a": 1}, - }, - ) - - assert response.status_code == 500 - assert response.json()["detail"] == "launch failed" - - -def test_SaveProtocolReturnsNullOnSuccess(protocolClient, fakeProtocolRouterService): - response = protocolClient.post( - "/protocols/save", - json={ - "protocolId": "10", - "protocolClassName": "ProtClass", - "params": {"a": 1}, - }, - ) - - assert response.status_code == 200 - assert response.json() is None - - assert fakeProtocolRouterService.lastSaveProtocolCall == { - "protocolId": "10", - "protocolClassName": "ProtClass", - "params": {"a": 1}, - } - + assert response.status_code == 404 -def test_SaveProtocolWrapsUnexpectedErrorAs500(protocolClient, fakeProtocolRouterService): - fakeProtocolRouterService.saveError = RuntimeError("save failed") +def test_SaveProtocolEndpointIsDisabled(protocolClient): response = protocolClient.post( "/protocols/save", json={ @@ -270,8 +225,7 @@ def test_SaveProtocolWrapsUnexpectedErrorAs500(protocolClient, fakeProtocolRoute }, ) - assert response.status_code == 500 - assert response.json()["detail"] == "save failed" + assert response.status_code == 404 def test_GetProtocolLogsReturns404WhenProjectMissing(protocolClient, fakeProtocolRouterService): diff --git a/tests/unit/backend/api/services/test_project_service_analyze_viewer.py b/tests/unit/backend/api/services/test_project_service_analyze_viewer.py index ba6fea98..4ea93a87 100644 --- a/tests/unit/backend/api/services/test_project_service_analyze_viewer.py +++ b/tests/unit/backend/api/services/test_project_service_analyze_viewer.py @@ -30,33 +30,6 @@ class FakeDb: def __init__(self): self.fetchOneCalls = [] - self.runtimeProtocolIdByDbId = {} - - def fetchOne(self, query, params=None): - self.fetchOneCalls.append({ - "query": query, - "params": params, - }) - - if params is None: - return None - - if len(params) == 3: - _projectId, protocolDbIdCandidate, rawProtocolId = params - protocolDbIdCandidate = int(protocolDbIdCandidate) - - if protocolDbIdCandidate in self.runtimeProtocolIdByDbId: - return { - "protocolId": self.runtimeProtocolIdByDbId[protocolDbIdCandidate], - } - - for dbId, runtimeId in self.runtimeProtocolIdByDbId.items(): - if str(runtimeId) == str(rawProtocolId): - return { - "protocolId": runtimeId, - } - - return None class FakeMapper: @@ -64,27 +37,7 @@ def __init__(self): self.db = FakeDb() -class FakeCurrentProject: - def __init__(self): - self.protocols = {} - - def getProtocol(self, protocolId): - return self.protocols[int(protocolId)] - - -class FakeProtocol: - def __init__(self, objId=10): - self.objId = objId - - def getObjId(self): - return self.objId - - -class FakeOutput: - pass - - -def test_ResolveAnalyzeViewerDecisionReturnsFalseForReactViewerWithoutRuntimeLookup(authTestEnv): +def test_ResolveAnalyzeViewerDecisionReturnsHandledFalse(authTestEnv): module = importlib.import_module("app.backend.api.services.project_service") service = module.ProjectService() mapper = FakeMapper() @@ -104,18 +57,10 @@ def test_ResolveAnalyzeViewerDecisionReturnsFalseForReactViewerWithoutRuntimeLoo assert mapper.db.fetchOneCalls == [] -def test_ResolveAnalyzeViewerDecisionResolvesPostgresqlProtocolIdWhenOutputNeedsRuntime(authTestEnv): +def test_ResolveAnalyzeViewerDecisionDoesNotResolveRuntimeObjects(authTestEnv): module = importlib.import_module("app.backend.api.services.project_service") service = module.ProjectService() - service.currentProject = FakeCurrentProject() - - protocol = FakeProtocol(objId=10) - protocol.outputCustom = FakeOutput() - - service.currentProject.protocols[10] = protocol - mapper = FakeMapper() - mapper.db.runtimeProtocolIdByDbId[500] = 10 result = service.resolveAnalyzeViewerDecision( mapper=mapper, @@ -129,4 +74,4 @@ def test_ResolveAnalyzeViewerDecisionResolvesPostgresqlProtocolIdWhenOutputNeeds ) assert result == {"handled": False} - assert mapper.db.fetchOneCalls[0]["params"] == (1, 500, "500") \ No newline at end of file + assert mapper.db.fetchOneCalls == [] \ No newline at end of file From da72a74df2e45e95a089a709159eead4fc5e27eb Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 11:26:07 +0200 Subject: [PATCH 157/278] Require PostgreSQL coords3d viewer data --- app/backend/api/services/project_service.py | 18 +++++ .../services/test_project_service_coords3d.py | 69 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index db0baaca..92bf85db 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -9906,6 +9906,15 @@ def getCoordinates3dPointsService( getattr(pgReader, "lastSkipReason", None), ) + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Coordinates3D points", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + ) + protocol, setOfCoordinates3D = self._resolveOutputForCoordinates3d( protocolId=protocolId, outputName=outputName, @@ -10184,6 +10193,15 @@ def renderCoords3dTomogramSliceService( getattr(pgReader, "lastSkipReason", None), ) + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="Coordinates3D tomogram slice", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + ) + protocol, setOfCoordinates3D = self._resolveOutputForCoordinates3d( protocolId=protocolId, outputName=outputName, diff --git a/tests/unit/backend/api/services/test_project_service_coords3d.py b/tests/unit/backend/api/services/test_project_service_coords3d.py index 2d142903..4907f44b 100644 --- a/tests/unit/backend/api/services/test_project_service_coords3d.py +++ b/tests/unit/backend/api/services/test_project_service_coords3d.py @@ -299,6 +299,7 @@ def test_ListCoordinates3dTomogramsServiceBuildsTomogramList(service, tmp_path): assert service.tomoList["TS_001"] is tomo1 assert service.tomoList["TS_002"] is tomo2 + def test_GetPostgresqlCoords3dReaderIfAvailableUsesResolvedProtocolDbId( service, monkeypatch, @@ -357,6 +358,74 @@ def hasOutput(self): assert createdReaders[0].outputName == "outputCoords3d" +@pytest.mark.parametrize( + "serviceCall, expectedDetail", + [ + ( + lambda service, mapper: service.listCoordinates3dTomogramsService( + projectId=1, + protocolId=10, + outputName="outputCoords3d", + mapper=mapper, + ), + "Coordinates3D output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.getCoordinates3dPointsService( + projectId=1, + protocolId=10, + outputName="outputCoords3d", + tomogramId="TS_001", + mapper=mapper, + ), + "Coordinates3D points output is not available in PostgreSQL metadata", + ), + ( + lambda service, mapper: service.renderCoords3dTomogramSliceService( + projectId=1, + protocolId=10, + outputName="outputCoords3d", + tomogramId="TS_001", + sliceIndex=0, + axis="z", + colormap=None, + normalize="minmax", + scale=1.0, + inline=True, + fmt="png", + thumb=None, + fast=True, + quality=75, + mapper=mapper, + ), + "Coordinates3D tomogram slice output is not available in PostgreSQL metadata", + ), + ], +) +def test_Coordinates3dServicesRequirePostgresqlWhenMapperIsPresent( + service, + monkeypatch, + serviceCall, + expectedDetail, +): + monkeypatch.setattr( + service, + "_getPostgresqlCoords3dReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy Coordinates3D fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForCoordinates3d", failRuntimeFallback) + + with pytest.raises(HTTPException) as exc: + serviceCall(service, object()) + + assert exc.value.status_code == 404 + assert expectedDetail in exc.value.detail + + def test_GetCoordinates3dPointsServiceBuildsPointPayload(service, tmp_path): tomoPath = tmp_path / "tomo1.mrc" tomoPath.write_text("placeholder", encoding="utf-8") From 1930a029a57dc1e2c90af96c707f2ec34834ed8b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 11:31:43 +0200 Subject: [PATCH 158/278] Require PostgreSQL tiltseries image viewer data --- app/backend/api/services/project_service.py | 11 ++++++ .../test_project_service_tiltseries.py | 34 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 92bf85db..b602b4b2 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -9261,6 +9261,17 @@ def renderTiltSeriesImageService( ) + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="TiltSeries image", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason=getattr(pgReader, "lastSkipReason", None) if pgReader is not None else "reader_not_available", + tiltSeriesId=tiltSeriesId, + index=index, + ) + protocol, setOfTiltSeries = self._resolveOutputForTiltSeries( protocolId=protocolId, outputName=outputName, diff --git a/tests/unit/backend/api/services/test_project_service_tiltseries.py b/tests/unit/backend/api/services/test_project_service_tiltseries.py index 06c8ae09..ebde0ca5 100644 --- a/tests/unit/backend/api/services/test_project_service_tiltseries.py +++ b/tests/unit/backend/api/services/test_project_service_tiltseries.py @@ -395,6 +395,40 @@ def failRuntimeFallback(**kwargs): assert "reader_not_available" in exc.value.detail +def test_RenderTiltSeriesImageServiceRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + monkeypatch.setattr( + service, + "_getPostgresqlTiltSeriesReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy TiltSeries image fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForTiltSeries", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service.renderTiltSeriesImageService( + projectId=1, + protocolId=10, + outputName="outputTiltSeries", + tiltSeriesId="TS_001", + index=0, + size=512, + fmt="png", + applyTransform=True, + inline=True, + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "TiltSeries image output is not available in PostgreSQL metadata" in exc.value.detail + assert "reader_not_available" in exc.value.detail + + def test_ListOutputTiltSeriesServiceBuildsSummaries(service): ts1 = FakeTiltSeries( tsId="TS_001", From 24b5a03434498ed2207f8dfdb4db00c40355d021 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 11:38:28 +0200 Subject: [PATCH 159/278] Require PostgreSQL ctftomo views data --- app/backend/api/services/project_service.py | 18 +++++------ .../services/test_project_service_ctftomo.py | 30 +++++++++++++++++++ 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index b602b4b2..595e4ff2 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -7726,17 +7726,13 @@ def getCtftomoSeriesViewsService( detail="CTF tomo series not found in PostgreSQL metadata", ) if mapper is not None: - logger.warning( - "PostgreSQL CTFTomo reader is not available. Skipping legacy fallback. " - "projectId=%s protocolId=%s outputName=%s tiltSeriesId=%s", - projectId, - protocolId, - outputName, - tiltSeriesId, - ) - raise HTTPException( - status_code=404, - detail="CTF tomo output is not available in PostgreSQL metadata", + self._raisePostgresqlViewerUnavailable( + viewerName="CTFTomo views", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="views_not_available" if pgReader is not None else "reader_not_available", + tiltSeriesId=tiltSeriesId, ) protocol, output = self._resolveOutputForCtftomoSeries( diff --git a/tests/unit/backend/api/services/test_project_service_ctftomo.py b/tests/unit/backend/api/services/test_project_service_ctftomo.py index e3864fdc..a1589b4e 100644 --- a/tests/unit/backend/api/services/test_project_service_ctftomo.py +++ b/tests/unit/backend/api/services/test_project_service_ctftomo.py @@ -371,6 +371,36 @@ def test_ListOutputCtftomoSeriesServiceBuildsSummaries(service, tmp_path): }, ] + +def test_GetCtftomoSeriesViewsServiceRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + monkeypatch.setattr( + service, + "_getPostgresqlCtftomoReaderIfAvailable", + lambda **kwargs: None, + ) + + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy CTFTomo views fallback should not be used") + + monkeypatch.setattr(service, "_resolveOutputForCtftomoSeries", failRuntimeFallback) + + with pytest.raises(Exception) as exc: + service.getCtftomoSeriesViewsService( + projectId=1, + protocolId=10, + outputName="outputCtftomo", + tiltSeriesId="TS_001", + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "CTFTomo views output is not available in PostgreSQL metadata" in exc.value.detail + assert "reader_not_available" in exc.value.detail + + def test_ListOutputCtftomoSeriesServiceRequiresPostgresqlWhenMapperIsPresent( service, monkeypatch, From 8fbafab2de471fd555a03c84f4c09d0e65466810 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 11:46:37 +0200 Subject: [PATCH 160/278] Require PostgreSQL FSC viewer data --- app/backend/api/services/project_service.py | 23 ++- .../api/services/test_project_service_fsc.py | 131 ++++++++++++++++++ 2 files changed, 140 insertions(+), 14 deletions(-) create mode 100644 tests/unit/backend/api/services/test_project_service_fsc.py diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 595e4ff2..d655de84 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -10770,21 +10770,16 @@ def getFscRowsService( ) -> Dict[str, Any]: """ Return FSC curves for a SetOfFSCs-like output. - - Response shape: - { - "threshold": 0.143, - "rows": [ - { - "label": "No mask", - "resolution": 3.21, - "x": [0.0, 0.01, 0.02, ...], - "y": [1.0, 0.98, 0.95, ...], - }, - ... - ], - } """ + if mapper is not None: + self._raisePostgresqlViewerUnavailable( + viewerName="FSC", + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + reason="reader_not_available", + ) + protocol = self._getScipionProtocolForRuntime( mapper=mapper, projectId=projectId, diff --git a/tests/unit/backend/api/services/test_project_service_fsc.py b/tests/unit/backend/api/services/test_project_service_fsc.py new file mode 100644 index 00000000..da035d13 --- /dev/null +++ b/tests/unit/backend/api/services/test_project_service_fsc.py @@ -0,0 +1,131 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +import importlib + +import pytest +from fastapi import HTTPException + + +class FakeFsc: + def __init__(self, label="FSC 1"): + self.label = label + + def clone(self): + return self + + def getObjLabel(self): + return self.label + + def getData(self): + return [ + [0.01, 0.02, 0.03], + [0.9, 0.5, 0.1], + ] + + def calculateResolution(self, threshold): + return 3.5 + + +class FakeFscOutput: + def __init__(self, items=None): + self.items = items or [] + + def iterItems(self, iterate=False): + return list(self.items) + + +class FakeProtocol: + def __init__(self, outputName, output): + setattr(self, outputName, output) + + +class FakeCurrentProject: + def __init__(self, protocol): + self.protocol = protocol + self.lastGetProtocolId = None + + def getProtocol(self, protocolId): + self.lastGetProtocolId = protocolId + return self.protocol + + +@pytest.fixture +def projectServiceModule(authTestEnv): + return importlib.import_module("app.backend.api.services.project_service") + + +@pytest.fixture +def service(projectServiceModule): + instance = object.__new__(projectServiceModule.ProjectService) + instance.currentProject = None + return instance + + +def test_GetFscRowsServiceRequiresPostgresqlWhenMapperIsPresent( + service, + monkeypatch, +): + def failRuntimeFallback(**kwargs): + raise AssertionError("Legacy FSC fallback should not be used") + + monkeypatch.setattr(service, "_getScipionProtocolForRuntime", failRuntimeFallback) + + with pytest.raises(HTTPException) as exc: + service.getFscRowsService( + projectId=1, + protocolId=10, + outputName="outputFsc", + mapper=object(), + ) + + assert exc.value.status_code == 404 + assert "FSC output is not available in PostgreSQL metadata" in exc.value.detail + assert "reader_not_available" in exc.value.detail + + +def test_GetFscRowsServiceBuildsRowsWithoutMapper(service): + fscOutput = FakeFscOutput(items=[FakeFsc(label="Half maps")]) + protocol = FakeProtocol("outputFsc", fscOutput) + service.currentProject = FakeCurrentProject(protocol) + + result = service.getFscRowsService( + projectId=1, + protocolId=10, + outputName="outputFsc", + ) + + assert result == { + "threshold": 0.143, + "rows": [ + { + "label": "Half maps", + "resolution": 3.5, + "x": [0.01, 0.02, 0.03], + "y": [0.9, 0.5, 0.1], + } + ], + } \ No newline at end of file From bc79e10a7653a4bad0f97cff55518ff61f64f2ec Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 11:52:33 +0200 Subject: [PATCH 161/278] Test metadata action protocol id resolution --- .../services/test_project_service_metadata.py | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/tests/unit/backend/api/services/test_project_service_metadata.py b/tests/unit/backend/api/services/test_project_service_metadata.py index 21a39a90..a5a664f4 100644 --- a/tests/unit/backend/api/services/test_project_service_metadata.py +++ b/tests/unit/backend/api/services/test_project_service_metadata.py @@ -988,6 +988,120 @@ def syncProjectProtocolsAndDependencies( assert selectionFiles[0].read_text(encoding="utf-8") == "3 5 7 " +def test_RunMetadataTableActionServiceUsesRuntimeResolverWithMapper( + service, + projectServiceModule, + monkeypatch, + tmp_path, +): + outputFile = tmp_path / "metadata.sqlite" + outputFile.write_text("placeholder", encoding="utf-8") + + output = FakeOutput(str(outputFile)) + protocol = FakeProtocol("outputParticles", output) + service.currentProject = FakeCurrentProject(protocol, projectPath=tmp_path) + + table = FakeTable( + name="objects", + alias="Particles", + columns=[], + actions=[FakeAction("create subset")], + ) + dao = FakeDao(objectsType={"create subset": "SetOfParticles"}) + objMgr = FakeObjectManager( + tables={"objects": table}, + rowsByTable={"objects": []}, + dao=dao, + ) + + class FakeDb: + # fakeDb + def fetchOne(self, *args, **kwargs): + return None + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + mapper = FakeMapper() + resolverCalls = [] + openTableCalls = [] + + def getScipionProtocolForRuntime(mapper, projectId, protocolId): + resolverCalls.append( + { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + } + ) + return protocol + + def syncProjectProtocolsAndDependencies( + mapperArg, + projectIdArg, + refresh=True, + checkPid=True, + ): + return { + "protocols": 2, + "dependencies": 1, + } + + patchOpenMetadataTable( + service, + monkeypatch, + objMgr, + table, + calls=openTableCalls, + ) + + monkeypatch.setattr(projectServiceModule, "OBJECT_TABLE", "objects") + monkeypatch.setattr(projectServiceModule, "ProtUserSubSet", object()) + monkeypatch.setattr( + service, + "_getScipionProtocolForRuntime", + getScipionProtocolForRuntime, + ) + monkeypatch.setattr( + service, + "syncProjectProtocolsAndDependencies", + syncProjectProtocolsAndDependencies, + ) + + result = service.runMetadataTableActionService( + projectId=1, + protocolId=500, + outputName="outputParticles", + tableName="objects", + action="create subset", + subsetName="subset A", + ids=[3, 5, 7], + currentUser={"id": 1}, + mapper=mapper, + ) + + assert result["success"] is True + assert resolverCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + assert openTableCalls == [ + { + "projectId": 1, + "protocolId": 500, + "outputName": "outputParticles", + "tableName": "objects", + "mapper": mapper, + } + ] + assert len(protocol.newProtocolCalls) == 1 + + def test_RunMetadataTableActionServiceBuildsChildTableSelectionArgument( service, projectServiceModule, From 652b462f8e2042e49352e5d2dd51d1b6a8a21dd9 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 12:01:11 +0200 Subject: [PATCH 162/278] Resolve protocol id for metadata image paths --- app/backend/api/services/project_service.py | 7 +- .../services/test_project_service_metadata.py | 100 ++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index d655de84..cd7d29e7 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -12068,7 +12068,12 @@ def renderMetadataImageCellService( try: if self.currentProject is not None: projPath = LocalPath(self.currentProject.getPath()) - prot = self.currentProject.getProtocol(int(protocolId)) + + prot = self._getScipionProtocolForRuntime( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) protPath = LocalPath(prot.getPath()) except Exception: protPath = None diff --git a/tests/unit/backend/api/services/test_project_service_metadata.py b/tests/unit/backend/api/services/test_project_service_metadata.py index a5a664f4..b6f35f29 100644 --- a/tests/unit/backend/api/services/test_project_service_metadata.py +++ b/tests/unit/backend/api/services/test_project_service_metadata.py @@ -868,6 +868,106 @@ def test_RenderMetadataImageCellServiceReturnsPlaceholderWhenRowMissing(service, assert response.media_type == "image/png" +def test_RenderMetadataImageCellServiceResolvesProtocolIdForRelativeImagePaths( + service, + monkeypatch, + tmp_path, +): + protocolPath = tmp_path / "Runs" / "000010_Prot" + protocolPath.mkdir(parents=True) + + imagePath = protocolPath / "thumb.png" + + try: + from PIL import Image + + Image.new("L", (4, 4), 128).save(imagePath) + except Exception: + imagePath.write_bytes(b"") + + class PathRenderer: + # pathRenderer + def render(self, rawValue, rowValues): + return rawValue + + class FakeProtocolWithPath: + # fakeProtocolWithPath + def getPath(self): + return str(protocolPath) + + class FakeProjectWithoutProtocolLookup: + # fakeProjectWithoutProtocolLookup + def getPath(self): + return str(tmp_path) + + def getProtocol(self, protocolId): + raise AssertionError("currentProject.getProtocol should not be used directly") + + columns = [FakeColumn("image", "Image", PathRenderer())] + table = FakeTable(name="objects", alias="Particles", columns=columns) + objMgr = FakeObjectManager( + tables={"objects": table}, + rowsByTable={"objects": [FakeRow(1, ["thumb.png"])]}, + rowCounts={"objects": 1}, + fileName="postgresql://project/1/protocol/500/output/outputParticles", + ) + + class FakeDb: + # fakeDb + pass + + class FakeMapper: + # fakeMapper + def __init__(self): + self.db = FakeDb() + + mapper = FakeMapper() + resolverCalls = [] + + def getScipionProtocolForRuntime(mapper, projectId, protocolId): + resolverCalls.append( + { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + } + ) + return FakeProtocolWithPath() + + service.currentProject = FakeProjectWithoutProtocolLookup() + patchOpenMetadataTable(service, monkeypatch, objMgr, table) + monkeypatch.setattr( + service, + "_getScipionProtocolForRuntime", + getScipionProtocolForRuntime, + ) + + response = service.renderMetadataImageCellService( + projectId=1, + protocolId=500, + outputName="outputParticles", + tableName="objects", + rowId=1, + rowIndex=None, + columnName="image", + size=64, + applyTransform=False, + inline=True, + fmt="png", + mapper=mapper, + ) + + assert response.media_type == "image/png" + assert response.headers.get("x-image-placeholder") is None + assert resolverCalls == [ + { + "mapper": mapper, + "projectId": 1, + "protocolId": 500, + } + ] + + def test_RunMetadataTableActionServiceLaunchesSubsetProtocol( service, projectServiceModule, From e56d3f4e68aa6339c9af0099b761d53c0da0b598 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 12:35:43 +0200 Subject: [PATCH 163/278] Resolve protocol ids for thumbnail item filtering --- app/backend/api/routers/project_router.py | 53 +------------ app/backend/api/services/project_service.py | 62 ++++++++++++++- .../test_project_service_io_thumbnails.py | 76 +++++++++++++++++++ 3 files changed, 138 insertions(+), 53 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 4152d0ff..dbd9b224 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -3613,59 +3613,8 @@ def listProjectThumbnailItems( maxProtocols=maxProtocols, maxOutputsPerProtocol=maxOutputsPerProtocol, inlineImages=inlineImages, + mapper=mapper, ) - - validItems = [] - - for group in items or []: - if not isinstance(group, dict): - continue - - protocolIdValue = group.get("protocolId") - - try: - protocol = service.currentProject.getProtocol(int(protocolIdValue)) - except Exception: - logger.warning( - "Skipping thumbnail group because protocol was not found. " - "projectId=%s protocolId=%s group=%s", - projectId, - protocolIdValue, - group, - ) - continue - - validOutputs = [] - - for output in group.get("outputs") or []: - if not isinstance(output, dict): - continue - - outputNameValue = output.get("outputName") - if not outputNameValue: - continue - - if not hasattr(protocol, str(outputNameValue)): - logger.warning( - "Skipping thumbnail output because it does not belong to protocol. " - "projectId=%s protocolId=%s outputName=%s", - projectId, - protocolIdValue, - outputNameValue, - ) - continue - - validOutputs.append(output) - - if not validOutputs: - continue - - nextGroup = dict(group) - nextGroup["outputs"] = validOutputs - validItems.append(nextGroup) - - items = validItems - response = JSONResponse(items) response.headers["Cache-Control"] = "private, max-age=60, stale-while-revalidate=300" response.headers["Access-Control-Expose-Headers"] = "Cache-Control" diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index cd7d29e7..48d7adf2 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -3836,9 +3836,10 @@ def listProjectThumbnailItems( maxProtocols: int = 12, maxOutputsPerProtocol: int = 4, inlineImages: bool = False, + mapper=None, ): thumbnailService = ThumbnailService(self.currentProject) - return thumbnailService.listProtocolThumbnailItems( + items = thumbnailService.listProtocolThumbnailItems( projectId=projectId, force=force, size=size, @@ -3847,6 +3848,65 @@ def listProjectThumbnailItems( inlineImages=inlineImages, ) + if mapper is None: + return items + + validItems = [] + + for group in items or []: + if not isinstance(group, dict): + continue + + protocolIdValue = group.get("protocolId") + + try: + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolIdValue, + ) + protocol = self.currentProject.getProtocol(int(scipionProtocolId)) + except Exception: + logger.warning( + "Skipping thumbnail group because protocol was not found. " + "projectId=%s protocolId=%s group=%s", + projectId, + protocolIdValue, + group, + ) + continue + + validOutputs = [] + + for output in group.get("outputs") or []: + if not isinstance(output, dict): + continue + + outputNameValue = output.get("outputName") + if not outputNameValue: + continue + + if not hasattr(protocol, str(outputNameValue)): + logger.warning( + "Skipping thumbnail output because it does not belong to protocol. " + "projectId=%s protocolId=%s outputName=%s", + projectId, + protocolIdValue, + outputNameValue, + ) + continue + + validOutputs.append(output) + + if not validOutputs: + continue + + nextGroup = dict(group) + nextGroup["outputs"] = validOutputs + validItems.append(nextGroup) + + return validItems + def _buildMissingOutputSyncItems( self, protocolId: Union[int, str], diff --git a/tests/unit/backend/api/services/test_project_service_io_thumbnails.py b/tests/unit/backend/api/services/test_project_service_io_thumbnails.py index 23cfbbf4..c43a14a6 100644 --- a/tests/unit/backend/api/services/test_project_service_io_thumbnails.py +++ b/tests/unit/backend/api/services/test_project_service_io_thumbnails.py @@ -433,6 +433,82 @@ def test_ListProjectThumbnailItemsDelegatesToThumbnailService(projectServiceModu ] +def test_ListProjectThumbnailItemsResolvesPostgresqlProtocolIdsWhenFilteringOutputs( + projectServiceModule, + service, + monkeypatch, +): + class FakeThumbnailServiceWithItems: + # fakeThumbnailServiceWithItems + instances = [] + + def __init__(self, currentProject): + self.currentProject = currentProject + FakeThumbnailServiceWithItems.instances.append(self) + + def listProtocolThumbnailItems( + self, + projectId, + force=False, + size=320, + maxProtocols=12, + maxOutputsPerProtocol=4, + inlineImages=False, + ): + return [ + { + "projectId": projectId, + "protocolId": 500, + "outputs": [ + {"outputName": "outputVol"}, + {"outputName": "missingOutput"}, + ], + }, + { + "projectId": projectId, + "protocolId": 999, + "outputs": [ + {"outputName": "outputVol"}, + ], + }, + ] + + output = FakeOutput("volume.mrc") + protocol = FakeProtocol(protocolId=10, outputName="outputVol", output=output) + service.currentProject = FakeCurrentProject(protocols={10: protocol}) + + mapper = FakeMapper(runtimeProtocolIdByDbId={500: 10}) + + monkeypatch.setattr( + projectServiceModule, + "ThumbnailService", + FakeThumbnailServiceWithItems, + ) + + result = service.listProjectThumbnailItems( + projectId=1, + force=False, + size=320, + maxProtocols=12, + maxOutputsPerProtocol=4, + inlineImages=False, + mapper=mapper, + ) + + assert result == [ + { + "projectId": 1, + "protocolId": 500, + "outputs": [ + {"outputName": "outputVol"}, + ], + } + ] + + assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + assert mapper.db.fetchCalls[1]["params"] == (1, 999, "999") + + def test_NormalizeExportJsonContentAcceptsJsonString(service): content = service._normalizeExportJsonContent('[{"id": 10}]') assert json.loads(content) == [{"id": 10}] From 30ea271abc8242f4d727947d71d8761005dcf623 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 13:21:57 +0200 Subject: [PATCH 164/278] Add project PostgreSQL consistency checker --- app/backend/api/routers/project_router.py | 23 ++ app/backend/api/services/project_service.py | 203 ++++++++++++++ .../test_project_service_consistency.py | 251 ++++++++++++++++++ 3 files changed, 477 insertions(+) create mode 100644 tests/unit/backend/api/services/test_project_service_consistency.py diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index dbd9b224..963e9101 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -217,6 +217,29 @@ def getProjectEffectiveSettings( detail=f"Failed to load project effective settings: {e}", ) + +@router.post( + "/{projectId}/consistency/check", + response_model=Any, + status_code=status.HTTP_200_OK, +) +def checkProjectPostgresqlConsistency( + projectId: int, + refresh: bool = Query(True), + checkPid: bool = Query(True), + currentUser=Depends(getCurrentUser), + mapper: PostgresqlFlatMapper = Depends(getMapper), + service: ProjectService = Depends(getProjectService), +): + return service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=projectId, + currentUser=currentUser, + refresh=refresh, + checkPid=checkPid, + ) + + @router.put("/{projectId}", response_model=Any, status_code=status.HTTP_200_OK) def updateProject( projectId: int, diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 48d7adf2..f2e980ad 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2561,6 +2561,209 @@ def normalizeStatus(value: Any) -> str: "thumbnailItemsUrl": self.buildProjectThumbnailItemsUrl(dbProj['id']), } + + def validateProjectPostgresqlConsistency( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser: dict, + refresh: bool = True, + checkPid: bool = True, + ) -> Dict[str, Any]: + dbProj = self.getProjectDbRow(mapper, projectId, currentUser) + if not dbProj: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Project not found", + ) + + self.loadProjectForThumbnails(dbProj) + + def normalizeStatus(value: Any) -> str: + return str(value or "").strip().lower() + + def normalizeProtocolId(value: Any) -> str: + return str(value).strip() + + def protocolSortKey(value: Any): + text = normalizeProtocolId(value) + try: + return 0, int(text) + except Exception: + return 1, text + + def dependencySortKey(item: Tuple[str, str]): + parentId, childId = item + return protocolSortKey(parentId), protocolSortKey(childId) + + def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: + return { + "parentId": normalizeProtocolId(parentId), + "childId": normalizeProtocolId(childId), + } + + runtimeStatuses: Dict[str, str] = {} + runtimeDependencies: Set[Tuple[str, str]] = set() + + try: + runs = self.currentProject.getRunsGraph(refresh=refresh, checkPids=checkPid) + nodesDict = getattr(runs, "_nodesDict", {}) or {} + except Exception as e: + logger.exception( + "Failed to load Scipion runtime graph for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load Scipion runtime graph: {e}", + ) + + for nodeId, nodeObj in nodesDict.items(): + protocolId = normalizeProtocolId(nodeId) + if not protocolId or protocolId == "PROJECT": + continue + + protocol = getattr(nodeObj, "run", None) + runtimeStatuses[protocolId] = normalizeStatus( + self._safeCall(protocol, "getStatus", None) + ) + + for parent in getattr(nodeObj, "_parents", []) or []: + try: + parentId = normalizeProtocolId(parent.getName()) + except Exception: + parentId = normalizeProtocolId(parent) + + if not parentId or parentId == "PROJECT": + continue + + runtimeDependencies.add((parentId, protocolId)) + + try: + protocolRows = mapper.getProtocols(projectId) or [] + except Exception as e: + logger.exception( + "Failed to load PostgreSQL protocols for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL protocols: {e}", + ) + + postgresqlStatuses: Dict[str, str] = {} + for row in protocolRows: + protocolId = normalizeProtocolId(row.get("protocolId")) + if not protocolId: + continue + + postgresqlStatuses[protocolId] = normalizeStatus(row.get("status")) + + try: + adjacencyMap = mapper.getProjectProtocolAdjacencyMap(projectId) or {} + except Exception as e: + logger.exception( + "Failed to load PostgreSQL protocol dependencies for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL protocol dependencies: {e}", + ) + + postgresqlDependencies: Set[Tuple[str, str]] = set() + for childId, refs in adjacencyMap.items(): + childProtocolId = normalizeProtocolId(childId) + if not childProtocolId or childProtocolId == "PROJECT": + continue + + for parentIdValue in refs.get("parents") or []: + parentProtocolId = normalizeProtocolId(parentIdValue) + if not parentProtocolId or parentProtocolId == "PROJECT": + continue + + postgresqlDependencies.add((parentProtocolId, childProtocolId)) + + runtimeProtocolIds = set(runtimeStatuses.keys()) + postgresqlProtocolIds = set(postgresqlStatuses.keys()) + + missingProtocolIds = sorted( + runtimeProtocolIds - postgresqlProtocolIds, + key=protocolSortKey, + ) + extraProtocolIds = sorted( + postgresqlProtocolIds - runtimeProtocolIds, + key=protocolSortKey, + ) + + commonProtocolIds = sorted( + runtimeProtocolIds.intersection(postgresqlProtocolIds), + key=protocolSortKey, + ) + + statusMismatches = [] + for protocolId in commonProtocolIds: + runtimeStatus = runtimeStatuses.get(protocolId, "") + postgresqlStatus = postgresqlStatuses.get(protocolId, "") + + if runtimeStatus != postgresqlStatus: + statusMismatches.append({ + "protocolId": protocolId, + "runtimeStatus": runtimeStatus, + "postgresqlStatus": postgresqlStatus, + }) + + missingDependencies = sorted( + runtimeDependencies - postgresqlDependencies, + key=dependencySortKey, + ) + extraDependencies = sorted( + postgresqlDependencies - runtimeDependencies, + key=dependencySortKey, + ) + + issues = { + "missingProtocols": [ + { + "protocolId": protocolId, + "runtimeStatus": runtimeStatuses.get(protocolId, ""), + } + for protocolId in missingProtocolIds + ], + "extraProtocols": [ + { + "protocolId": protocolId, + "postgresqlStatus": postgresqlStatuses.get(protocolId, ""), + } + for protocolId in extraProtocolIds + ], + "statusMismatches": statusMismatches, + "missingDependencies": [ + buildDependency(parentId, childId) + for parentId, childId in missingDependencies + ], + "extraDependencies": [ + buildDependency(parentId, childId) + for parentId, childId in extraDependencies + ], + } + + issuesCount = sum(len(items) for items in issues.values()) + + return { + "ok": issuesCount == 0, + "projectId": projectId, + "checkedAt": datetime.utcnow().isoformat() + "Z", + "summary": { + "runtimeProtocols": len(runtimeProtocolIds), + "postgresqlProtocols": len(postgresqlProtocolIds), + "runtimeDependencies": len(runtimeDependencies), + "postgresqlDependencies": len(postgresqlDependencies), + "issues": issuesCount, + }, + "issues": issues, + } + def listProjectWorkflows(self, raw: bool = False): """ Return available workflow templates. diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py new file mode 100644 index 00000000..deaa76df --- /dev/null +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -0,0 +1,251 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** + +import importlib + +import pytest + + +class FakeProtocol: + def __init__(self, status): + self.status = status + + def getStatus(self): + return self.status + + +class FakeParentNode: + def __init__(self, name): + self.name = name + + def getName(self): + return self.name + + +class FakeRunNode: + def __init__(self, name, status=None, parents=None): + self.name = name + self.run = FakeProtocol(status) if status is not None else None + self._parents = [FakeParentNode(parentName) for parentName in (parents or [])] + + def getName(self): + return self.name + + +class FakeRunsGraph: + def __init__(self, nodes): + self._nodesDict = nodes + + +class FakeCurrentProject: + def __init__(self, nodes): + self.nodes = nodes + self.lastRefresh = None + self.lastCheckPids = None + + def getRunsGraph(self, refresh=True, checkPids=True): + self.lastRefresh = refresh + self.lastCheckPids = checkPids + return FakeRunsGraph(self.nodes) + + +class FakeMapper: + def __init__(self, projectRow, protocolRows, adjacencyMap): + self.projectRow = projectRow + self.protocolRows = protocolRows + self.adjacencyMap = adjacencyMap + + def getProject(self, projectId, userId): + if int(projectId) != int(self.projectRow["id"]): + return None + if int(userId) != int(self.projectRow["ownerId"]): + return None + return dict(self.projectRow) + + def getProtocols(self, projectId): + return list(self.protocolRows) + + def getProjectProtocolAdjacencyMap(self, projectId): + return dict(self.adjacencyMap) + + +@pytest.fixture +def projectServiceModule(authTestEnv): + return importlib.import_module("app.backend.api.services.project_service") + + +@pytest.fixture +def service(projectServiceModule): + instance = object.__new__(projectServiceModule.ProjectService) + instance.currentProject = None + instance.tomoList = {} + return instance + + +def patchRuntimeProject(service, monkeypatch, currentProject): + def loadProjectForThumbnails(dbProj): + service.currentProject = currentProject + return currentProject + + monkeypatch.setattr(service, "loadProjectForThumbnails", loadProjectForThumbnails) + + +def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode("10", status="finished", parents=["PROJECT"]), + "11": FakeRunNode("11", status="running", parents=["10"]), + } + ) + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished"}, + {"protocolId": "11", "status": "running"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": ["11"]}, + "11": {"parents": ["10"], "children": []}, + }, + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=False, + checkPid=False, + ) + + assert result["ok"] is True + assert result["summary"] == { + "runtimeProtocols": 2, + "postgresqlProtocols": 2, + "runtimeDependencies": 1, + "postgresqlDependencies": 1, + "issues": 0, + } + assert result["issues"] == { + "missingProtocols": [], + "extraProtocols": [], + "statusMismatches": [], + "missingDependencies": [], + "extraDependencies": [], + } + assert currentProject.lastRefresh is False + assert currentProject.lastCheckPids is False + + +def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode("10", status="finished", parents=["PROJECT"]), + "11": FakeRunNode("11", status="running", parents=["10"]), + } + ) + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "failed"}, + {"protocolId": "12", "status": "finished"}, + ], + adjacencyMap={ + "10": {"parents": ["99"], "children": []}, + "12": {"parents": [], "children": []}, + }, + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"] == { + "runtimeProtocols": 2, + "postgresqlProtocols": 2, + "runtimeDependencies": 1, + "postgresqlDependencies": 1, + "issues": 5, + } + assert result["issues"] == { + "missingProtocols": [ + { + "protocolId": "11", + "runtimeStatus": "running", + } + ], + "extraProtocols": [ + { + "protocolId": "12", + "postgresqlStatus": "finished", + } + ], + "statusMismatches": [ + { + "protocolId": "10", + "runtimeStatus": "finished", + "postgresqlStatus": "failed", + } + ], + "missingDependencies": [ + { + "parentId": "10", + "childId": "11", + } + ], + "extraDependencies": [ + { + "parentId": "99", + "childId": "10", + } + ], + } \ No newline at end of file From 10106acda2d53fef2ce0d2a96d0df56a3186c113 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 13:47:05 +0200 Subject: [PATCH 165/278] Expose project PostgreSQL consistency checker --- tests/conftest.py | 41 +++++++++++++++++++ tests/integration/api/test_projects_router.py | 20 +++++++++ 2 files changed, 61 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 23a27eb8..9a7b15be 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -149,6 +149,30 @@ def __init__(self): ) self.lastExportMetadataTableCall = None + self.consistencyResult = { + "ok": True, + "projectId": 1, + "summary": { + "runtimeProtocols": 2, + "postgresqlProtocols": 2, + "runtimeDependencies": 1, + "postgresqlDependencies": 1, + "issues": 0, + }, + "issues": { + "runtimeDependencies": 1, + "postgresqlDependencies": 1, + "issues": 0, + }, + "issues": { + "missingProtocols": [], + "extraProtocols": [], + "statusMismatches": [], + "missingDependencies": [], + "extraDependencies": [], + }, + } + self.lastValidateProjectPostgresqlConsistencyCall = None self.metadataTablesResult = [ { @@ -285,6 +309,23 @@ def getProjectById(self, mapper, projectId, currentUser, refresh=False, checkPid } return self.projectByIdResult + def validateProjectPostgresqlConsistency( + self, + mapper, + projectId, + currentUser, + refresh=True, + checkPid=True, + ): + self.lastValidateProjectPostgresqlConsistencyCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + "refresh": refresh, + "checkPid": checkPid, + } + return self.consistencyResult + def getProtocols(self, mapper, projectId, currentUser): return self.protocolsResult diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 904a779b..34856e82 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -66,6 +66,26 @@ def test_GetProjectCallsServiceWithRefreshAndCheckPid(projectClient, fakeProject } +def test_CheckProjectPostgresqlConsistencyCallsService(projectClient, fakeProjectService): + response = projectClient.post( + "/projects/1/consistency/check?refresh=false&checkPid=false" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.consistencyResult + assert fakeProjectService.lastValidateProjectPostgresqlConsistencyCall == { + "mapper": fakeProjectService.lastValidateProjectPostgresqlConsistencyCall["mapper"], + "projectId": 1, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + "refresh": False, + "checkPid": False, + } + + def test_LoadProtocolsReturns404WhenProjectDoesNotExist(projectClient, fakeProjectService): fakeProjectService.projectByIdResult = None From 5e70683ed4e4bd7230854fb6ae81c99b58a3e551 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 13:56:20 +0200 Subject: [PATCH 166/278] Expose project PostgreSQL consistency checker --- tests/conftest.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9a7b15be..97452a06 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -159,11 +159,6 @@ def __init__(self): "postgresqlDependencies": 1, "issues": 0, }, - "issues": { - "runtimeDependencies": 1, - "postgresqlDependencies": 1, - "issues": 0, - }, "issues": { "missingProtocols": [], "extraProtocols": [], From 6a7f546004acb4625b1e171fedc6a6a716583f06 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 14:23:09 +0200 Subject: [PATCH 167/278] Check persisted outputs in PostgreSQL consistency report --- app/backend/api/services/project_service.py | 95 +++++++++- .../test_project_service_consistency.py | 164 +++++++++++++++++- 2 files changed, 255 insertions(+), 4 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index f2e980ad..2b416413 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2561,7 +2561,6 @@ def normalizeStatus(value: Any) -> str: "thumbnailItemsUrl": self.buildProjectThumbnailItemsUrl(dbProj['id']), } - def validateProjectPostgresqlConsistency( self, mapper: PostgresqlFlatMapper, @@ -2604,6 +2603,7 @@ def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: runtimeStatuses: Dict[str, str] = {} runtimeDependencies: Set[Tuple[str, str]] = set() + runtimeOutputsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} try: runs = self.currentProject.getRunsGraph(refresh=refresh, checkPids=checkPid) @@ -2627,6 +2627,37 @@ def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: runtimeStatuses[protocolId] = normalizeStatus( self._safeCall(protocol, "getStatus", None) ) + runtimeOutputsByProtocolId.setdefault(protocolId, {}) + + if protocol is not None: + try: + for outputItem in protocol.iterOutputAttributes(): + outputName = None + outputObj = None + + if isinstance(outputItem, (tuple, list)) and len(outputItem) >= 2: + outputName = outputItem[0] + outputObj = outputItem[1] + else: + outputName = self._safeCall(outputItem, "getName", None) + outputObj = outputItem + + outputName = str(outputName or "").strip() + if not outputName or outputObj is None: + continue + + runtimeOutputsByProtocolId[protocolId][outputName] = { + "outputName": outputName, + "className": self._getScipionClassName(outputObj), + } + except Exception: + logger.debug( + "Could not inspect runtime protocol outputs during consistency check. " + "projectId=%s protocolId=%s", + projectId, + protocolId, + exc_info=True, + ) for parent in getattr(nodeObj, "_parents", []) or []: try: @@ -2684,6 +2715,31 @@ def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: postgresqlDependencies.add((parentProtocolId, childProtocolId)) + try: + persistedOutputsByProtocolId = self._loadPersistedOutputsByProtocolId( + mapper, + projectId, + ) + except Exception as e: + logger.exception( + "Failed to load PostgreSQL persisted outputs for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL persisted outputs: {e}", + ) + + runtimeOutputs: Set[Tuple[str, str]] = set() + for protocolId, outputsByName in runtimeOutputsByProtocolId.items(): + for outputName in outputsByName.keys(): + runtimeOutputs.add((protocolId, outputName)) + + postgresqlOutputs: Set[Tuple[str, str]] = set() + for protocolId, outputsByName in persistedOutputsByProtocolId.items(): + for outputName in outputsByName.keys(): + postgresqlOutputs.add((normalizeProtocolId(protocolId), str(outputName))) + runtimeProtocolIds = set(runtimeStatuses.keys()) postgresqlProtocolIds = set(postgresqlStatuses.keys()) @@ -2722,6 +2778,15 @@ def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: key=dependencySortKey, ) + missingOutputs = sorted( + runtimeOutputs - postgresqlOutputs, + key=dependencySortKey, + ) + extraOutputs = sorted( + postgresqlOutputs - runtimeOutputs, + key=dependencySortKey, + ) + issues = { "missingProtocols": [ { @@ -2746,6 +2811,32 @@ def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: buildDependency(parentId, childId) for parentId, childId in extraDependencies ], + "missingOutputs": [ + { + "protocolId": protocolId, + "outputName": outputName, + "className": runtimeOutputsByProtocolId + .get(protocolId, {}) + .get(outputName, {}) + .get("className"), + } + for protocolId, outputName in missingOutputs + ], + "extraOutputs": [ + { + "protocolId": protocolId, + "outputName": outputName, + "mapperKind": persistedOutputsByProtocolId + .get(protocolId, {}) + .get(outputName, {}) + .get("mapperKind"), + "className": persistedOutputsByProtocolId + .get(protocolId, {}) + .get(outputName, {}) + .get("className"), + } + for protocolId, outputName in extraOutputs + ], } issuesCount = sum(len(items) for items in issues.values()) @@ -2760,6 +2851,8 @@ def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: "runtimeDependencies": len(runtimeDependencies), "postgresqlDependencies": len(postgresqlDependencies), "issues": issuesCount, + "runtimeOutputs": len(runtimeOutputs), + "postgresqlOutputs": len(postgresqlOutputs), }, "issues": issues, } diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index deaa76df..fdd3abea 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -30,12 +30,23 @@ class FakeProtocol: - def __init__(self, status): + def __init__(self, status, outputs=None): self.status = status + self.outputs = outputs or [] def getStatus(self): return self.status + def iterOutputAttributes(self): + return list(self.outputs) + +class FakeOutput: + def __init__(self, className): + self.className = className + + def getClassName(self): + return self.className + class FakeParentNode: def __init__(self, name): @@ -73,10 +84,21 @@ def getRunsGraph(self, refresh=True, checkPids=True): class FakeMapper: - def __init__(self, projectRow, protocolRows, adjacencyMap): + def __init__( + self, + projectRow, + protocolRows, + adjacencyMap, + setRows=None, + treeRows=None, + ): self.projectRow = projectRow self.protocolRows = protocolRows self.adjacencyMap = adjacencyMap + self.db = FakeDb( + setRows=setRows, + treeRows=treeRows, + ) def getProject(self, projectId, userId): if int(projectId) != int(self.projectRow["id"]): @@ -92,6 +114,30 @@ def getProjectProtocolAdjacencyMap(self, projectId): return dict(self.adjacencyMap) +class FakeDb: + def __init__(self, setRows=None, treeRows=None): + self.setRows = setRows or [] + self.treeRows = treeRows or [] + self.fetchAllCalls = [] + + def fetchAll(self, query, params): + normalizedQuery = " ".join(str(query).split()) + + self.fetchAllCalls.append( + { + "query": query, + "params": params, + } + ) + + if "FROM scipion_objects o" in normalizedQuery: + return list(self.treeRows) + + if "FROM scipion_sets s" in normalizedQuery and "JOIN protocols p" in normalizedQuery: + return list(self.setRows) + + return [] + @pytest.fixture def projectServiceModule(authTestEnv): return importlib.import_module("app.backend.api.services.project_service") @@ -158,6 +204,8 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "runtimeDependencies": 1, "postgresqlDependencies": 1, "issues": 0, + "runtimeOutputs": 0, + "postgresqlOutputs": 0, } assert result["issues"] == { "missingProtocols": [], @@ -165,6 +213,8 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "statusMismatches": [], "missingDependencies": [], "extraDependencies": [], + "missingOutputs": [], + "extraOutputs": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -215,6 +265,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "runtimeDependencies": 1, "postgresqlDependencies": 1, "issues": 5, + "runtimeOutputs": 0, + "postgresqlOutputs": 0, } assert result["issues"] == { "missingProtocols": [ @@ -248,4 +300,110 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "childId": "10", } ], - } \ No newline at end of file + "missingOutputs": [], + "extraOutputs": [], + } + + +def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles")), + ("outputVolume", FakeOutput("Volume")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + { + "protocolId": "10", + "id": 100, + "objectId": 200, + "outputName": "outputParticles", + "setClassName": "SetOfParticles", + "itemClassName": "Particle", + "properties": {"itemsCount": 12}, + "createdAt": None, + "updatedAt": None, + } + ], + treeRows=[ + { + "protocolId": "10", + "id": 300, + "scipionObjId": 400, + "name": "outputExtra", + "path": "outputExtra", + "className": "Volume", + "value": None, + "label": None, + "comment": None, + "metadata": {}, + "createdAt": None, + "updatedAt": None, + } + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"] == { + "runtimeProtocols": 1, + "postgresqlProtocols": 1, + "runtimeDependencies": 0, + "postgresqlDependencies": 0, + "runtimeOutputs": 2, + "postgresqlOutputs": 2, + "issues": 2, + } + assert result["issues"]["missingOutputs"] == [ + { + "protocolId": "10", + "outputName": "outputVolume", + "className": "Volume", + } + ] + assert result["issues"]["extraOutputs"] == [ + { + "protocolId": "10", + "outputName": "outputExtra", + "mapperKind": "tree", + "className": "Volume", + } + ] + assert result["issues"]["missingProtocols"] == [] + assert result["issues"]["extraProtocols"] == [] + assert result["issues"]["statusMismatches"] == [] + assert result["issues"]["missingDependencies"] == [] + assert result["issues"]["extraDependencies"] == [] \ No newline at end of file From fe6bc3ead19c88650b85dade95b4d94a75d503e4 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 14:26:58 +0200 Subject: [PATCH 168/278] Fixing consistency --- .../backend/api/services/test_project_service_consistency.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index fdd3abea..d47c20b4 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -203,9 +203,9 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "postgresqlProtocols": 2, "runtimeDependencies": 1, "postgresqlDependencies": 1, - "issues": 0, "runtimeOutputs": 0, "postgresqlOutputs": 0, + "issues": 0, } assert result["issues"] == { "missingProtocols": [], @@ -264,9 +264,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "postgresqlProtocols": 2, "runtimeDependencies": 1, "postgresqlDependencies": 1, - "issues": 5, "runtimeOutputs": 0, "postgresqlOutputs": 0, + "issues": 5, } assert result["issues"] == { "missingProtocols": [ From 15ecb2d53c8d04d8691b14e984aa1666325d64e0 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 14:51:25 +0200 Subject: [PATCH 169/278] Check protocol steps in PostgreSQL consistency report --- app/backend/api/services/project_service.py | 167 +++++++++++++++++- .../test_project_service_consistency.py | 155 +++++++++++++++- 2 files changed, 312 insertions(+), 10 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 2b416413..c4ff1a6f 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2601,9 +2601,47 @@ def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: "childId": normalizeProtocolId(childId), } + def toOptionalInt(value: Any) -> Optional[int]: + try: + if value is None or value == "": + return None + return int(value) + except Exception: + return None + + def stepSortKey(item: Tuple[str, int]): + protocolId, stepIndex = item + return protocolSortKey(protocolId), int(stepIndex) + + def stepValue(step: Any, attrName: str, fallback: Any = None) -> Any: + try: + value = getattr(step, attrName, None) + if hasattr(value, "get"): + return value.get() + return value if value is not None else fallback + except Exception: + return fallback + + def getStepName(step: Any) -> str: + name = stepValue(step, "funcName", None) + if name: + return str(name) + + className = self._safeCall(step, "getClassName", None) + return str(className or "") + + def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[str, Any]: + return { + "protocolId": normalizeProtocolId(protocolId), + "index": int(stepIndex), + "name": str(payload.get("name") or ""), + "status": normalizeStatus(payload.get("status")), + } + runtimeStatuses: Dict[str, str] = {} runtimeDependencies: Set[Tuple[str, str]] = set() runtimeOutputsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} + runtimeStepsByProtocolId: Dict[str, Dict[int, Dict[str, Any]]] = {} try: runs = self.currentProject.getRunsGraph(refresh=refresh, checkPids=checkPid) @@ -2730,6 +2768,32 @@ def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: detail=f"Failed to load PostgreSQL persisted outputs: {e}", ) + try: + postgresqlStepsByProtocolId = mapper.getProjectProtocolStepsByProtocolId(projectId) or {} + except Exception as e: + logger.exception( + "Failed to load PostgreSQL protocol steps for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL protocol steps: {e}", + ) + + normalizedPostgresqlStepsByProtocolId: Dict[str, Dict[int, Dict[str, Any]]] = {} + for protocolId, steps in postgresqlStepsByProtocolId.items(): + protocolIdText = normalizeProtocolId(protocolId) + for step in steps or []: + stepIndex = toOptionalInt(step.get("index")) + if stepIndex is None: + continue + + normalizedPostgresqlStepsByProtocolId.setdefault(protocolIdText, {})[stepIndex] = { + "index": stepIndex, + "name": str(step.get("name") or ""), + "status": normalizeStatus(step.get("status")), + } + runtimeOutputs: Set[Tuple[str, str]] = set() for protocolId, outputsByName in runtimeOutputsByProtocolId.items(): for outputName in outputsByName.keys(): @@ -2740,6 +2804,38 @@ def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: for outputName in outputsByName.keys(): postgresqlOutputs.add((normalizeProtocolId(protocolId), str(outputName))) + runtimeStepsByProtocolId.setdefault(protocolId, {}) + if protocol is not None: + try: + for step in protocol.loadSteps() or []: + stepIndex = toOptionalInt(self._safeCall(step, "getIndex", None)) + if stepIndex is None: + continue + + runtimeStepsByProtocolId[protocolId][stepIndex] = { + "index": stepIndex, + "name": getStepName(step), + "status": normalizeStatus(self._safeCall(step, "getStatus", None)), + } + except Exception: + logger.debug( + "Could not inspect runtime protocol steps during consistency check. " + "projectId=%s protocolId=%s", + projectId, + protocolId, + exc_info=True, + ) + + runtimeSteps: Set[Tuple[str, int]] = set() + for protocolId, stepsByIndex in runtimeStepsByProtocolId.items(): + for stepIndex in stepsByIndex.keys(): + runtimeSteps.add((protocolId, int(stepIndex))) + + postgresqlSteps: Set[Tuple[str, int]] = set() + for protocolId, stepsByIndex in normalizedPostgresqlStepsByProtocolId.items(): + for stepIndex in stepsByIndex.keys(): + postgresqlSteps.add((protocolId, int(stepIndex))) + runtimeProtocolIds = set(runtimeStatuses.keys()) postgresqlProtocolIds = set(postgresqlStatuses.keys()) @@ -2787,6 +2883,42 @@ def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: key=dependencySortKey, ) + missingSteps = sorted( + runtimeSteps - postgresqlSteps, + key=stepSortKey, + ) + extraSteps = sorted( + postgresqlSteps - runtimeSteps, + key=stepSortKey, + ) + + stepMismatches = [] + for protocolId, stepIndex in sorted(runtimeSteps.intersection(postgresqlSteps), key=stepSortKey): + runtimeStep = runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) + postgresqlStep = normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) + + runtimeName = str(runtimeStep.get("name") or "") + postgresqlName = str(postgresqlStep.get("name") or "") + runtimeStatus = normalizeStatus(runtimeStep.get("status")) + postgresqlStatus = normalizeStatus(postgresqlStep.get("status")) + + changedFields = [] + if runtimeName != postgresqlName: + changedFields.append("name") + if runtimeStatus != postgresqlStatus: + changedFields.append("status") + + if changedFields: + stepMismatches.append({ + "protocolId": protocolId, + "index": int(stepIndex), + "fields": changedFields, + "runtimeName": runtimeName, + "postgresqlName": postgresqlName, + "runtimeStatus": runtimeStatus, + "postgresqlStatus": postgresqlStatus, + }) + issues = { "missingProtocols": [ { @@ -2837,6 +2969,23 @@ def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: } for protocolId, outputName in extraOutputs ], + "missingSteps": [ + buildStep( + protocolId, + stepIndex, + runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}), + ) + for protocolId, stepIndex in missingSteps + ], + "extraSteps": [ + buildStep( + protocolId, + stepIndex, + normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}), + ) + for protocolId, stepIndex in extraSteps + ], + "stepMismatches": stepMismatches, } issuesCount = sum(len(items) for items in issues.values()) @@ -2846,14 +2995,16 @@ def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: "projectId": projectId, "checkedAt": datetime.utcnow().isoformat() + "Z", "summary": { - "runtimeProtocols": len(runtimeProtocolIds), - "postgresqlProtocols": len(postgresqlProtocolIds), - "runtimeDependencies": len(runtimeDependencies), - "postgresqlDependencies": len(postgresqlDependencies), - "issues": issuesCount, - "runtimeOutputs": len(runtimeOutputs), - "postgresqlOutputs": len(postgresqlOutputs), - }, + "runtimeProtocols": len(runtimeProtocolIds), + "postgresqlProtocols": len(postgresqlProtocolIds), + "runtimeDependencies": len(runtimeDependencies), + "postgresqlDependencies": len(postgresqlDependencies), + "runtimeOutputs": len(runtimeOutputs), + "postgresqlOutputs": len(postgresqlOutputs), + "runtimeSteps": len(runtimeSteps), + "postgresqlSteps": len(postgresqlSteps), + "issues": issuesCount, + }, "issues": issues, } diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index d47c20b4..d3b1341b 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -30,9 +30,10 @@ class FakeProtocol: - def __init__(self, status, outputs=None): + def __init__(self, status, outputs=None, steps=None): self.status = status self.outputs = outputs or [] + self.steps = steps or [] def getStatus(self): return self.status @@ -40,6 +41,10 @@ def getStatus(self): def iterOutputAttributes(self): return list(self.outputs) + def loadSteps(self): + return list(self.steps) + + class FakeOutput: def __init__(self, className): self.className = className @@ -48,6 +53,29 @@ def getClassName(self): return self.className +class FakeValueHolder: + def __init__(self, value): + self.value = value + + def get(self): + return self.value + + +class FakeStep: + def __init__(self, index, name, status): + self.index = index + self.funcName = FakeValueHolder(name) + self.status = status + + def getIndex(self): + return self.index + + def getStatus(self): + return self.status + + def getClassName(self): + return "FakeStep" + class FakeParentNode: def __init__(self, name): self.name = name @@ -91,10 +119,12 @@ def __init__( adjacencyMap, setRows=None, treeRows=None, + stepsByProtocolId=None, ): self.projectRow = projectRow self.protocolRows = protocolRows self.adjacencyMap = adjacencyMap + self.stepsByProtocolId = stepsByProtocolId or {} self.db = FakeDb( setRows=setRows, treeRows=treeRows, @@ -113,6 +143,9 @@ def getProtocols(self, projectId): def getProjectProtocolAdjacencyMap(self, projectId): return dict(self.adjacencyMap) + def getProjectProtocolStepsByProtocolId(self, projectId): + return dict(self.stepsByProtocolId) + class FakeDb: def __init__(self, setRows=None, treeRows=None): @@ -205,6 +238,8 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "postgresqlDependencies": 1, "runtimeOutputs": 0, "postgresqlOutputs": 0, + "runtimeSteps": 0, + "postgresqlSteps": 0, "issues": 0, } assert result["issues"] == { @@ -215,6 +250,9 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "extraDependencies": [], "missingOutputs": [], "extraOutputs": [], + "missingSteps": [], + "extraSteps": [], + "stepMismatches": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -266,6 +304,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "postgresqlDependencies": 1, "runtimeOutputs": 0, "postgresqlOutputs": 0, + "runtimeSteps": 0, + "postgresqlSteps": 0, "issues": 5, } assert result["issues"] == { @@ -302,6 +342,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc ], "missingOutputs": [], "extraOutputs": [], + "missingSteps": [], + "extraSteps": [], + "stepMismatches": [], } @@ -385,6 +428,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( "postgresqlDependencies": 0, "runtimeOutputs": 2, "postgresqlOutputs": 2, + "runtimeSteps": 0, + "postgresqlSteps": 0, "issues": 2, } assert result["issues"]["missingOutputs"] == [ @@ -406,4 +451,110 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( assert result["issues"]["extraProtocols"] == [] assert result["issues"]["statusMismatches"] == [] assert result["issues"]["missingDependencies"] == [] - assert result["issues"]["extraDependencies"] == [] \ No newline at end of file + assert result["issues"]["extraDependencies"] == [] + assert result["issues"]["missingSteps"] == [] + assert result["issues"]["extraSteps"] == [] + assert result["issues"]["stepMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="running", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.steps = [ + FakeStep(index=1, name="importStep", status="finished"), + FakeStep(index=2, name="processStep", status="running"), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "running"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + stepsByProtocolId={ + "10": [ + { + "index": 1, + "name": "importStep", + "status": "finished", + }, + { + "index": 2, + "name": "processStep", + "status": "scheduled", + }, + { + "index": 3, + "name": "extraStep", + "status": "new", + }, + ], + }, + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"] == { + "runtimeProtocols": 1, + "postgresqlProtocols": 1, + "runtimeDependencies": 0, + "postgresqlDependencies": 0, + "runtimeOutputs": 0, + "postgresqlOutputs": 0, + "runtimeSteps": 2, + "postgresqlSteps": 3, + "issues": 2, + } + assert result["issues"]["missingSteps"] == [] + assert result["issues"]["extraSteps"] == [ + { + "protocolId": "10", + "index": 3, + "name": "extraStep", + "status": "new", + } + ] + assert result["issues"]["stepMismatches"] == [ + { + "protocolId": "10", + "index": 2, + "fields": ["status"], + "runtimeName": "processStep", + "postgresqlName": "processStep", + "runtimeStatus": "running", + "postgresqlStatus": "scheduled", + } + ] + assert result["issues"]["missingProtocols"] == [] + assert result["issues"]["extraProtocols"] == [] + assert result["issues"]["statusMismatches"] == [] + assert result["issues"]["missingDependencies"] == [] + assert result["issues"]["extraDependencies"] == [] + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] \ No newline at end of file From 4b8a5ad0280960db5660ba07ea86f01eb5ba2419 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 14:58:21 +0200 Subject: [PATCH 170/278] Fix consistency checker runtime step collection --- app/backend/api/services/project_service.py | 44 +++++------ .../test_project_service_consistency.py | 78 ++++++++++++++++++- 2 files changed, 99 insertions(+), 23 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index c4ff1a6f..6b78dcd9 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2696,6 +2696,28 @@ def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[ protocolId, exc_info=True, ) + runtimeStepsByProtocolId.setdefault(protocolId, {}) + + if protocol is not None: + try: + for step in protocol.loadSteps() or []: + stepIndex = toOptionalInt(self._safeCall(step, "getIndex", None)) + if stepIndex is None: + continue + + runtimeStepsByProtocolId[protocolId][stepIndex] = { + "index": stepIndex, + "name": getStepName(step), + "status": normalizeStatus(self._safeCall(step, "getStatus", None)), + } + except Exception: + logger.debug( + "Could not inspect runtime protocol steps during consistency check. " + "projectId=%s protocolId=%s", + projectId, + protocolId, + exc_info=True, + ) for parent in getattr(nodeObj, "_parents", []) or []: try: @@ -2804,28 +2826,6 @@ def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[ for outputName in outputsByName.keys(): postgresqlOutputs.add((normalizeProtocolId(protocolId), str(outputName))) - runtimeStepsByProtocolId.setdefault(protocolId, {}) - if protocol is not None: - try: - for step in protocol.loadSteps() or []: - stepIndex = toOptionalInt(self._safeCall(step, "getIndex", None)) - if stepIndex is None: - continue - - runtimeStepsByProtocolId[protocolId][stepIndex] = { - "index": stepIndex, - "name": getStepName(step), - "status": normalizeStatus(self._safeCall(step, "getStatus", None)), - } - except Exception: - logger.debug( - "Could not inspect runtime protocol steps during consistency check. " - "projectId=%s protocolId=%s", - projectId, - protocolId, - exc_info=True, - ) - runtimeSteps: Set[Tuple[str, int]] = set() for protocolId, stepsByIndex in runtimeStepsByProtocolId.items(): for stepIndex in stepsByIndex.keys(): diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index d3b1341b..56b46b08 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -557,4 +557,80 @@ def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( assert result["issues"]["missingDependencies"] == [] assert result["issues"]["extraDependencies"] == [] assert result["issues"]["missingOutputs"] == [] - assert result["issues"]["extraOutputs"] == [] \ No newline at end of file + assert result["issues"]["extraOutputs"] == [] + + +def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + "11": FakeRunNode( + "11", + status="running", + parents=["10"], + ), + } + ) + currentProject.nodes["10"].run.steps = [ + FakeStep(index=1, name="importStep", status="finished"), + ] + currentProject.nodes["11"].run.steps = [ + FakeStep(index=1, name="processStep", status="running"), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished"}, + {"protocolId": "11", "status": "running"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": ["11"]}, + "11": {"parents": ["10"], "children": []}, + }, + stepsByProtocolId={ + "10": [ + { + "index": 1, + "name": "importStep", + "status": "finished", + } + ], + "11": [ + { + "index": 1, + "name": "processStep", + "status": "running", + } + ], + }, + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is True + assert result["summary"]["runtimeSteps"] == 2 + assert result["summary"]["postgresqlSteps"] == 2 + assert result["issues"]["missingSteps"] == [] + assert result["issues"]["extraSteps"] == [] + assert result["issues"]["stepMismatches"] == [] \ No newline at end of file From d9f83cdc8d9cb35d562ffd9ddd86c03c23f8e958 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 15:25:59 +0200 Subject: [PATCH 171/278] Fix consistency checker runtime step collection --- app/backend/api/services/project_service.py | 39 ++++++++++----------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 6b78dcd9..baea5e02 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2698,26 +2698,25 @@ def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[ ) runtimeStepsByProtocolId.setdefault(protocolId, {}) - if protocol is not None: - try: - for step in protocol.loadSteps() or []: - stepIndex = toOptionalInt(self._safeCall(step, "getIndex", None)) - if stepIndex is None: - continue - - runtimeStepsByProtocolId[protocolId][stepIndex] = { - "index": stepIndex, - "name": getStepName(step), - "status": normalizeStatus(self._safeCall(step, "getStatus", None)), - } - except Exception: - logger.debug( - "Could not inspect runtime protocol steps during consistency check. " - "projectId=%s protocolId=%s", - projectId, - protocolId, - exc_info=True, - ) + try: + for step in protocol.loadSteps() or []: + stepIndex = toOptionalInt(self._safeCall(step, "getIndex", None)) + if stepIndex is None: + continue + + runtimeStepsByProtocolId[protocolId][stepIndex] = { + "index": stepIndex, + "name": getStepName(step), + "status": normalizeStatus(self._safeCall(step, "getStatus", None)), + } + except Exception: + logger.debug( + "Could not inspect runtime protocol steps during consistency check. " + "projectId=%s protocolId=%s", + projectId, + protocolId, + exc_info=True, + ) for parent in getattr(nodeObj, "_parents", []) or []: try: From 25a926f1a39677c79ba96bc6270792ca0585ae20 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 15:42:41 +0200 Subject: [PATCH 172/278] Check protocol input refs in PostgreSQL consistency report --- app/backend/api/services/project_service.py | 241 +++++++++++++++++- .../test_project_service_consistency.py | 197 +++++++++++++- 2 files changed, 426 insertions(+), 12 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index baea5e02..b74ac1b5 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2638,10 +2638,98 @@ def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[ "status": normalizeStatus(payload.get("status")), } + def normalizeOptionalText(value: Any) -> Optional[str]: + if value is None or value == "": + return None + + text = str(value).strip() + return text or None + + def inputRefSortKey(item: Tuple[str, str, int]): + protocolId, inputName, itemIndex = item + return protocolSortKey(protocolId), str(inputName), int(itemIndex) + + def buildInputRef( + key: Tuple[str, str, int], + payload: Dict[str, Any], + ) -> Dict[str, Any]: + protocolId, inputName, itemIndex = key + return { + "protocolId": normalizeProtocolId(protocolId), + "inputName": str(inputName), + "itemIndex": int(itemIndex), + "parentProtocolId": normalizeOptionalText(payload.get("parentProtocolId")), + "parentOutputName": normalizeOptionalText(payload.get("parentOutputName")), + "objectClassName": normalizeOptionalText(payload.get("objectClassName")), + } + + def iterPointerItems(attr: Any) -> List[Tuple[int, Any]]: + try: + if isinstance(attr, PointerList): + return [ + (index, pointer) + for index, pointer in enumerate(attr) + ] + except Exception: + pass + + return [(0, attr)] + + def extractRuntimeInputRef( + protocolId: str, + inputName: str, + itemIndex: int, + pointer: Any, + ) -> Optional[Dict[str, Any]]: + parentProtocolId = None + parentOutputName = None + objectClassName = None + objectId = None + + try: + parentObj = pointer.getObjValue() + parentProtocolId = normalizeOptionalText( + self._safeCall(parentObj, "getObjId", None) + ) + except Exception: + parentProtocolId = None + + try: + parentOutputName = normalizeOptionalText(pointer.getExtended()) + except Exception: + parentOutputName = None + + try: + targetObj = pointer.get() + if targetObj is not None: + objectClassName = normalizeOptionalText( + self._getScipionClassName(targetObj) + ) + objectId = normalizeOptionalText( + self._safeCall(targetObj, "getObjId", None) + ) + except Exception: + objectClassName = None + objectId = None + + if parentProtocolId is None and parentOutputName is None: + return None + + return { + "protocolId": normalizeProtocolId(protocolId), + "inputName": str(inputName), + "itemIndex": int(itemIndex), + "parentProtocolId": parentProtocolId, + "parentOutputName": parentOutputName, + "objectClassName": objectClassName, + "objectId": objectId, + } + runtimeStatuses: Dict[str, str] = {} runtimeDependencies: Set[Tuple[str, str]] = set() runtimeOutputsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} runtimeStepsByProtocolId: Dict[str, Dict[int, Dict[str, Any]]] = {} + runtimeInputRefsByKey: Dict[Tuple[str, str, int], Dict[str, Any]] = {} try: runs = self.currentProject.getRunsGraph(refresh=refresh, checkPids=checkPid) @@ -2717,6 +2805,37 @@ def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[ protocolId, exc_info=True, ) + try: + for inputName, attr in protocol.iterInputAttributes(): + inputNameText = str(inputName or "").strip() + if not inputNameText: + continue + + for itemIndex, pointer in iterPointerItems(attr): + inputRef = extractRuntimeInputRef( + protocolId=protocolId, + inputName=inputNameText, + itemIndex=int(itemIndex), + pointer=pointer, + ) + if inputRef is None: + continue + + key = ( + inputRef["protocolId"], + inputRef["inputName"], + inputRef["itemIndex"], + ) + runtimeInputRefsByKey[key] = inputRef + except Exception: + logger.debug( + "Could not inspect runtime protocol input refs during consistency check. " + "projectId=%s protocolId=%s", + projectId, + protocolId, + exc_info=True, + ) + for parent in getattr(nodeObj, "_parents", []) or []: try: @@ -2815,6 +2934,41 @@ def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[ "status": normalizeStatus(step.get("status")), } + try: + postgresqlInputRefs = mapper.listProtocolInputRefs(projectId) or [] + except Exception as e: + logger.exception( + "Failed to load PostgreSQL protocol input refs for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL protocol input refs: {e}", + ) + + postgresqlInputRefsByKey: Dict[Tuple[str, str, int], Dict[str, Any]] = {} + for ref in postgresqlInputRefs: + protocolIdText = normalizeProtocolId(ref.get("protocolId")) + inputName = str(ref.get("inputName") or "").strip() + itemIndex = toOptionalInt(ref.get("itemIndex")) + + if not protocolIdText or not inputName: + continue + + if itemIndex is None or itemIndex < 0: + itemIndex = 0 + + key = (protocolIdText, inputName, int(itemIndex)) + postgresqlInputRefsByKey[key] = { + "protocolId": protocolIdText, + "inputName": inputName, + "itemIndex": int(itemIndex), + "parentProtocolId": normalizeOptionalText(ref.get("parentProtocolId")), + "parentOutputName": normalizeOptionalText(ref.get("parentOutputName")), + "objectClassName": normalizeOptionalText(ref.get("objectClassName")), + "objectId": normalizeOptionalText(ref.get("objectId")), + } + runtimeOutputs: Set[Tuple[str, str]] = set() for protocolId, outputsByName in runtimeOutputsByProtocolId.items(): for outputName in outputsByName.keys(): @@ -2837,6 +2991,8 @@ def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[ runtimeProtocolIds = set(runtimeStatuses.keys()) postgresqlProtocolIds = set(postgresqlStatuses.keys()) + runtimeInputRefs = set(runtimeInputRefsByKey.keys()) + postgresqlInputRefsKeys = set(postgresqlInputRefsByKey.keys()) missingProtocolIds = sorted( runtimeProtocolIds - postgresqlProtocolIds, @@ -2918,6 +3074,60 @@ def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[ "postgresqlStatus": postgresqlStatus, }) + + missingInputRefs = sorted( + runtimeInputRefs - postgresqlInputRefsKeys, + key=inputRefSortKey, + ) + extraInputRefs = sorted( + postgresqlInputRefsKeys - runtimeInputRefs, + key=inputRefSortKey, + ) + + inputRefMismatches = [] + for key in sorted(runtimeInputRefs.intersection(postgresqlInputRefsKeys), key=inputRefSortKey): + runtimeRef = runtimeInputRefsByKey.get(key, {}) + postgresqlRef = postgresqlInputRefsByKey.get(key, {}) + + changedFields = [] + + runtimeParentProtocolId = normalizeOptionalText(runtimeRef.get("parentProtocolId")) + postgresqlParentProtocolId = normalizeOptionalText(postgresqlRef.get("parentProtocolId")) + + runtimeParentOutputName = normalizeOptionalText(runtimeRef.get("parentOutputName")) + postgresqlParentOutputName = normalizeOptionalText(postgresqlRef.get("parentOutputName")) + + runtimeObjectClassName = normalizeOptionalText(runtimeRef.get("objectClassName")) + postgresqlObjectClassName = normalizeOptionalText(postgresqlRef.get("objectClassName")) + + if runtimeParentProtocolId != postgresqlParentProtocolId: + changedFields.append("parentProtocolId") + + if runtimeParentOutputName != postgresqlParentOutputName: + changedFields.append("parentOutputName") + + if ( + runtimeObjectClassName is not None + and postgresqlObjectClassName is not None + and runtimeObjectClassName != postgresqlObjectClassName + ): + changedFields.append("objectClassName") + + if changedFields: + protocolId, inputName, itemIndex = key + inputRefMismatches.append({ + "protocolId": protocolId, + "inputName": inputName, + "itemIndex": int(itemIndex), + "fields": changedFields, + "runtimeParentProtocolId": runtimeParentProtocolId, + "postgresqlParentProtocolId": postgresqlParentProtocolId, + "runtimeParentOutputName": runtimeParentOutputName, + "postgresqlParentOutputName": postgresqlParentOutputName, + "runtimeObjectClassName": runtimeObjectClassName, + "postgresqlObjectClassName": postgresqlObjectClassName, + }) + issues = { "missingProtocols": [ { @@ -2985,6 +3195,15 @@ def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[ for protocolId, stepIndex in extraSteps ], "stepMismatches": stepMismatches, + "missingInputRefs": [ + buildInputRef(key, runtimeInputRefsByKey.get(key, {})) + for key in missingInputRefs + ], + "extraInputRefs": [ + buildInputRef(key, postgresqlInputRefsByKey.get(key, {})) + for key in extraInputRefs + ], + "inputRefMismatches": inputRefMismatches, } issuesCount = sum(len(items) for items in issues.values()) @@ -2994,16 +3213,18 @@ def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[ "projectId": projectId, "checkedAt": datetime.utcnow().isoformat() + "Z", "summary": { - "runtimeProtocols": len(runtimeProtocolIds), - "postgresqlProtocols": len(postgresqlProtocolIds), - "runtimeDependencies": len(runtimeDependencies), - "postgresqlDependencies": len(postgresqlDependencies), - "runtimeOutputs": len(runtimeOutputs), - "postgresqlOutputs": len(postgresqlOutputs), - "runtimeSteps": len(runtimeSteps), - "postgresqlSteps": len(postgresqlSteps), - "issues": issuesCount, - }, + "runtimeProtocols": len(runtimeProtocolIds), + "postgresqlProtocols": len(postgresqlProtocolIds), + "runtimeDependencies": len(runtimeDependencies), + "postgresqlDependencies": len(postgresqlDependencies), + "runtimeOutputs": len(runtimeOutputs), + "postgresqlOutputs": len(postgresqlOutputs), + "runtimeSteps": len(runtimeSteps), + "postgresqlSteps": len(postgresqlSteps), + "runtimeInputRefs": len(runtimeInputRefs), + "postgresqlInputRefs": len(postgresqlInputRefsKeys), + "issues": issuesCount, + }, "issues": issues, } diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 56b46b08..d1ffa545 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -30,10 +30,11 @@ class FakeProtocol: - def __init__(self, status, outputs=None, steps=None): + def __init__(self, status, outputs=None, steps=None, inputs=None): self.status = status self.outputs = outputs or [] self.steps = steps or [] + self.inputs = inputs or [] def getStatus(self): return self.status @@ -44,6 +45,37 @@ def iterOutputAttributes(self): def loadSteps(self): return list(self.steps) + def iterInputAttributes(self): + return list(self.inputs) + + +class FakePointerTarget: + def __init__(self, objId, className): + self.objId = objId + self.className = className + + def getObjId(self): + return self.objId + + def getClassName(self): + return self.className + + +class FakePointer: + def __init__(self, parentProtocolId, outputName, className="SetOfParticles", objectId=100): + self.parent = FakePointerTarget(parentProtocolId, className) + self.target = FakePointerTarget(objectId, className) + self.outputName = outputName + + def getObjValue(self): + return self.parent + + def getExtended(self): + return self.outputName + + def get(self): + return self.target + class FakeOutput: def __init__(self, className): @@ -76,6 +108,7 @@ def getStatus(self): def getClassName(self): return "FakeStep" + class FakeParentNode: def __init__(self, name): self.name = name @@ -120,11 +153,13 @@ def __init__( setRows=None, treeRows=None, stepsByProtocolId=None, + inputRefs=None, ): self.projectRow = projectRow self.protocolRows = protocolRows self.adjacencyMap = adjacencyMap self.stepsByProtocolId = stepsByProtocolId or {} + self.inputRefs = inputRefs or [] self.db = FakeDb( setRows=setRows, treeRows=treeRows, @@ -146,6 +181,9 @@ def getProjectProtocolAdjacencyMap(self, projectId): def getProjectProtocolStepsByProtocolId(self, projectId): return dict(self.stepsByProtocolId) + def listProtocolInputRefs(self, projectId): + return list(self.inputRefs) + class FakeDb: def __init__(self, setRows=None, treeRows=None): @@ -240,6 +278,8 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "postgresqlOutputs": 0, "runtimeSteps": 0, "postgresqlSteps": 0, + "runtimeInputRefs": 0, + "postgresqlInputRefs": 0, "issues": 0, } assert result["issues"] == { @@ -253,6 +293,9 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "missingSteps": [], "extraSteps": [], "stepMismatches": [], + "missingInputRefs": [], + "extraInputRefs": [], + "inputRefMismatches": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -306,6 +349,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "postgresqlOutputs": 0, "runtimeSteps": 0, "postgresqlSteps": 0, + "runtimeInputRefs": 0, + "postgresqlInputRefs": 0, "issues": 5, } assert result["issues"] == { @@ -345,6 +390,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "missingSteps": [], "extraSteps": [], "stepMismatches": [], + "missingInputRefs": [], + "extraInputRefs": [], + "inputRefMismatches": [], } @@ -430,6 +478,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( "postgresqlOutputs": 2, "runtimeSteps": 0, "postgresqlSteps": 0, + "runtimeInputRefs": 0, + "postgresqlInputRefs": 0, "issues": 2, } assert result["issues"]["missingOutputs"] == [ @@ -455,6 +505,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( assert result["issues"]["missingSteps"] == [] assert result["issues"]["extraSteps"] == [] assert result["issues"]["stepMismatches"] == [] + assert result["issues"]["missingInputRefs"] == [] + assert result["issues"]["extraInputRefs"] == [] + assert result["issues"]["inputRefMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( @@ -529,6 +582,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( "postgresqlOutputs": 0, "runtimeSteps": 2, "postgresqlSteps": 3, + "runtimeInputRefs": 0, + "postgresqlInputRefs": 0, "issues": 2, } assert result["issues"]["missingSteps"] == [] @@ -558,6 +613,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( assert result["issues"]["extraDependencies"] == [] assert result["issues"]["missingOutputs"] == [] assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["missingInputRefs"] == [] + assert result["issues"]["extraInputRefs"] == [] + assert result["issues"]["inputRefMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols( @@ -633,4 +691,139 @@ def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols assert result["summary"]["postgresqlSteps"] == 2 assert result["issues"]["missingSteps"] == [] assert result["issues"]["extraSteps"] == [] - assert result["issues"]["stepMismatches"] == [] \ No newline at end of file + assert result["issues"]["stepMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + "11": FakeRunNode( + "11", + status="running", + parents=["10"], + ), + } + ) + currentProject.nodes["11"].run.inputs = [ + ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), + ("inputVolume", FakePointer(parentProtocolId=10, outputName="outputVolume", className="Volume")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished"}, + {"protocolId": "11", "status": "running"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": ["11"]}, + "11": {"parents": ["10"], "children": []}, + }, + inputRefs=[ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "wrongOutput", + "objectClassName": "SetOfParticles", + "objectId": "100", + }, + { + "protocolId": "11", + "inputName": "inputMask", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputMask", + "objectClassName": "VolumeMask", + "objectId": "101", + }, + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"] == { + "runtimeProtocols": 2, + "postgresqlProtocols": 2, + "runtimeDependencies": 1, + "postgresqlDependencies": 1, + "runtimeOutputs": 0, + "postgresqlOutputs": 0, + "runtimeSteps": 0, + "postgresqlSteps": 0, + "runtimeInputRefs": 2, + "postgresqlInputRefs": 2, + "issues": 3, + } + + assert result["issues"]["missingInputRefs"] == [ + { + "protocolId": "11", + "inputName": "inputVolume", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputVolume", + "objectClassName": "Volume", + } + ] + + assert result["issues"]["extraInputRefs"] == [ + { + "protocolId": "11", + "inputName": "inputMask", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputMask", + "objectClassName": "VolumeMask", + } + ] + + assert result["issues"]["inputRefMismatches"] == [ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "fields": ["parentOutputName"], + "runtimeParentProtocolId": "10", + "postgresqlParentProtocolId": "10", + "runtimeParentOutputName": "outputParticles", + "postgresqlParentOutputName": "wrongOutput", + "runtimeObjectClassName": "SetOfParticles", + "postgresqlObjectClassName": "SetOfParticles", + } + ] + + assert result["issues"]["missingProtocols"] == [] + assert result["issues"]["extraProtocols"] == [] + assert result["issues"]["statusMismatches"] == [] + assert result["issues"]["missingDependencies"] == [] + assert result["issues"]["extraDependencies"] == [] + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["missingSteps"] == [] + assert result["issues"]["extraSteps"] == [] + assert result["issues"]["stepMismatches"] == [] + From b4d05d6152b0f00fba97ec810b586e7428f10c96 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 16:10:25 +0200 Subject: [PATCH 173/278] Check input refs against protocol dependencies --- app/backend/api/services/project_service.py | 63 ++++++ .../test_project_service_consistency.py | 209 +++++++++++++++++- 2 files changed, 269 insertions(+), 3 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index b74ac1b5..dddd97ab 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2601,6 +2601,21 @@ def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: "childId": normalizeProtocolId(childId), } + def dependencyKeyFromInputRef(payload: Dict[str, Any]) -> Optional[Tuple[str, str]]: + parentProtocolId = normalizeOptionalText(payload.get("parentProtocolId")) + childProtocolId = normalizeOptionalText(payload.get("protocolId")) + + if not parentProtocolId or not childProtocolId: + return None + + if parentProtocolId == "PROJECT" or childProtocolId == "PROJECT": + return None + + if parentProtocolId == childProtocolId: + return None + + return parentProtocolId, childProtocolId + def toOptionalInt(value: Any) -> Optional[int]: try: if value is None or value == "": @@ -2993,6 +3008,18 @@ def extractRuntimeInputRef( postgresqlProtocolIds = set(postgresqlStatuses.keys()) runtimeInputRefs = set(runtimeInputRefsByKey.keys()) postgresqlInputRefsKeys = set(postgresqlInputRefsByKey.keys()) + runtimeDependenciesFromInputRefs: Set[Tuple[str, str]] = set() + + for inputRef in runtimeInputRefsByKey.values(): + dependencyKey = dependencyKeyFromInputRef(inputRef) + if dependencyKey is not None: + runtimeDependenciesFromInputRefs.add(dependencyKey) + + postgresqlDependenciesFromInputRefs: Set[Tuple[str, str]] = set() + for inputRef in postgresqlInputRefsByKey.values(): + dependencyKey = dependencyKeyFromInputRef(inputRef) + if dependencyKey is not None: + postgresqlDependenciesFromInputRefs.add(dependencyKey) missingProtocolIds = sorted( runtimeProtocolIds - postgresqlProtocolIds, @@ -3128,6 +3155,24 @@ def extractRuntimeInputRef( "postgresqlObjectClassName": postgresqlObjectClassName, }) + runtimeInputRefDependenciesMissing = sorted( + runtimeDependenciesFromInputRefs - runtimeDependencies, + key=dependencySortKey, + ) + runtimeDependenciesWithoutInputRefs = sorted( + runtimeDependencies - runtimeDependenciesFromInputRefs, + key=dependencySortKey, + ) + + postgresqlInputRefDependenciesMissing = sorted( + postgresqlDependenciesFromInputRefs - postgresqlDependencies, + key=dependencySortKey, + ) + postgresqlDependenciesWithoutInputRefs = sorted( + postgresqlDependencies - postgresqlDependenciesFromInputRefs, + key=dependencySortKey, + ) + issues = { "missingProtocols": [ { @@ -3204,6 +3249,22 @@ def extractRuntimeInputRef( for key in extraInputRefs ], "inputRefMismatches": inputRefMismatches, + "runtimeInputRefDependenciesMissing": [ + buildDependency(parentId, childId) + for parentId, childId in runtimeInputRefDependenciesMissing + ], + "runtimeDependenciesWithoutInputRefs": [ + buildDependency(parentId, childId) + for parentId, childId in runtimeDependenciesWithoutInputRefs + ], + "postgresqlInputRefDependenciesMissing": [ + buildDependency(parentId, childId) + for parentId, childId in postgresqlInputRefDependenciesMissing + ], + "postgresqlDependenciesWithoutInputRefs": [ + buildDependency(parentId, childId) + for parentId, childId in postgresqlDependenciesWithoutInputRefs + ], } issuesCount = sum(len(items) for items in issues.values()) @@ -3223,6 +3284,8 @@ def extractRuntimeInputRef( "postgresqlSteps": len(postgresqlSteps), "runtimeInputRefs": len(runtimeInputRefs), "postgresqlInputRefs": len(postgresqlInputRefsKeys), + "runtimeInputRefDependencies": len(runtimeDependenciesFromInputRefs), + "postgresqlInputRefDependencies": len(postgresqlDependenciesFromInputRefs), "issues": issuesCount, }, "issues": issues, diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index d1ffa545..2d0a3755 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -242,6 +242,9 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "11": FakeRunNode("11", status="running", parents=["10"]), } ) + currentProject.nodes["11"].run.inputs = [ + ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), + ] patchRuntimeProject(service, monkeypatch, currentProject) mapper = FakeMapper( @@ -249,6 +252,7 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "id": 1, "ownerId": 7, "name": str(tmp_path), + }, protocolRows=[ {"protocolId": "10", "status": "finished"}, @@ -258,6 +262,17 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "10": {"parents": [], "children": ["11"]}, "11": {"parents": ["10"], "children": []}, }, + inputRefs=[ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "objectId": "100", + }, + ], ) result = service.validateProjectPostgresqlConsistency( @@ -278,8 +293,10 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "postgresqlOutputs": 0, "runtimeSteps": 0, "postgresqlSteps": 0, - "runtimeInputRefs": 0, - "postgresqlInputRefs": 0, + "runtimeInputRefs": 1, + "postgresqlInputRefs": 1, + "runtimeInputRefDependencies": 1, + "postgresqlInputRefDependencies": 1, "issues": 0, } assert result["issues"] == { @@ -296,6 +313,10 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "missingInputRefs": [], "extraInputRefs": [], "inputRefMismatches": [], + "runtimeInputRefDependenciesMissing": [], + "runtimeDependenciesWithoutInputRefs": [], + "postgresqlInputRefDependenciesMissing": [], + "postgresqlDependenciesWithoutInputRefs": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -351,7 +372,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "postgresqlSteps": 0, "runtimeInputRefs": 0, "postgresqlInputRefs": 0, - "issues": 5, + "runtimeInputRefDependencies": 0, + "postgresqlInputRefDependencies": 0, + "issues": 7, } assert result["issues"] == { "missingProtocols": [ @@ -393,6 +416,20 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "missingInputRefs": [], "extraInputRefs": [], "inputRefMismatches": [], + "runtimeInputRefDependenciesMissing": [], + "runtimeDependenciesWithoutInputRefs": [ + { + "parentId": "10", + "childId": "11", + } + ], + "postgresqlInputRefDependenciesMissing": [], + "postgresqlDependenciesWithoutInputRefs": [ + { + "parentId": "99", + "childId": "10", + } + ], } @@ -480,6 +517,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( "postgresqlSteps": 0, "runtimeInputRefs": 0, "postgresqlInputRefs": 0, + "runtimeInputRefDependencies": 0, + "postgresqlInputRefDependencies": 0, "issues": 2, } assert result["issues"]["missingOutputs"] == [ @@ -508,6 +547,10 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( assert result["issues"]["missingInputRefs"] == [] assert result["issues"]["extraInputRefs"] == [] assert result["issues"]["inputRefMismatches"] == [] + assert result["issues"]["runtimeInputRefDependenciesMissing"] == [] + assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] + assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] + assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( @@ -584,6 +627,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( "postgresqlSteps": 3, "runtimeInputRefs": 0, "postgresqlInputRefs": 0, + "runtimeInputRefDependencies": 0, + "postgresqlInputRefDependencies": 0, "issues": 2, } assert result["issues"]["missingSteps"] == [] @@ -616,6 +661,10 @@ def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( assert result["issues"]["missingInputRefs"] == [] assert result["issues"]["extraInputRefs"] == [] assert result["issues"]["inputRefMismatches"] == [] + assert result["issues"]["runtimeInputRefDependenciesMissing"] == [] + assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] + assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] + assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols( @@ -638,6 +687,9 @@ def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols ), } ) + currentProject.nodes["11"].run.inputs = [ + ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), + ] currentProject.nodes["10"].run.steps = [ FakeStep(index=1, name="importStep", status="finished"), ] @@ -676,6 +728,17 @@ def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols } ], }, + inputRefs=[ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "objectId": "100", + }, + ], ) result = service.validateProjectPostgresqlConsistency( @@ -692,6 +755,15 @@ def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols assert result["issues"]["missingSteps"] == [] assert result["issues"]["extraSteps"] == [] assert result["issues"]["stepMismatches"] == [] + assert result["summary"]["runtimeInputRefs"] == 1 + assert result["summary"]["postgresqlInputRefs"] == 1 + assert result["summary"]["runtimeInputRefDependencies"] == 1 + assert result["summary"]["postgresqlInputRefDependencies"] == 1 + + assert result["issues"]["runtimeInputRefDependenciesMissing"] == [] + assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] + assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] + assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( @@ -776,6 +848,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( "postgresqlSteps": 0, "runtimeInputRefs": 2, "postgresqlInputRefs": 2, + "runtimeInputRefDependencies": 1, + "postgresqlInputRefDependencies": 1, "issues": 3, } @@ -826,4 +900,133 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( assert result["issues"]["missingSteps"] == [] assert result["issues"]["extraSteps"] == [] assert result["issues"]["stepMismatches"] == [] + assert result["issues"]["runtimeInputRefDependenciesMissing"] == [] + assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] + assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] + assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + "11": FakeRunNode( + "11", + status="running", + parents=["10"], + ), + "12": FakeRunNode( + "12", + status="running", + parents=[], + ), + } + ) + currentProject.nodes["11"].run.inputs = [ + ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), + ] + currentProject.nodes["12"].run.inputs = [ + ("inputVolume", FakePointer(parentProtocolId=10, outputName="outputVolume", className="Volume")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished"}, + {"protocolId": "11", "status": "running"}, + {"protocolId": "12", "status": "running"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": ["11"]}, + "11": {"parents": ["10"], "children": []}, + "12": {"parents": [], "children": []}, + }, + inputRefs=[ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "objectId": "100", + }, + { + "protocolId": "12", + "inputName": "inputVolume", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputVolume", + "objectClassName": "Volume", + "objectId": "101", + }, + ], + ) + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"] == { + "runtimeProtocols": 3, + "postgresqlProtocols": 3, + "runtimeDependencies": 1, + "postgresqlDependencies": 1, + "runtimeOutputs": 0, + "postgresqlOutputs": 0, + "runtimeSteps": 0, + "postgresqlSteps": 0, + "runtimeInputRefs": 2, + "postgresqlInputRefs": 2, + "runtimeInputRefDependencies": 2, + "postgresqlInputRefDependencies": 2, + "issues": 2, + } + + assert result["issues"]["runtimeInputRefDependenciesMissing"] == [ + { + "parentId": "10", + "childId": "12", + } + ] + assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] + assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [ + { + "parentId": "10", + "childId": "12", + } + ] + assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] + + assert result["issues"]["missingProtocols"] == [] + assert result["issues"]["extraProtocols"] == [] + assert result["issues"]["statusMismatches"] == [] + assert result["issues"]["missingDependencies"] == [] + assert result["issues"]["extraDependencies"] == [] + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["missingSteps"] == [] + assert result["issues"]["extraSteps"] == [] + assert result["issues"]["stepMismatches"] == [] + assert result["issues"]["missingInputRefs"] == [] + assert result["issues"]["extraInputRefs"] == [] + assert result["issues"]["inputRefMismatches"] == [] \ No newline at end of file From e5a4c4470243cf37ad7cb3c3627d99725c32a01a Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 16:33:45 +0200 Subject: [PATCH 174/278] Check protocol params in PostgreSQL consistency report --- app/backend/api/services/project_service.py | 171 ++++++++++++++++++ .../test_project_service_consistency.py | 115 +++++++++++- 2 files changed, 284 insertions(+), 2 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index dddd97ab..0821f157 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2740,11 +2740,130 @@ def extractRuntimeInputRef( "objectId": objectId, } + def normalizeParamValue(value: Any) -> Any: + if value is None: + return None + + if isinstance(value, bool): + return value + + if isinstance(value, (int, float)): + return value + + if isinstance(value, str): + text = value.strip() + if text == "": + return "" + lowerText = text.lower() + if lowerText in ("true", "false"): + return lowerText == "true" + + try: + if "." not in text: + return int(text) + except Exception: + pass + + try: + return float(text) + except Exception: + return text + + if isinstance(value, (list, tuple)): + return [normalizeParamValue(item) for item in value] + + if isinstance(value, dict): + return { + str(key): normalizeParamValue(itemValue) + for key, itemValue in value.items() + } + + try: + if hasattr(value, "get"): + return normalizeParamValue(value.get()) + except Exception: + pass + + return str(value) + + def paramSortKey(item: Tuple[str, str]): + protocolId, paramName = item + return protocolSortKey(protocolId), str(paramName) + + def buildParamIssue( + key: Tuple[str, str], + payload: Dict[str, Any], + ) -> Dict[str, Any]: + protocolId, paramName = key + return { + "protocolId": normalizeProtocolId(protocolId), + "paramName": str(paramName), + "value": payload.get("value"), + } + + def isPointerParam(param: Any) -> bool: + return isinstance(param, (PointerParam, MultiPointerParam, RelationParam)) + + def extractRuntimeParams(protocol: Any) -> Dict[str, Dict[str, Any]]: + paramsByName: Dict[str, Dict[str, Any]] = {} + + try: + self.currentProject._fixProtParamsConfiguration(protocol) + except Exception: + pass + + try: + for paramName, param in protocol.iterParams(): + paramNameText = str(paramName or "").strip() + if not paramNameText: + continue + + if isPointerParam(param): + continue + + rawValue = None + try: + rawValue = protocol.getAttributeValue(paramNameText) + except Exception: + try: + rawValue = getattr(protocol, paramNameText, None) + except Exception: + rawValue = None + + paramsByName[paramNameText] = { + "value": normalizeParamValue(rawValue), + } + except Exception: + logger.debug( + "Could not inspect runtime protocol params during consistency check.", + exc_info=True, + ) + + return paramsByName + + def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + rawParams = row.get("params") + if not isinstance(rawParams, dict): + return {} + + paramsByName: Dict[str, Dict[str, Any]] = {} + for paramName, rawValue in rawParams.items(): + paramNameText = str(paramName or "").strip() + if not paramNameText: + continue + + paramsByName[paramNameText] = { + "value": normalizeParamValue(rawValue), + } + + return paramsByName + runtimeStatuses: Dict[str, str] = {} runtimeDependencies: Set[Tuple[str, str]] = set() runtimeOutputsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} runtimeStepsByProtocolId: Dict[str, Dict[int, Dict[str, Any]]] = {} runtimeInputRefsByKey: Dict[Tuple[str, str, int], Dict[str, Any]] = {} + runtimeParamsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} try: runs = self.currentProject.getRunsGraph(refresh=refresh, checkPids=checkPid) @@ -2851,6 +2970,7 @@ def extractRuntimeInputRef( exc_info=True, ) + runtimeParamsByProtocolId[protocolId] = extractRuntimeParams(protocol) for parent in getattr(nodeObj, "_parents", []) or []: try: @@ -2876,12 +2996,15 @@ def extractRuntimeInputRef( ) postgresqlStatuses: Dict[str, str] = {} + postgresqlParamsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} + for row in protocolRows: protocolId = normalizeProtocolId(row.get("protocolId")) if not protocolId: continue postgresqlStatuses[protocolId] = normalizeStatus(row.get("status")) + postgresqlParamsByProtocolId[protocolId] = extractPostgresqlParams(row) try: adjacencyMap = mapper.getProjectProtocolAdjacencyMap(projectId) or {} @@ -3010,6 +3133,16 @@ def extractRuntimeInputRef( postgresqlInputRefsKeys = set(postgresqlInputRefsByKey.keys()) runtimeDependenciesFromInputRefs: Set[Tuple[str, str]] = set() + runtimeParams: Set[Tuple[str, str]] = set() + for protocolId, paramsByName in runtimeParamsByProtocolId.items(): + for paramName in paramsByName.keys(): + runtimeParams.add((protocolId, paramName)) + + postgresqlParams: Set[Tuple[str, str]] = set() + for protocolId, paramsByName in postgresqlParamsByProtocolId.items(): + for paramName in paramsByName.keys(): + postgresqlParams.add((protocolId, paramName)) + for inputRef in runtimeInputRefsByKey.values(): dependencyKey = dependencyKeyFromInputRef(inputRef) if dependencyKey is not None: @@ -3111,6 +3244,33 @@ def extractRuntimeInputRef( key=inputRefSortKey, ) + missingParams = sorted( + runtimeParams - postgresqlParams, + key=paramSortKey, + ) + extraParams = sorted( + postgresqlParams - runtimeParams, + key=paramSortKey, + ) + + paramValueMismatches = [] + for key in sorted(runtimeParams.intersection(postgresqlParams), key=paramSortKey): + protocolId, paramName = key + + runtimeParam = runtimeParamsByProtocolId.get(protocolId, {}).get(paramName, {}) + postgresqlParam = postgresqlParamsByProtocolId.get(protocolId, {}).get(paramName, {}) + + runtimeValue = normalizeParamValue(runtimeParam.get("value")) + postgresqlValue = normalizeParamValue(postgresqlParam.get("value")) + + if runtimeValue != postgresqlValue: + paramValueMismatches.append({ + "protocolId": protocolId, + "paramName": paramName, + "runtimeValue": runtimeValue, + "postgresqlValue": postgresqlValue, + }) + inputRefMismatches = [] for key in sorted(runtimeInputRefs.intersection(postgresqlInputRefsKeys), key=inputRefSortKey): runtimeRef = runtimeInputRefsByKey.get(key, {}) @@ -3265,6 +3425,15 @@ def extractRuntimeInputRef( buildDependency(parentId, childId) for parentId, childId in postgresqlDependenciesWithoutInputRefs ], + "missingParams": [ + buildParamIssue(key, runtimeParamsByProtocolId.get(key[0], {}).get(key[1], {})) + for key in missingParams + ], + "extraParams": [ + buildParamIssue(key, postgresqlParamsByProtocolId.get(key[0], {}).get(key[1], {})) + for key in extraParams + ], + "paramValueMismatches": paramValueMismatches, } issuesCount = sum(len(items) for items in issues.values()) @@ -3286,6 +3455,8 @@ def extractRuntimeInputRef( "postgresqlInputRefs": len(postgresqlInputRefsKeys), "runtimeInputRefDependencies": len(runtimeDependenciesFromInputRefs), "postgresqlInputRefDependencies": len(postgresqlDependenciesFromInputRefs), + "runtimeParams": len(runtimeParams), + "postgresqlParams": len(postgresqlParams), "issues": issuesCount, }, "issues": issues, diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 2d0a3755..ead971d5 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -29,12 +29,18 @@ import pytest +class FakeParam: + def __init__(self, value): + self.value = value + + class FakeProtocol: - def __init__(self, status, outputs=None, steps=None, inputs=None): + def __init__(self, status, outputs=None, steps=None, inputs=None, params=None): self.status = status self.outputs = outputs or [] self.steps = steps or [] self.inputs = inputs or [] + self.params = params or {} def getStatus(self): return self.status @@ -48,6 +54,14 @@ def loadSteps(self): def iterInputAttributes(self): return list(self.inputs) + def iterParams(self): + return [ + (paramName, FakeParam(value)) + for paramName, value in self.params.items() + ] + + def getAttributeValue(self, paramName): + return self.params.get(paramName) class FakePointerTarget: def __init__(self, objId, className): @@ -297,6 +311,8 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "postgresqlInputRefs": 1, "runtimeInputRefDependencies": 1, "postgresqlInputRefDependencies": 1, + "runtimeParams": 0, + "postgresqlParams": 0, "issues": 0, } assert result["issues"] == { @@ -317,6 +333,9 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "runtimeDependenciesWithoutInputRefs": [], "postgresqlInputRefDependenciesMissing": [], "postgresqlDependenciesWithoutInputRefs": [], + "missingParams": [], + "extraParams": [], + "paramValueMismatches": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -374,6 +393,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "postgresqlInputRefs": 0, "runtimeInputRefDependencies": 0, "postgresqlInputRefDependencies": 0, + "missingParams": [], + "extraParams": [], + "paramValueMismatches": [], "issues": 7, } assert result["issues"] == { @@ -430,6 +452,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "childId": "10", } ], + "missingParams": [], + "extraParams": [], + "paramValueMismatches": [], } @@ -519,6 +544,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( "postgresqlInputRefs": 0, "runtimeInputRefDependencies": 0, "postgresqlInputRefDependencies": 0, + "missingParams": [], + "extraParams": [], + "paramValueMismatches": [], "issues": 2, } assert result["issues"]["missingOutputs"] == [ @@ -629,6 +657,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( "postgresqlInputRefs": 0, "runtimeInputRefDependencies": 0, "postgresqlInputRefDependencies": 0, + "missingParams": [], + "extraParams": [], + "paramValueMismatches": [], "issues": 2, } assert result["issues"]["missingSteps"] == [] @@ -850,6 +881,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( "postgresqlInputRefs": 2, "runtimeInputRefDependencies": 1, "postgresqlInputRefDependencies": 1, + "runtimeParams": 0, + "postgresqlParams": 0, "issues": 3, } @@ -999,6 +1032,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatche "postgresqlInputRefs": 2, "runtimeInputRefDependencies": 2, "postgresqlInputRefDependencies": 2, + "runtimeParams": 0, + "postgresqlParams": 0, "issues": 2, } @@ -1029,4 +1064,80 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatche assert result["issues"]["stepMismatches"] == [] assert result["issues"]["missingInputRefs"] == [] assert result["issues"]["extraInputRefs"] == [] - assert result["issues"]["inputRefMismatches"] == [] \ No newline at end of file + assert result["issues"]["inputRefMismatches"] == [] + +def test_ValidateProjectPostgresqlConsistencyReportsParamMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.params = { + "boxSize": 128, + "threshold": 0.5, + } + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + "params": { + "boxSize": 256, + "extraParam": "abc", + }, + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeParams"] == 2 + assert result["summary"]["postgresqlParams"] == 2 + assert result["issues"]["missingParams"] == [ + { + "protocolId": "10", + "paramName": "threshold", + "value": 0.5, + } + ] + assert result["issues"]["extraParams"] == [ + { + "protocolId": "10", + "paramName": "extraParam", + "value": "abc", + } + ] + assert result["issues"]["paramValueMismatches"] == [ + { + "protocolId": "10", + "paramName": "boxSize", + "runtimeValue": 128, + "postgresqlValue": 256, + } + ] \ No newline at end of file From 2ac1310ee24ffe767a62dda2e8ecdbec1b26cd52 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 16:37:31 +0200 Subject: [PATCH 175/278] Fixing tests --- .../services/test_project_service_consistency.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index ead971d5..7fa98c15 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -393,9 +393,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "postgresqlInputRefs": 0, "runtimeInputRefDependencies": 0, "postgresqlInputRefDependencies": 0, - "missingParams": [], - "extraParams": [], - "paramValueMismatches": [], + "runtimeParams": 0, + "postgresqlParams": 0, "issues": 7, } assert result["issues"] == { @@ -544,9 +543,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( "postgresqlInputRefs": 0, "runtimeInputRefDependencies": 0, "postgresqlInputRefDependencies": 0, - "missingParams": [], - "extraParams": [], - "paramValueMismatches": [], + "runtimeParams": 0, + "postgresqlParams": 0, "issues": 2, } assert result["issues"]["missingOutputs"] == [ @@ -657,9 +655,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( "postgresqlInputRefs": 0, "runtimeInputRefDependencies": 0, "postgresqlInputRefDependencies": 0, - "missingParams": [], - "extraParams": [], - "paramValueMismatches": [], + "runtimeParams": 0, + "postgresqlParams": 0, "issues": 2, } assert result["issues"]["missingSteps"] == [] @@ -1066,6 +1063,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatche assert result["issues"]["extraInputRefs"] == [] assert result["issues"]["inputRefMismatches"] == [] + def test_ValidateProjectPostgresqlConsistencyReportsParamMismatches( service, monkeypatch, From 867db318bd29311fbfd5712130dadd54f6bdf6c1 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 16:40:55 +0200 Subject: [PATCH 176/278] Check protocol params in PostgreSQL consistency report --- .../backend/api/services/test_project_service_consistency.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 7fa98c15..330a98c6 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -693,6 +693,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] + assert result["issues"]["missingParams"] == [] + assert result["issues"]["extraParams"] == [] + assert result["issues"]["paramValueMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols( From c558f5c6404c07c3cca343aca2ac1a6b120a07cf Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 16:52:51 +0200 Subject: [PATCH 177/278] Check output classes in PostgreSQL consistency report --- app/backend/api/services/project_service.py | 23 ++++++ .../test_project_service_consistency.py | 79 +++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 0821f157..def6f6c9 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -3198,6 +3198,27 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: key=dependencySortKey, ) + outputClassMismatches = [] + for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=dependencySortKey): + runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + + runtimeClassName = normalizeOptionalText(runtimeOutput.get("className")) + postgresqlClassName = normalizeOptionalText(postgresqlOutput.get("className")) + + if ( + runtimeClassName is not None + and postgresqlClassName is not None + and runtimeClassName != postgresqlClassName + ): + outputClassMismatches.append({ + "protocolId": protocolId, + "outputName": outputName, + "runtimeClassName": runtimeClassName, + "postgresqlClassName": postgresqlClassName, + "mapperKind": postgresqlOutput.get("mapperKind"), + }) + missingSteps = sorted( runtimeSteps - postgresqlSteps, key=stepSortKey, @@ -3383,6 +3404,8 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: } for protocolId, outputName in extraOutputs ], + "outputClassMismatches": outputClassMismatches, + "missingSteps": [ buildStep( protocolId, diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 330a98c6..b3812dad 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -336,6 +336,7 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "missingParams": [], "extraParams": [], "paramValueMismatches": [], + "outputClassMismatches": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -454,6 +455,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "missingParams": [], "extraParams": [], "paramValueMismatches": [], + "outputClassMismatches": [], } @@ -577,6 +579,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] + assert result["issues"]["outputClassMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( @@ -696,6 +699,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( assert result["issues"]["missingParams"] == [] assert result["issues"]["extraParams"] == [] assert result["issues"]["paramValueMismatches"] == [] + assert result["issues"]["outputClassMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols( @@ -795,6 +799,7 @@ def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] + assert result["issues"]["outputClassMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( @@ -937,6 +942,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( assert result["issues"]["runtimeDependenciesWithoutInputRefs"] == [] assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] + assert result["issues"]["outputClassMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatches( @@ -1065,6 +1071,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatche assert result["issues"]["missingInputRefs"] == [] assert result["issues"]["extraInputRefs"] == [] assert result["issues"]["inputRefMismatches"] == [] + assert result["issues"]["outputClassMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsParamMismatches( @@ -1141,4 +1148,76 @@ def test_ValidateProjectPostgresqlConsistencyReportsParamMismatches( "runtimeValue": 128, "postgresqlValue": 256, } + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsOutputClassMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + { + "protocolId": "10", + "id": 100, + "objectId": 200, + "outputName": "outputParticles", + "setClassName": "Volume", + "itemClassName": "Particle", + "properties": {"itemsCount": 12}, + "createdAt": None, + "updatedAt": None, + } + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeOutputs"] == 1 + assert result["summary"]["postgresqlOutputs"] == 1 + assert result["summary"]["issues"] == 1 + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["outputClassMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "runtimeClassName": "SetOfParticles", + "postgresqlClassName": "Volume", + "mapperKind": "flat_set", + } ] \ No newline at end of file From d4180b7d5349504ceb19498a9409cd12f11442f2 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 16:59:50 +0200 Subject: [PATCH 178/278] Check output mapper kinds in PostgreSQL consistency report --- app/backend/api/services/project_service.py | 33 +++++++ .../test_project_service_consistency.py | 89 +++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index def6f6c9..ce8b605b 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2678,6 +2678,16 @@ def buildInputRef( "objectClassName": normalizeOptionalText(payload.get("objectClassName")), } + def expectedOutputMapperKind(className: Any) -> Optional[str]: + classNameText = normalizeOptionalText(className) + if classNameText is None: + return None + + if classNameText.startswith("SetOf"): + return "flat_set" + + return "tree" + def iterPointerItems(attr: Any) -> List[Tuple[int, Any]]: try: if isinstance(attr, PointerList): @@ -3219,6 +3229,28 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: "mapperKind": postgresqlOutput.get("mapperKind"), }) + outputMapperKindMismatches = [] + for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=dependencySortKey): + runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + + runtimeClassName = normalizeOptionalText(runtimeOutput.get("className")) + expectedMapperKind = expectedOutputMapperKind(runtimeClassName) + postgresqlMapperKind = normalizeOptionalText(postgresqlOutput.get("mapperKind")) + + if ( + expectedMapperKind is not None + and postgresqlMapperKind is not None + and expectedMapperKind != postgresqlMapperKind + ): + outputMapperKindMismatches.append({ + "protocolId": protocolId, + "outputName": outputName, + "className": runtimeClassName, + "expectedMapperKind": expectedMapperKind, + "postgresqlMapperKind": postgresqlMapperKind, + }) + missingSteps = sorted( runtimeSteps - postgresqlSteps, key=stepSortKey, @@ -3405,6 +3437,7 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: for protocolId, outputName in extraOutputs ], "outputClassMismatches": outputClassMismatches, + "outputMapperKindMismatches": outputMapperKindMismatches, "missingSteps": [ buildStep( diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index b3812dad..b9bd5825 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -337,6 +337,7 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "extraParams": [], "paramValueMismatches": [], "outputClassMismatches": [], + "outputMapperKindMismatches": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -456,6 +457,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "extraParams": [], "paramValueMismatches": [], "outputClassMismatches": [], + "outputMapperKindMismatches": [], } @@ -580,6 +582,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( @@ -700,6 +703,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( assert result["issues"]["extraParams"] == [] assert result["issues"]["paramValueMismatches"] == [] assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols( @@ -800,6 +804,7 @@ def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( @@ -943,6 +948,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( assert result["issues"]["postgresqlInputRefDependenciesMissing"] == [] assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatches( @@ -1072,6 +1078,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatche assert result["issues"]["extraInputRefs"] == [] assert result["issues"]["inputRefMismatches"] == [] assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsParamMismatches( @@ -1149,6 +1156,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsParamMismatches( "postgresqlValue": 256, } ] + assert result["issues"]["outputMapperKindMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsOutputClassMismatches( @@ -1220,4 +1228,85 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputClassMismatches( "postgresqlClassName": "Volume", "mapperKind": "flat_set", } + ] + assert result["issues"]["outputMapperKindMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsOutputMapperKindMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + treeRows=[ + { + "protocolId": "10", + "id": 300, + "scipionObjId": 400, + "name": "outputParticles", + "path": "outputParticles", + "className": "SetOfParticles", + "value": None, + "label": None, + "comment": None, + "metadata": {}, + "createdAt": None, + "updatedAt": None, + }, + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeOutputs"] == 1 + assert result["summary"]["postgresqlOutputs"] == 1 + assert result["summary"]["issues"] == 1 + + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "className": "SetOfParticles", + "expectedMapperKind": "flat_set", + "postgresqlMapperKind": "tree", + } ] \ No newline at end of file From 195aa6cf2dd9893e993db046be594a67ebae92ce Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 17:11:12 +0200 Subject: [PATCH 179/278] Check output item counts in PostgreSQL consistency report --- app/backend/api/services/project_service.py | 55 ++++++++++- .../test_project_service_consistency.py | 97 ++++++++++++++++++- 2 files changed, 150 insertions(+), 2 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index ce8b605b..20478827 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2688,6 +2688,29 @@ def expectedOutputMapperKind(className: Any) -> Optional[str]: return "tree" + def getRuntimeOutputItemsCount(outputObj: Any) -> Optional[int]: + if outputObj is None: + return None + + for methodName in ("getSize", "getDim", "__len__"): + try: + if methodName == "__len__": + value = len(outputObj) + else: + method = getattr(outputObj, methodName, None) + if method is None: + continue + value = method() + + if value is None or value == "": + continue + + return int(value) + except Exception: + continue + + return None + def iterPointerItems(attr: Any) -> List[Tuple[int, Any]]: try: if isinstance(attr, PointerList): @@ -2916,9 +2939,13 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: if not outputName or outputObj is None: continue + outputClassName = self._getScipionClassName(outputObj) runtimeOutputsByProtocolId[protocolId][outputName] = { "outputName": outputName, - "className": self._getScipionClassName(outputObj), + "className": outputClassName, + "itemsCount": getRuntimeOutputItemsCount(outputObj) + if expectedOutputMapperKind(outputClassName) == "flat_set" + else None, } except Exception: logger.debug( @@ -3251,6 +3278,31 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: "postgresqlMapperKind": postgresqlMapperKind, }) + outputItemsCountMismatches = [] + for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=dependencySortKey): + runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + + runtimeClassName = normalizeOptionalText(runtimeOutput.get("className")) + if expectedOutputMapperKind(runtimeClassName) != "flat_set": + continue + + runtimeItemsCount = toOptionalInt(runtimeOutput.get("itemsCount")) + postgresqlItemsCount = toOptionalInt(postgresqlOutput.get("itemsCount")) + + if runtimeItemsCount is None or postgresqlItemsCount is None: + continue + + if runtimeItemsCount != postgresqlItemsCount: + outputItemsCountMismatches.append({ + "protocolId": protocolId, + "outputName": outputName, + "className": runtimeClassName, + "runtimeItemsCount": runtimeItemsCount, + "postgresqlItemsCount": postgresqlItemsCount, + "mapperKind": postgresqlOutput.get("mapperKind"), + }) + missingSteps = sorted( runtimeSteps - postgresqlSteps, key=stepSortKey, @@ -3438,6 +3490,7 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: ], "outputClassMismatches": outputClassMismatches, "outputMapperKindMismatches": outputMapperKindMismatches, + "outputItemsCountMismatches": outputItemsCountMismatches, "missingSteps": [ buildStep( diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index b9bd5825..18a6702b 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -92,12 +92,16 @@ def get(self): class FakeOutput: - def __init__(self, className): + def __init__(self, className, itemsCount=None): self.className = className + self.itemsCount = itemsCount def getClassName(self): return self.className + def getSize(self): + return self.itemsCount + class FakeValueHolder: def __init__(self, value): @@ -338,6 +342,7 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "paramValueMismatches": [], "outputClassMismatches": [], "outputMapperKindMismatches": [], + "outputItemsCountMismatches": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -458,6 +463,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "paramValueMismatches": [], "outputClassMismatches": [], "outputMapperKindMismatches": [], + "outputItemsCountMismatches": [], } @@ -583,6 +589,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] assert result["issues"]["outputClassMismatches"] == [] assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( @@ -704,6 +711,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( assert result["issues"]["paramValueMismatches"] == [] assert result["issues"]["outputClassMismatches"] == [] assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols( @@ -805,6 +813,7 @@ def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] assert result["issues"]["outputClassMismatches"] == [] assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( @@ -949,6 +958,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( assert result["issues"]["postgresqlDependenciesWithoutInputRefs"] == [] assert result["issues"]["outputClassMismatches"] == [] assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatches( @@ -1079,6 +1089,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatche assert result["issues"]["inputRefMismatches"] == [] assert result["issues"]["outputClassMismatches"] == [] assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsParamMismatches( @@ -1157,6 +1168,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsParamMismatches( } ] assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsOutputClassMismatches( @@ -1230,6 +1242,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputClassMismatches( } ] assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsOutputMapperKindMismatches( @@ -1309,4 +1322,86 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMapperKindMismatches( "expectedMapperKind": "flat_set", "postgresqlMapperKind": "tree", } + ] + assert result["issues"]["outputItemsCountMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsOutputItemsCountMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=12)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + { + "protocolId": "10", + "id": 100, + "objectId": 200, + "outputName": "outputParticles", + "setClassName": "SetOfParticles", + "itemClassName": "Particle", + "properties": { + "itemsCount": 10, + }, + "createdAt": None, + "updatedAt": None, + } + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeOutputs"] == 1 + assert result["summary"]["postgresqlOutputs"] == 1 + assert result["summary"]["issues"] == 1 + + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "className": "SetOfParticles", + "runtimeItemsCount": 12, + "postgresqlItemsCount": 10, + "mapperKind": "flat_set", + } ] \ No newline at end of file From 72540483beef6a81711fb2298babb2ca1635000d Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 17:23:09 +0200 Subject: [PATCH 180/278] Check protocol classes in PostgreSQL consistency report --- app/backend/api/services/project_service.py | 26 +++++ .../test_project_service_consistency.py | 95 ++++++++++++++++--- 2 files changed, 109 insertions(+), 12 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 20478827..7dadc695 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2581,6 +2581,9 @@ def validateProjectPostgresqlConsistency( def normalizeStatus(value: Any) -> str: return str(value or "").strip().lower() + def normalizeClassName(value: Any) -> str: + return str(value or "").strip() + def normalizeProtocolId(value: Any) -> str: return str(value).strip() @@ -2892,6 +2895,7 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: return paramsByName runtimeStatuses: Dict[str, str] = {} + runtimeClassNames: Dict[str, str] = {} runtimeDependencies: Set[Tuple[str, str]] = set() runtimeOutputsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} runtimeStepsByProtocolId: Dict[str, Dict[int, Dict[str, Any]]] = {} @@ -2920,6 +2924,9 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: runtimeStatuses[protocolId] = normalizeStatus( self._safeCall(protocol, "getStatus", None) ) + runtimeClassNames[protocolId] = normalizeClassName( + self._safeCall(protocol, "getClassName", None) + ) runtimeOutputsByProtocolId.setdefault(protocolId, {}) if protocol is not None: @@ -3033,6 +3040,7 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: ) postgresqlStatuses: Dict[str, str] = {} + postgresqlClassNames: Dict[str, str] = {} postgresqlParamsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} for row in protocolRows: @@ -3041,6 +3049,7 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: continue postgresqlStatuses[protocolId] = normalizeStatus(row.get("status")) + postgresqlClassNames[protocolId] = normalizeClassName(row.get("protocolClassName")) postgresqlParamsByProtocolId[protocolId] = extractPostgresqlParams(row) try: @@ -3217,6 +3226,22 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: "postgresqlStatus": postgresqlStatus, }) + protocolClassMismatches = [] + for protocolId in commonProtocolIds: + runtimeClassName = normalizeClassName(runtimeClassNames.get(protocolId)) + postgresqlClassName = normalizeClassName(postgresqlClassNames.get(protocolId)) + + if ( + runtimeClassName + and postgresqlClassName + and runtimeClassName != postgresqlClassName + ): + protocolClassMismatches.append({ + "protocolId": protocolId, + "runtimeClassName": runtimeClassName, + "postgresqlClassName": postgresqlClassName, + }) + missingDependencies = sorted( runtimeDependencies - postgresqlDependencies, key=dependencySortKey, @@ -3454,6 +3479,7 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: for protocolId in extraProtocolIds ], "statusMismatches": statusMismatches, + "protocolClassMismatches": protocolClassMismatches, "missingDependencies": [ buildDependency(parentId, childId) for parentId, childId in missingDependencies diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 18a6702b..3c34a475 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -35,12 +35,16 @@ def __init__(self, value): class FakeProtocol: - def __init__(self, status, outputs=None, steps=None, inputs=None, params=None): + def __init__(self, status, outputs=None, steps=None, inputs=None, params=None, className="FakeProtocol"): self.status = status self.outputs = outputs or [] self.steps = steps or [] self.inputs = inputs or [] self.params = params or {} + self.className = className + + def getClassName(self): + return self.className def getStatus(self): return self.status @@ -273,8 +277,8 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( }, protocolRows=[ - {"protocolId": "10", "status": "finished"}, - {"protocolId": "11", "status": "running"}, + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + {"protocolId": "11", "status": "running", "protocolClassName": "FakeProtocol"}, ], adjacencyMap={ "10": {"parents": [], "children": ["11"]}, @@ -343,6 +347,7 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "outputClassMismatches": [], "outputMapperKindMismatches": [], "outputItemsCountMismatches": [], + "protocolClassMismatches": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -464,6 +469,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "outputClassMismatches": [], "outputMapperKindMismatches": [], "outputItemsCountMismatches": [], + "protocolClassMismatches": [], } @@ -495,7 +501,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( "name": str(tmp_path), }, protocolRows=[ - {"protocolId": "10", "status": "finished"}, + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, ], adjacencyMap={ "10": {"parents": [], "children": []}, @@ -590,6 +596,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( assert result["issues"]["outputClassMismatches"] == [] assert result["issues"]["outputMapperKindMismatches"] == [] assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( @@ -712,6 +719,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsStepMismatches( assert result["issues"]["outputClassMismatches"] == [] assert result["issues"]["outputMapperKindMismatches"] == [] assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols( @@ -752,8 +760,8 @@ def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols "name": str(tmp_path), }, protocolRows=[ - {"protocolId": "10", "status": "finished"}, - {"protocolId": "11", "status": "running"}, + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + {"protocolId": "11", "status": "running", "protocolClassName": "FakeProtocol"}, ], adjacencyMap={ "10": {"parents": [], "children": ["11"]}, @@ -814,6 +822,7 @@ def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols assert result["issues"]["outputClassMismatches"] == [] assert result["issues"]["outputMapperKindMismatches"] == [] assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( @@ -849,8 +858,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( "name": str(tmp_path), }, protocolRows=[ - {"protocolId": "10", "status": "finished"}, - {"protocolId": "11", "status": "running"}, + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + {"protocolId": "11", "status": "running", "protocolClassName": "FakeProtocol"}, ], adjacencyMap={ "10": {"parents": [], "children": ["11"]}, @@ -959,6 +968,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( assert result["issues"]["outputClassMismatches"] == [] assert result["issues"]["outputMapperKindMismatches"] == [] assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatches( @@ -1001,9 +1011,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatche "name": str(tmp_path), }, protocolRows=[ - {"protocolId": "10", "status": "finished"}, - {"protocolId": "11", "status": "running"}, - {"protocolId": "12", "status": "running"}, + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + {"protocolId": "11", "status": "running", "protocolClassName": "FakeProtocol"}, + {"protocolId": "12", "status": "running", "protocolClassName": "FakeProtocol"}, ], adjacencyMap={ "10": {"parents": [], "children": ["11"]}, @@ -1090,6 +1100,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatche assert result["issues"]["outputClassMismatches"] == [] assert result["issues"]["outputMapperKindMismatches"] == [] assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsParamMismatches( @@ -1169,6 +1180,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsParamMismatches( ] assert result["issues"]["outputMapperKindMismatches"] == [] assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsOutputClassMismatches( @@ -1198,7 +1210,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputClassMismatches( "name": str(tmp_path), }, protocolRows=[ - {"protocolId": "10", "status": "finished"}, + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, ], adjacencyMap={ "10": {"parents": [], "children": []}, @@ -1243,6 +1255,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputClassMismatches( ] assert result["issues"]["outputMapperKindMismatches"] == [] assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsOutputMapperKindMismatches( @@ -1324,6 +1337,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMapperKindMismatches( } ] assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["protocolClassMismatches"] == [] def test_ValidateProjectPostgresqlConsistencyReportsOutputItemsCountMismatches( @@ -1404,4 +1418,61 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputItemsCountMismatches( "postgresqlItemsCount": 10, "mapperKind": "flat_set", } + ] + assert result["issues"]["protocolClassMismatches"] == [] + +def test_ValidateProjectPostgresqlConsistencyReportsProtocolClassMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.className = "RuntimeProtocolClass" + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + "protocolClassName": "PostgresqlProtocolClass", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeProtocols"] == 1 + assert result["summary"]["postgresqlProtocols"] == 1 + assert result["summary"]["issues"] == 1 + assert result["issues"]["protocolClassMismatches"] == [ + { + "protocolId": "10", + "runtimeClassName": "RuntimeProtocolClass", + "postgresqlClassName": "PostgresqlProtocolClass", + } ] \ No newline at end of file From d187b62dbbcdfb5e3c1832407cff8c15b9a705e7 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 17:40:43 +0200 Subject: [PATCH 181/278] Extract PostgreSQL consistency checker service --- .../services/project_consistency_service.py | 1083 +++++++++++++++++ app/backend/api/services/project_service.py | 1032 +--------------- 2 files changed, 1090 insertions(+), 1025 deletions(-) create mode 100644 app/backend/api/services/project_consistency_service.py diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py new file mode 100644 index 00000000..6c4269a5 --- /dev/null +++ b/app/backend/api/services/project_consistency_service.py @@ -0,0 +1,1083 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# ****************************************************************************** + +from __future__ import annotations +from datetime import datetime + +import logging +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple + +from fastapi import HTTPException, status +from pyworkflow.object import PointerList +from pyworkflow.protocol.params import PointerParam, MultiPointerParam, RelationParam + +from app.backend.mapper.postgresql import PostgresqlFlatMapper + +if TYPE_CHECKING: + from app.backend.api.services.project_service import ProjectService + +logger = logging.getLogger(__name__) + + +class ProjectConsistencyService: + def __init__(self, projectService: "ProjectService"): + self.projectService = projectService + + @property + def currentProject(self): + return self.projectService.currentProject + + @currentProject.setter + def currentProject(self, value): + self.projectService.currentProject = value + + def __getattr__(self, name: str) -> Any: + return getattr(self.projectService, name) + + + def validateProjectPostgresqlConsistency( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser: dict, + refresh: bool = True, + checkPid: bool = True, + ) -> Dict[str, Any]: + dbProj = self.getProjectDbRow(mapper, projectId, currentUser) + if not dbProj: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Project not found", + ) + + self.loadProjectForThumbnails(dbProj) + + def normalizeStatus(value: Any) -> str: + return str(value or "").strip().lower() + + def normalizeClassName(value: Any) -> str: + return str(value or "").strip() + + def normalizeProtocolId(value: Any) -> str: + return str(value).strip() + + def protocolSortKey(value: Any): + text = normalizeProtocolId(value) + try: + return 0, int(text) + except Exception: + return 1, text + + def dependencySortKey(item: Tuple[str, str]): + parentId, childId = item + return protocolSortKey(parentId), protocolSortKey(childId) + + def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: + return { + "parentId": normalizeProtocolId(parentId), + "childId": normalizeProtocolId(childId), + } + + def dependencyKeyFromInputRef(payload: Dict[str, Any]) -> Optional[Tuple[str, str]]: + parentProtocolId = normalizeOptionalText(payload.get("parentProtocolId")) + childProtocolId = normalizeOptionalText(payload.get("protocolId")) + + if not parentProtocolId or not childProtocolId: + return None + + if parentProtocolId == "PROJECT" or childProtocolId == "PROJECT": + return None + + if parentProtocolId == childProtocolId: + return None + + return parentProtocolId, childProtocolId + + def toOptionalInt(value: Any) -> Optional[int]: + try: + if value is None or value == "": + return None + return int(value) + except Exception: + return None + + def stepSortKey(item: Tuple[str, int]): + protocolId, stepIndex = item + return protocolSortKey(protocolId), int(stepIndex) + + def stepValue(step: Any, attrName: str, fallback: Any = None) -> Any: + try: + value = getattr(step, attrName, None) + if hasattr(value, "get"): + return value.get() + return value if value is not None else fallback + except Exception: + return fallback + + def getStepName(step: Any) -> str: + name = stepValue(step, "funcName", None) + if name: + return str(name) + + className = self._safeCall(step, "getClassName", None) + return str(className or "") + + def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[str, Any]: + return { + "protocolId": normalizeProtocolId(protocolId), + "index": int(stepIndex), + "name": str(payload.get("name") or ""), + "status": normalizeStatus(payload.get("status")), + } + + def normalizeOptionalText(value: Any) -> Optional[str]: + if value is None or value == "": + return None + + text = str(value).strip() + return text or None + + def inputRefSortKey(item: Tuple[str, str, int]): + protocolId, inputName, itemIndex = item + return protocolSortKey(protocolId), str(inputName), int(itemIndex) + + def buildInputRef( + key: Tuple[str, str, int], + payload: Dict[str, Any], + ) -> Dict[str, Any]: + protocolId, inputName, itemIndex = key + return { + "protocolId": normalizeProtocolId(protocolId), + "inputName": str(inputName), + "itemIndex": int(itemIndex), + "parentProtocolId": normalizeOptionalText(payload.get("parentProtocolId")), + "parentOutputName": normalizeOptionalText(payload.get("parentOutputName")), + "objectClassName": normalizeOptionalText(payload.get("objectClassName")), + } + + def expectedOutputMapperKind(className: Any) -> Optional[str]: + classNameText = normalizeOptionalText(className) + if classNameText is None: + return None + + if classNameText.startswith("SetOf"): + return "flat_set" + + return "tree" + + def getRuntimeOutputItemsCount(outputObj: Any) -> Optional[int]: + if outputObj is None: + return None + + for methodName in ("getSize", "getDim", "__len__"): + try: + if methodName == "__len__": + value = len(outputObj) + else: + method = getattr(outputObj, methodName, None) + if method is None: + continue + value = method() + + if value is None or value == "": + continue + + return int(value) + except Exception: + continue + + return None + + def iterPointerItems(attr: Any) -> List[Tuple[int, Any]]: + try: + if isinstance(attr, PointerList): + return [ + (index, pointer) + for index, pointer in enumerate(attr) + ] + except Exception: + pass + + return [(0, attr)] + + def extractRuntimeInputRef( + protocolId: str, + inputName: str, + itemIndex: int, + pointer: Any, + ) -> Optional[Dict[str, Any]]: + parentProtocolId = None + parentOutputName = None + objectClassName = None + objectId = None + + try: + parentObj = pointer.getObjValue() + parentProtocolId = normalizeOptionalText( + self._safeCall(parentObj, "getObjId", None) + ) + except Exception: + parentProtocolId = None + + try: + parentOutputName = normalizeOptionalText(pointer.getExtended()) + except Exception: + parentOutputName = None + + try: + targetObj = pointer.get() + if targetObj is not None: + objectClassName = normalizeOptionalText( + self._getScipionClassName(targetObj) + ) + objectId = normalizeOptionalText( + self._safeCall(targetObj, "getObjId", None) + ) + except Exception: + objectClassName = None + objectId = None + + if parentProtocolId is None and parentOutputName is None: + return None + + return { + "protocolId": normalizeProtocolId(protocolId), + "inputName": str(inputName), + "itemIndex": int(itemIndex), + "parentProtocolId": parentProtocolId, + "parentOutputName": parentOutputName, + "objectClassName": objectClassName, + "objectId": objectId, + } + + def normalizeParamValue(value: Any) -> Any: + if value is None: + return None + + if isinstance(value, bool): + return value + + if isinstance(value, (int, float)): + return value + + if isinstance(value, str): + text = value.strip() + if text == "": + return "" + lowerText = text.lower() + if lowerText in ("true", "false"): + return lowerText == "true" + + try: + if "." not in text: + return int(text) + except Exception: + pass + + try: + return float(text) + except Exception: + return text + + if isinstance(value, (list, tuple)): + return [normalizeParamValue(item) for item in value] + + if isinstance(value, dict): + return { + str(key): normalizeParamValue(itemValue) + for key, itemValue in value.items() + } + + try: + if hasattr(value, "get"): + return normalizeParamValue(value.get()) + except Exception: + pass + + return str(value) + + def paramSortKey(item: Tuple[str, str]): + protocolId, paramName = item + return protocolSortKey(protocolId), str(paramName) + + def buildParamIssue( + key: Tuple[str, str], + payload: Dict[str, Any], + ) -> Dict[str, Any]: + protocolId, paramName = key + return { + "protocolId": normalizeProtocolId(protocolId), + "paramName": str(paramName), + "value": payload.get("value"), + } + + def isPointerParam(param: Any) -> bool: + return isinstance(param, (PointerParam, MultiPointerParam, RelationParam)) + + def extractRuntimeParams(protocol: Any) -> Dict[str, Dict[str, Any]]: + paramsByName: Dict[str, Dict[str, Any]] = {} + + try: + self.currentProject._fixProtParamsConfiguration(protocol) + except Exception: + pass + + try: + for paramName, param in protocol.iterParams(): + paramNameText = str(paramName or "").strip() + if not paramNameText: + continue + + if isPointerParam(param): + continue + + rawValue = None + try: + rawValue = protocol.getAttributeValue(paramNameText) + except Exception: + try: + rawValue = getattr(protocol, paramNameText, None) + except Exception: + rawValue = None + + paramsByName[paramNameText] = { + "value": normalizeParamValue(rawValue), + } + except Exception: + logger.debug( + "Could not inspect runtime protocol params during consistency check.", + exc_info=True, + ) + + return paramsByName + + def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + rawParams = row.get("params") + if not isinstance(rawParams, dict): + return {} + + paramsByName: Dict[str, Dict[str, Any]] = {} + for paramName, rawValue in rawParams.items(): + paramNameText = str(paramName or "").strip() + if not paramNameText: + continue + + paramsByName[paramNameText] = { + "value": normalizeParamValue(rawValue), + } + + return paramsByName + + runtimeStatuses: Dict[str, str] = {} + runtimeClassNames: Dict[str, str] = {} + runtimeDependencies: Set[Tuple[str, str]] = set() + runtimeOutputsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} + runtimeStepsByProtocolId: Dict[str, Dict[int, Dict[str, Any]]] = {} + runtimeInputRefsByKey: Dict[Tuple[str, str, int], Dict[str, Any]] = {} + runtimeParamsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} + + try: + runs = self.currentProject.getRunsGraph(refresh=refresh, checkPids=checkPid) + nodesDict = getattr(runs, "_nodesDict", {}) or {} + except Exception as e: + logger.exception( + "Failed to load Scipion runtime graph for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load Scipion runtime graph: {e}", + ) + + for nodeId, nodeObj in nodesDict.items(): + protocolId = normalizeProtocolId(nodeId) + if not protocolId or protocolId == "PROJECT": + continue + + protocol = getattr(nodeObj, "run", None) + runtimeStatuses[protocolId] = normalizeStatus( + self._safeCall(protocol, "getStatus", None) + ) + runtimeClassNames[protocolId] = normalizeClassName( + self._safeCall(protocol, "getClassName", None) + ) + runtimeOutputsByProtocolId.setdefault(protocolId, {}) + + if protocol is not None: + try: + for outputItem in protocol.iterOutputAttributes(): + outputName = None + outputObj = None + + if isinstance(outputItem, (tuple, list)) and len(outputItem) >= 2: + outputName = outputItem[0] + outputObj = outputItem[1] + else: + outputName = self._safeCall(outputItem, "getName", None) + outputObj = outputItem + + outputName = str(outputName or "").strip() + if not outputName or outputObj is None: + continue + + outputClassName = self._getScipionClassName(outputObj) + runtimeOutputsByProtocolId[protocolId][outputName] = { + "outputName": outputName, + "className": outputClassName, + "itemsCount": getRuntimeOutputItemsCount(outputObj) + if expectedOutputMapperKind(outputClassName) == "flat_set" + else None, + } + except Exception: + logger.debug( + "Could not inspect runtime protocol outputs during consistency check. " + "projectId=%s protocolId=%s", + projectId, + protocolId, + exc_info=True, + ) + runtimeStepsByProtocolId.setdefault(protocolId, {}) + + try: + for step in protocol.loadSteps() or []: + stepIndex = toOptionalInt(self._safeCall(step, "getIndex", None)) + if stepIndex is None: + continue + + runtimeStepsByProtocolId[protocolId][stepIndex] = { + "index": stepIndex, + "name": getStepName(step), + "status": normalizeStatus(self._safeCall(step, "getStatus", None)), + } + except Exception: + logger.debug( + "Could not inspect runtime protocol steps during consistency check. " + "projectId=%s protocolId=%s", + projectId, + protocolId, + exc_info=True, + ) + try: + for inputName, attr in protocol.iterInputAttributes(): + inputNameText = str(inputName or "").strip() + if not inputNameText: + continue + + for itemIndex, pointer in iterPointerItems(attr): + inputRef = extractRuntimeInputRef( + protocolId=protocolId, + inputName=inputNameText, + itemIndex=int(itemIndex), + pointer=pointer, + ) + if inputRef is None: + continue + + key = ( + inputRef["protocolId"], + inputRef["inputName"], + inputRef["itemIndex"], + ) + runtimeInputRefsByKey[key] = inputRef + except Exception: + logger.debug( + "Could not inspect runtime protocol input refs during consistency check. " + "projectId=%s protocolId=%s", + projectId, + protocolId, + exc_info=True, + ) + + runtimeParamsByProtocolId[protocolId] = extractRuntimeParams(protocol) + + for parent in getattr(nodeObj, "_parents", []) or []: + try: + parentId = normalizeProtocolId(parent.getName()) + except Exception: + parentId = normalizeProtocolId(parent) + + if not parentId or parentId == "PROJECT": + continue + + runtimeDependencies.add((parentId, protocolId)) + + try: + protocolRows = mapper.getProtocols(projectId) or [] + except Exception as e: + logger.exception( + "Failed to load PostgreSQL protocols for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL protocols: {e}", + ) + + postgresqlStatuses: Dict[str, str] = {} + postgresqlClassNames: Dict[str, str] = {} + postgresqlParamsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} + + for row in protocolRows: + protocolId = normalizeProtocolId(row.get("protocolId")) + if not protocolId: + continue + + postgresqlStatuses[protocolId] = normalizeStatus(row.get("status")) + postgresqlClassNames[protocolId] = normalizeClassName(row.get("protocolClassName")) + postgresqlParamsByProtocolId[protocolId] = extractPostgresqlParams(row) + + try: + adjacencyMap = mapper.getProjectProtocolAdjacencyMap(projectId) or {} + except Exception as e: + logger.exception( + "Failed to load PostgreSQL protocol dependencies for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL protocol dependencies: {e}", + ) + + postgresqlDependencies: Set[Tuple[str, str]] = set() + for childId, refs in adjacencyMap.items(): + childProtocolId = normalizeProtocolId(childId) + if not childProtocolId or childProtocolId == "PROJECT": + continue + + for parentIdValue in refs.get("parents") or []: + parentProtocolId = normalizeProtocolId(parentIdValue) + if not parentProtocolId or parentProtocolId == "PROJECT": + continue + + postgresqlDependencies.add((parentProtocolId, childProtocolId)) + + try: + persistedOutputsByProtocolId = self._loadPersistedOutputsByProtocolId( + mapper, + projectId, + ) + except Exception as e: + logger.exception( + "Failed to load PostgreSQL persisted outputs for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL persisted outputs: {e}", + ) + + try: + postgresqlStepsByProtocolId = mapper.getProjectProtocolStepsByProtocolId(projectId) or {} + except Exception as e: + logger.exception( + "Failed to load PostgreSQL protocol steps for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL protocol steps: {e}", + ) + + normalizedPostgresqlStepsByProtocolId: Dict[str, Dict[int, Dict[str, Any]]] = {} + for protocolId, steps in postgresqlStepsByProtocolId.items(): + protocolIdText = normalizeProtocolId(protocolId) + for step in steps or []: + stepIndex = toOptionalInt(step.get("index")) + if stepIndex is None: + continue + + normalizedPostgresqlStepsByProtocolId.setdefault(protocolIdText, {})[stepIndex] = { + "index": stepIndex, + "name": str(step.get("name") or ""), + "status": normalizeStatus(step.get("status")), + } + + try: + postgresqlInputRefs = mapper.listProtocolInputRefs(projectId) or [] + except Exception as e: + logger.exception( + "Failed to load PostgreSQL protocol input refs for consistency check. projectId=%s", + projectId, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to load PostgreSQL protocol input refs: {e}", + ) + + postgresqlInputRefsByKey: Dict[Tuple[str, str, int], Dict[str, Any]] = {} + for ref in postgresqlInputRefs: + protocolIdText = normalizeProtocolId(ref.get("protocolId")) + inputName = str(ref.get("inputName") or "").strip() + itemIndex = toOptionalInt(ref.get("itemIndex")) + + if not protocolIdText or not inputName: + continue + + if itemIndex is None or itemIndex < 0: + itemIndex = 0 + + key = (protocolIdText, inputName, int(itemIndex)) + postgresqlInputRefsByKey[key] = { + "protocolId": protocolIdText, + "inputName": inputName, + "itemIndex": int(itemIndex), + "parentProtocolId": normalizeOptionalText(ref.get("parentProtocolId")), + "parentOutputName": normalizeOptionalText(ref.get("parentOutputName")), + "objectClassName": normalizeOptionalText(ref.get("objectClassName")), + "objectId": normalizeOptionalText(ref.get("objectId")), + } + + runtimeOutputs: Set[Tuple[str, str]] = set() + for protocolId, outputsByName in runtimeOutputsByProtocolId.items(): + for outputName in outputsByName.keys(): + runtimeOutputs.add((protocolId, outputName)) + + postgresqlOutputs: Set[Tuple[str, str]] = set() + for protocolId, outputsByName in persistedOutputsByProtocolId.items(): + for outputName in outputsByName.keys(): + postgresqlOutputs.add((normalizeProtocolId(protocolId), str(outputName))) + + runtimeSteps: Set[Tuple[str, int]] = set() + for protocolId, stepsByIndex in runtimeStepsByProtocolId.items(): + for stepIndex in stepsByIndex.keys(): + runtimeSteps.add((protocolId, int(stepIndex))) + + postgresqlSteps: Set[Tuple[str, int]] = set() + for protocolId, stepsByIndex in normalizedPostgresqlStepsByProtocolId.items(): + for stepIndex in stepsByIndex.keys(): + postgresqlSteps.add((protocolId, int(stepIndex))) + + runtimeProtocolIds = set(runtimeStatuses.keys()) + postgresqlProtocolIds = set(postgresqlStatuses.keys()) + runtimeInputRefs = set(runtimeInputRefsByKey.keys()) + postgresqlInputRefsKeys = set(postgresqlInputRefsByKey.keys()) + runtimeDependenciesFromInputRefs: Set[Tuple[str, str]] = set() + + runtimeParams: Set[Tuple[str, str]] = set() + for protocolId, paramsByName in runtimeParamsByProtocolId.items(): + for paramName in paramsByName.keys(): + runtimeParams.add((protocolId, paramName)) + + postgresqlParams: Set[Tuple[str, str]] = set() + for protocolId, paramsByName in postgresqlParamsByProtocolId.items(): + for paramName in paramsByName.keys(): + postgresqlParams.add((protocolId, paramName)) + + for inputRef in runtimeInputRefsByKey.values(): + dependencyKey = dependencyKeyFromInputRef(inputRef) + if dependencyKey is not None: + runtimeDependenciesFromInputRefs.add(dependencyKey) + + postgresqlDependenciesFromInputRefs: Set[Tuple[str, str]] = set() + for inputRef in postgresqlInputRefsByKey.values(): + dependencyKey = dependencyKeyFromInputRef(inputRef) + if dependencyKey is not None: + postgresqlDependenciesFromInputRefs.add(dependencyKey) + + missingProtocolIds = sorted( + runtimeProtocolIds - postgresqlProtocolIds, + key=protocolSortKey, + ) + extraProtocolIds = sorted( + postgresqlProtocolIds - runtimeProtocolIds, + key=protocolSortKey, + ) + + commonProtocolIds = sorted( + runtimeProtocolIds.intersection(postgresqlProtocolIds), + key=protocolSortKey, + ) + + statusMismatches = [] + for protocolId in commonProtocolIds: + runtimeStatus = runtimeStatuses.get(protocolId, "") + postgresqlStatus = postgresqlStatuses.get(protocolId, "") + + if runtimeStatus != postgresqlStatus: + statusMismatches.append({ + "protocolId": protocolId, + "runtimeStatus": runtimeStatus, + "postgresqlStatus": postgresqlStatus, + }) + + protocolClassMismatches = [] + for protocolId in commonProtocolIds: + runtimeClassName = normalizeClassName(runtimeClassNames.get(protocolId)) + postgresqlClassName = normalizeClassName(postgresqlClassNames.get(protocolId)) + + if ( + runtimeClassName + and postgresqlClassName + and runtimeClassName != postgresqlClassName + ): + protocolClassMismatches.append({ + "protocolId": protocolId, + "runtimeClassName": runtimeClassName, + "postgresqlClassName": postgresqlClassName, + }) + + missingDependencies = sorted( + runtimeDependencies - postgresqlDependencies, + key=dependencySortKey, + ) + extraDependencies = sorted( + postgresqlDependencies - runtimeDependencies, + key=dependencySortKey, + ) + + missingOutputs = sorted( + runtimeOutputs - postgresqlOutputs, + key=dependencySortKey, + ) + extraOutputs = sorted( + postgresqlOutputs - runtimeOutputs, + key=dependencySortKey, + ) + + outputClassMismatches = [] + for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=dependencySortKey): + runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + + runtimeClassName = normalizeOptionalText(runtimeOutput.get("className")) + postgresqlClassName = normalizeOptionalText(postgresqlOutput.get("className")) + + if ( + runtimeClassName is not None + and postgresqlClassName is not None + and runtimeClassName != postgresqlClassName + ): + outputClassMismatches.append({ + "protocolId": protocolId, + "outputName": outputName, + "runtimeClassName": runtimeClassName, + "postgresqlClassName": postgresqlClassName, + "mapperKind": postgresqlOutput.get("mapperKind"), + }) + + outputMapperKindMismatches = [] + for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=dependencySortKey): + runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + + runtimeClassName = normalizeOptionalText(runtimeOutput.get("className")) + expectedMapperKind = expectedOutputMapperKind(runtimeClassName) + postgresqlMapperKind = normalizeOptionalText(postgresqlOutput.get("mapperKind")) + + if ( + expectedMapperKind is not None + and postgresqlMapperKind is not None + and expectedMapperKind != postgresqlMapperKind + ): + outputMapperKindMismatches.append({ + "protocolId": protocolId, + "outputName": outputName, + "className": runtimeClassName, + "expectedMapperKind": expectedMapperKind, + "postgresqlMapperKind": postgresqlMapperKind, + }) + + outputItemsCountMismatches = [] + for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=dependencySortKey): + runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + + runtimeClassName = normalizeOptionalText(runtimeOutput.get("className")) + if expectedOutputMapperKind(runtimeClassName) != "flat_set": + continue + + runtimeItemsCount = toOptionalInt(runtimeOutput.get("itemsCount")) + postgresqlItemsCount = toOptionalInt(postgresqlOutput.get("itemsCount")) + + if runtimeItemsCount is None or postgresqlItemsCount is None: + continue + + if runtimeItemsCount != postgresqlItemsCount: + outputItemsCountMismatches.append({ + "protocolId": protocolId, + "outputName": outputName, + "className": runtimeClassName, + "runtimeItemsCount": runtimeItemsCount, + "postgresqlItemsCount": postgresqlItemsCount, + "mapperKind": postgresqlOutput.get("mapperKind"), + }) + + missingSteps = sorted( + runtimeSteps - postgresqlSteps, + key=stepSortKey, + ) + extraSteps = sorted( + postgresqlSteps - runtimeSteps, + key=stepSortKey, + ) + + stepMismatches = [] + for protocolId, stepIndex in sorted(runtimeSteps.intersection(postgresqlSteps), key=stepSortKey): + runtimeStep = runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) + postgresqlStep = normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) + + runtimeName = str(runtimeStep.get("name") or "") + postgresqlName = str(postgresqlStep.get("name") or "") + runtimeStatus = normalizeStatus(runtimeStep.get("status")) + postgresqlStatus = normalizeStatus(postgresqlStep.get("status")) + + changedFields = [] + if runtimeName != postgresqlName: + changedFields.append("name") + if runtimeStatus != postgresqlStatus: + changedFields.append("status") + + if changedFields: + stepMismatches.append({ + "protocolId": protocolId, + "index": int(stepIndex), + "fields": changedFields, + "runtimeName": runtimeName, + "postgresqlName": postgresqlName, + "runtimeStatus": runtimeStatus, + "postgresqlStatus": postgresqlStatus, + }) + + + missingInputRefs = sorted( + runtimeInputRefs - postgresqlInputRefsKeys, + key=inputRefSortKey, + ) + extraInputRefs = sorted( + postgresqlInputRefsKeys - runtimeInputRefs, + key=inputRefSortKey, + ) + + missingParams = sorted( + runtimeParams - postgresqlParams, + key=paramSortKey, + ) + extraParams = sorted( + postgresqlParams - runtimeParams, + key=paramSortKey, + ) + + paramValueMismatches = [] + for key in sorted(runtimeParams.intersection(postgresqlParams), key=paramSortKey): + protocolId, paramName = key + + runtimeParam = runtimeParamsByProtocolId.get(protocolId, {}).get(paramName, {}) + postgresqlParam = postgresqlParamsByProtocolId.get(protocolId, {}).get(paramName, {}) + + runtimeValue = normalizeParamValue(runtimeParam.get("value")) + postgresqlValue = normalizeParamValue(postgresqlParam.get("value")) + + if runtimeValue != postgresqlValue: + paramValueMismatches.append({ + "protocolId": protocolId, + "paramName": paramName, + "runtimeValue": runtimeValue, + "postgresqlValue": postgresqlValue, + }) + + inputRefMismatches = [] + for key in sorted(runtimeInputRefs.intersection(postgresqlInputRefsKeys), key=inputRefSortKey): + runtimeRef = runtimeInputRefsByKey.get(key, {}) + postgresqlRef = postgresqlInputRefsByKey.get(key, {}) + + changedFields = [] + + runtimeParentProtocolId = normalizeOptionalText(runtimeRef.get("parentProtocolId")) + postgresqlParentProtocolId = normalizeOptionalText(postgresqlRef.get("parentProtocolId")) + + runtimeParentOutputName = normalizeOptionalText(runtimeRef.get("parentOutputName")) + postgresqlParentOutputName = normalizeOptionalText(postgresqlRef.get("parentOutputName")) + + runtimeObjectClassName = normalizeOptionalText(runtimeRef.get("objectClassName")) + postgresqlObjectClassName = normalizeOptionalText(postgresqlRef.get("objectClassName")) + + if runtimeParentProtocolId != postgresqlParentProtocolId: + changedFields.append("parentProtocolId") + + if runtimeParentOutputName != postgresqlParentOutputName: + changedFields.append("parentOutputName") + + if ( + runtimeObjectClassName is not None + and postgresqlObjectClassName is not None + and runtimeObjectClassName != postgresqlObjectClassName + ): + changedFields.append("objectClassName") + + if changedFields: + protocolId, inputName, itemIndex = key + inputRefMismatches.append({ + "protocolId": protocolId, + "inputName": inputName, + "itemIndex": int(itemIndex), + "fields": changedFields, + "runtimeParentProtocolId": runtimeParentProtocolId, + "postgresqlParentProtocolId": postgresqlParentProtocolId, + "runtimeParentOutputName": runtimeParentOutputName, + "postgresqlParentOutputName": postgresqlParentOutputName, + "runtimeObjectClassName": runtimeObjectClassName, + "postgresqlObjectClassName": postgresqlObjectClassName, + }) + + runtimeInputRefDependenciesMissing = sorted( + runtimeDependenciesFromInputRefs - runtimeDependencies, + key=dependencySortKey, + ) + runtimeDependenciesWithoutInputRefs = sorted( + runtimeDependencies - runtimeDependenciesFromInputRefs, + key=dependencySortKey, + ) + + postgresqlInputRefDependenciesMissing = sorted( + postgresqlDependenciesFromInputRefs - postgresqlDependencies, + key=dependencySortKey, + ) + postgresqlDependenciesWithoutInputRefs = sorted( + postgresqlDependencies - postgresqlDependenciesFromInputRefs, + key=dependencySortKey, + ) + + issues = { + "missingProtocols": [ + { + "protocolId": protocolId, + "runtimeStatus": runtimeStatuses.get(protocolId, ""), + } + for protocolId in missingProtocolIds + ], + "extraProtocols": [ + { + "protocolId": protocolId, + "postgresqlStatus": postgresqlStatuses.get(protocolId, ""), + } + for protocolId in extraProtocolIds + ], + "statusMismatches": statusMismatches, + "protocolClassMismatches": protocolClassMismatches, + "missingDependencies": [ + buildDependency(parentId, childId) + for parentId, childId in missingDependencies + ], + "extraDependencies": [ + buildDependency(parentId, childId) + for parentId, childId in extraDependencies + ], + "missingOutputs": [ + { + "protocolId": protocolId, + "outputName": outputName, + "className": runtimeOutputsByProtocolId + .get(protocolId, {}) + .get(outputName, {}) + .get("className"), + } + for protocolId, outputName in missingOutputs + ], + "extraOutputs": [ + { + "protocolId": protocolId, + "outputName": outputName, + "mapperKind": persistedOutputsByProtocolId + .get(protocolId, {}) + .get(outputName, {}) + .get("mapperKind"), + "className": persistedOutputsByProtocolId + .get(protocolId, {}) + .get(outputName, {}) + .get("className"), + } + for protocolId, outputName in extraOutputs + ], + "outputClassMismatches": outputClassMismatches, + "outputMapperKindMismatches": outputMapperKindMismatches, + "outputItemsCountMismatches": outputItemsCountMismatches, + + "missingSteps": [ + buildStep( + protocolId, + stepIndex, + runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}), + ) + for protocolId, stepIndex in missingSteps + ], + "extraSteps": [ + buildStep( + protocolId, + stepIndex, + normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}), + ) + for protocolId, stepIndex in extraSteps + ], + "stepMismatches": stepMismatches, + "missingInputRefs": [ + buildInputRef(key, runtimeInputRefsByKey.get(key, {})) + for key in missingInputRefs + ], + "extraInputRefs": [ + buildInputRef(key, postgresqlInputRefsByKey.get(key, {})) + for key in extraInputRefs + ], + "inputRefMismatches": inputRefMismatches, + "runtimeInputRefDependenciesMissing": [ + buildDependency(parentId, childId) + for parentId, childId in runtimeInputRefDependenciesMissing + ], + "runtimeDependenciesWithoutInputRefs": [ + buildDependency(parentId, childId) + for parentId, childId in runtimeDependenciesWithoutInputRefs + ], + "postgresqlInputRefDependenciesMissing": [ + buildDependency(parentId, childId) + for parentId, childId in postgresqlInputRefDependenciesMissing + ], + "postgresqlDependenciesWithoutInputRefs": [ + buildDependency(parentId, childId) + for parentId, childId in postgresqlDependenciesWithoutInputRefs + ], + "missingParams": [ + buildParamIssue(key, runtimeParamsByProtocolId.get(key[0], {}).get(key[1], {})) + for key in missingParams + ], + "extraParams": [ + buildParamIssue(key, postgresqlParamsByProtocolId.get(key[0], {}).get(key[1], {})) + for key in extraParams + ], + "paramValueMismatches": paramValueMismatches, + } + + issuesCount = sum(len(items) for items in issues.values()) + + return { + "ok": issuesCount == 0, + "projectId": projectId, + "checkedAt": datetime.utcnow().isoformat() + "Z", + "summary": { + "runtimeProtocols": len(runtimeProtocolIds), + "postgresqlProtocols": len(postgresqlProtocolIds), + "runtimeDependencies": len(runtimeDependencies), + "postgresqlDependencies": len(postgresqlDependencies), + "runtimeOutputs": len(runtimeOutputs), + "postgresqlOutputs": len(postgresqlOutputs), + "runtimeSteps": len(runtimeSteps), + "postgresqlSteps": len(postgresqlSteps), + "runtimeInputRefs": len(runtimeInputRefs), + "postgresqlInputRefs": len(postgresqlInputRefsKeys), + "runtimeInputRefDependencies": len(runtimeDependenciesFromInputRefs), + "postgresqlInputRefDependencies": len(postgresqlDependenciesFromInputRefs), + "runtimeParams": len(runtimeParams), + "postgresqlParams": len(postgresqlParams), + "issues": issuesCount, + }, + "issues": issues, + } + diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 7dadc695..cf533822 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2569,1034 +2569,16 @@ def validateProjectPostgresqlConsistency( refresh: bool = True, checkPid: bool = True, ) -> Dict[str, Any]: - dbProj = self.getProjectDbRow(mapper, projectId, currentUser) - if not dbProj: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Project not found", - ) - - self.loadProjectForThumbnails(dbProj) - - def normalizeStatus(value: Any) -> str: - return str(value or "").strip().lower() - - def normalizeClassName(value: Any) -> str: - return str(value or "").strip() - - def normalizeProtocolId(value: Any) -> str: - return str(value).strip() - - def protocolSortKey(value: Any): - text = normalizeProtocolId(value) - try: - return 0, int(text) - except Exception: - return 1, text - - def dependencySortKey(item: Tuple[str, str]): - parentId, childId = item - return protocolSortKey(parentId), protocolSortKey(childId) - - def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: - return { - "parentId": normalizeProtocolId(parentId), - "childId": normalizeProtocolId(childId), - } - - def dependencyKeyFromInputRef(payload: Dict[str, Any]) -> Optional[Tuple[str, str]]: - parentProtocolId = normalizeOptionalText(payload.get("parentProtocolId")) - childProtocolId = normalizeOptionalText(payload.get("protocolId")) - - if not parentProtocolId or not childProtocolId: - return None - - if parentProtocolId == "PROJECT" or childProtocolId == "PROJECT": - return None - - if parentProtocolId == childProtocolId: - return None - - return parentProtocolId, childProtocolId - - def toOptionalInt(value: Any) -> Optional[int]: - try: - if value is None or value == "": - return None - return int(value) - except Exception: - return None - - def stepSortKey(item: Tuple[str, int]): - protocolId, stepIndex = item - return protocolSortKey(protocolId), int(stepIndex) - - def stepValue(step: Any, attrName: str, fallback: Any = None) -> Any: - try: - value = getattr(step, attrName, None) - if hasattr(value, "get"): - return value.get() - return value if value is not None else fallback - except Exception: - return fallback - - def getStepName(step: Any) -> str: - name = stepValue(step, "funcName", None) - if name: - return str(name) - - className = self._safeCall(step, "getClassName", None) - return str(className or "") - - def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[str, Any]: - return { - "protocolId": normalizeProtocolId(protocolId), - "index": int(stepIndex), - "name": str(payload.get("name") or ""), - "status": normalizeStatus(payload.get("status")), - } - - def normalizeOptionalText(value: Any) -> Optional[str]: - if value is None or value == "": - return None - - text = str(value).strip() - return text or None - - def inputRefSortKey(item: Tuple[str, str, int]): - protocolId, inputName, itemIndex = item - return protocolSortKey(protocolId), str(inputName), int(itemIndex) - - def buildInputRef( - key: Tuple[str, str, int], - payload: Dict[str, Any], - ) -> Dict[str, Any]: - protocolId, inputName, itemIndex = key - return { - "protocolId": normalizeProtocolId(protocolId), - "inputName": str(inputName), - "itemIndex": int(itemIndex), - "parentProtocolId": normalizeOptionalText(payload.get("parentProtocolId")), - "parentOutputName": normalizeOptionalText(payload.get("parentOutputName")), - "objectClassName": normalizeOptionalText(payload.get("objectClassName")), - } - - def expectedOutputMapperKind(className: Any) -> Optional[str]: - classNameText = normalizeOptionalText(className) - if classNameText is None: - return None - - if classNameText.startswith("SetOf"): - return "flat_set" - - return "tree" - - def getRuntimeOutputItemsCount(outputObj: Any) -> Optional[int]: - if outputObj is None: - return None - - for methodName in ("getSize", "getDim", "__len__"): - try: - if methodName == "__len__": - value = len(outputObj) - else: - method = getattr(outputObj, methodName, None) - if method is None: - continue - value = method() - - if value is None or value == "": - continue - - return int(value) - except Exception: - continue - - return None - - def iterPointerItems(attr: Any) -> List[Tuple[int, Any]]: - try: - if isinstance(attr, PointerList): - return [ - (index, pointer) - for index, pointer in enumerate(attr) - ] - except Exception: - pass - - return [(0, attr)] - - def extractRuntimeInputRef( - protocolId: str, - inputName: str, - itemIndex: int, - pointer: Any, - ) -> Optional[Dict[str, Any]]: - parentProtocolId = None - parentOutputName = None - objectClassName = None - objectId = None - - try: - parentObj = pointer.getObjValue() - parentProtocolId = normalizeOptionalText( - self._safeCall(parentObj, "getObjId", None) - ) - except Exception: - parentProtocolId = None - - try: - parentOutputName = normalizeOptionalText(pointer.getExtended()) - except Exception: - parentOutputName = None - - try: - targetObj = pointer.get() - if targetObj is not None: - objectClassName = normalizeOptionalText( - self._getScipionClassName(targetObj) - ) - objectId = normalizeOptionalText( - self._safeCall(targetObj, "getObjId", None) - ) - except Exception: - objectClassName = None - objectId = None - - if parentProtocolId is None and parentOutputName is None: - return None - - return { - "protocolId": normalizeProtocolId(protocolId), - "inputName": str(inputName), - "itemIndex": int(itemIndex), - "parentProtocolId": parentProtocolId, - "parentOutputName": parentOutputName, - "objectClassName": objectClassName, - "objectId": objectId, - } - - def normalizeParamValue(value: Any) -> Any: - if value is None: - return None - - if isinstance(value, bool): - return value - - if isinstance(value, (int, float)): - return value - - if isinstance(value, str): - text = value.strip() - if text == "": - return "" - lowerText = text.lower() - if lowerText in ("true", "false"): - return lowerText == "true" - - try: - if "." not in text: - return int(text) - except Exception: - pass - - try: - return float(text) - except Exception: - return text - - if isinstance(value, (list, tuple)): - return [normalizeParamValue(item) for item in value] - - if isinstance(value, dict): - return { - str(key): normalizeParamValue(itemValue) - for key, itemValue in value.items() - } - - try: - if hasattr(value, "get"): - return normalizeParamValue(value.get()) - except Exception: - pass - - return str(value) - - def paramSortKey(item: Tuple[str, str]): - protocolId, paramName = item - return protocolSortKey(protocolId), str(paramName) - - def buildParamIssue( - key: Tuple[str, str], - payload: Dict[str, Any], - ) -> Dict[str, Any]: - protocolId, paramName = key - return { - "protocolId": normalizeProtocolId(protocolId), - "paramName": str(paramName), - "value": payload.get("value"), - } - - def isPointerParam(param: Any) -> bool: - return isinstance(param, (PointerParam, MultiPointerParam, RelationParam)) - - def extractRuntimeParams(protocol: Any) -> Dict[str, Dict[str, Any]]: - paramsByName: Dict[str, Dict[str, Any]] = {} - - try: - self.currentProject._fixProtParamsConfiguration(protocol) - except Exception: - pass - - try: - for paramName, param in protocol.iterParams(): - paramNameText = str(paramName or "").strip() - if not paramNameText: - continue - - if isPointerParam(param): - continue - - rawValue = None - try: - rawValue = protocol.getAttributeValue(paramNameText) - except Exception: - try: - rawValue = getattr(protocol, paramNameText, None) - except Exception: - rawValue = None - - paramsByName[paramNameText] = { - "value": normalizeParamValue(rawValue), - } - except Exception: - logger.debug( - "Could not inspect runtime protocol params during consistency check.", - exc_info=True, - ) - - return paramsByName - - def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: - rawParams = row.get("params") - if not isinstance(rawParams, dict): - return {} - - paramsByName: Dict[str, Dict[str, Any]] = {} - for paramName, rawValue in rawParams.items(): - paramNameText = str(paramName or "").strip() - if not paramNameText: - continue - - paramsByName[paramNameText] = { - "value": normalizeParamValue(rawValue), - } - - return paramsByName - - runtimeStatuses: Dict[str, str] = {} - runtimeClassNames: Dict[str, str] = {} - runtimeDependencies: Set[Tuple[str, str]] = set() - runtimeOutputsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} - runtimeStepsByProtocolId: Dict[str, Dict[int, Dict[str, Any]]] = {} - runtimeInputRefsByKey: Dict[Tuple[str, str, int], Dict[str, Any]] = {} - runtimeParamsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} - - try: - runs = self.currentProject.getRunsGraph(refresh=refresh, checkPids=checkPid) - nodesDict = getattr(runs, "_nodesDict", {}) or {} - except Exception as e: - logger.exception( - "Failed to load Scipion runtime graph for consistency check. projectId=%s", - projectId, - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to load Scipion runtime graph: {e}", - ) - - for nodeId, nodeObj in nodesDict.items(): - protocolId = normalizeProtocolId(nodeId) - if not protocolId or protocolId == "PROJECT": - continue - - protocol = getattr(nodeObj, "run", None) - runtimeStatuses[protocolId] = normalizeStatus( - self._safeCall(protocol, "getStatus", None) - ) - runtimeClassNames[protocolId] = normalizeClassName( - self._safeCall(protocol, "getClassName", None) - ) - runtimeOutputsByProtocolId.setdefault(protocolId, {}) - - if protocol is not None: - try: - for outputItem in protocol.iterOutputAttributes(): - outputName = None - outputObj = None - - if isinstance(outputItem, (tuple, list)) and len(outputItem) >= 2: - outputName = outputItem[0] - outputObj = outputItem[1] - else: - outputName = self._safeCall(outputItem, "getName", None) - outputObj = outputItem - - outputName = str(outputName or "").strip() - if not outputName or outputObj is None: - continue - - outputClassName = self._getScipionClassName(outputObj) - runtimeOutputsByProtocolId[protocolId][outputName] = { - "outputName": outputName, - "className": outputClassName, - "itemsCount": getRuntimeOutputItemsCount(outputObj) - if expectedOutputMapperKind(outputClassName) == "flat_set" - else None, - } - except Exception: - logger.debug( - "Could not inspect runtime protocol outputs during consistency check. " - "projectId=%s protocolId=%s", - projectId, - protocolId, - exc_info=True, - ) - runtimeStepsByProtocolId.setdefault(protocolId, {}) - - try: - for step in protocol.loadSteps() or []: - stepIndex = toOptionalInt(self._safeCall(step, "getIndex", None)) - if stepIndex is None: - continue - - runtimeStepsByProtocolId[protocolId][stepIndex] = { - "index": stepIndex, - "name": getStepName(step), - "status": normalizeStatus(self._safeCall(step, "getStatus", None)), - } - except Exception: - logger.debug( - "Could not inspect runtime protocol steps during consistency check. " - "projectId=%s protocolId=%s", - projectId, - protocolId, - exc_info=True, - ) - try: - for inputName, attr in protocol.iterInputAttributes(): - inputNameText = str(inputName or "").strip() - if not inputNameText: - continue - - for itemIndex, pointer in iterPointerItems(attr): - inputRef = extractRuntimeInputRef( - protocolId=protocolId, - inputName=inputNameText, - itemIndex=int(itemIndex), - pointer=pointer, - ) - if inputRef is None: - continue - - key = ( - inputRef["protocolId"], - inputRef["inputName"], - inputRef["itemIndex"], - ) - runtimeInputRefsByKey[key] = inputRef - except Exception: - logger.debug( - "Could not inspect runtime protocol input refs during consistency check. " - "projectId=%s protocolId=%s", - projectId, - protocolId, - exc_info=True, - ) - - runtimeParamsByProtocolId[protocolId] = extractRuntimeParams(protocol) - - for parent in getattr(nodeObj, "_parents", []) or []: - try: - parentId = normalizeProtocolId(parent.getName()) - except Exception: - parentId = normalizeProtocolId(parent) - - if not parentId or parentId == "PROJECT": - continue - - runtimeDependencies.add((parentId, protocolId)) - - try: - protocolRows = mapper.getProtocols(projectId) or [] - except Exception as e: - logger.exception( - "Failed to load PostgreSQL protocols for consistency check. projectId=%s", - projectId, - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to load PostgreSQL protocols: {e}", - ) - - postgresqlStatuses: Dict[str, str] = {} - postgresqlClassNames: Dict[str, str] = {} - postgresqlParamsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} - - for row in protocolRows: - protocolId = normalizeProtocolId(row.get("protocolId")) - if not protocolId: - continue - - postgresqlStatuses[protocolId] = normalizeStatus(row.get("status")) - postgresqlClassNames[protocolId] = normalizeClassName(row.get("protocolClassName")) - postgresqlParamsByProtocolId[protocolId] = extractPostgresqlParams(row) - - try: - adjacencyMap = mapper.getProjectProtocolAdjacencyMap(projectId) or {} - except Exception as e: - logger.exception( - "Failed to load PostgreSQL protocol dependencies for consistency check. projectId=%s", - projectId, - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to load PostgreSQL protocol dependencies: {e}", - ) - - postgresqlDependencies: Set[Tuple[str, str]] = set() - for childId, refs in adjacencyMap.items(): - childProtocolId = normalizeProtocolId(childId) - if not childProtocolId or childProtocolId == "PROJECT": - continue - - for parentIdValue in refs.get("parents") or []: - parentProtocolId = normalizeProtocolId(parentIdValue) - if not parentProtocolId or parentProtocolId == "PROJECT": - continue - - postgresqlDependencies.add((parentProtocolId, childProtocolId)) - - try: - persistedOutputsByProtocolId = self._loadPersistedOutputsByProtocolId( - mapper, - projectId, - ) - except Exception as e: - logger.exception( - "Failed to load PostgreSQL persisted outputs for consistency check. projectId=%s", - projectId, - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to load PostgreSQL persisted outputs: {e}", - ) - - try: - postgresqlStepsByProtocolId = mapper.getProjectProtocolStepsByProtocolId(projectId) or {} - except Exception as e: - logger.exception( - "Failed to load PostgreSQL protocol steps for consistency check. projectId=%s", - projectId, - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to load PostgreSQL protocol steps: {e}", - ) - - normalizedPostgresqlStepsByProtocolId: Dict[str, Dict[int, Dict[str, Any]]] = {} - for protocolId, steps in postgresqlStepsByProtocolId.items(): - protocolIdText = normalizeProtocolId(protocolId) - for step in steps or []: - stepIndex = toOptionalInt(step.get("index")) - if stepIndex is None: - continue - - normalizedPostgresqlStepsByProtocolId.setdefault(protocolIdText, {})[stepIndex] = { - "index": stepIndex, - "name": str(step.get("name") or ""), - "status": normalizeStatus(step.get("status")), - } - - try: - postgresqlInputRefs = mapper.listProtocolInputRefs(projectId) or [] - except Exception as e: - logger.exception( - "Failed to load PostgreSQL protocol input refs for consistency check. projectId=%s", - projectId, - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to load PostgreSQL protocol input refs: {e}", - ) - - postgresqlInputRefsByKey: Dict[Tuple[str, str, int], Dict[str, Any]] = {} - for ref in postgresqlInputRefs: - protocolIdText = normalizeProtocolId(ref.get("protocolId")) - inputName = str(ref.get("inputName") or "").strip() - itemIndex = toOptionalInt(ref.get("itemIndex")) - - if not protocolIdText or not inputName: - continue - - if itemIndex is None or itemIndex < 0: - itemIndex = 0 - - key = (protocolIdText, inputName, int(itemIndex)) - postgresqlInputRefsByKey[key] = { - "protocolId": protocolIdText, - "inputName": inputName, - "itemIndex": int(itemIndex), - "parentProtocolId": normalizeOptionalText(ref.get("parentProtocolId")), - "parentOutputName": normalizeOptionalText(ref.get("parentOutputName")), - "objectClassName": normalizeOptionalText(ref.get("objectClassName")), - "objectId": normalizeOptionalText(ref.get("objectId")), - } - - runtimeOutputs: Set[Tuple[str, str]] = set() - for protocolId, outputsByName in runtimeOutputsByProtocolId.items(): - for outputName in outputsByName.keys(): - runtimeOutputs.add((protocolId, outputName)) - - postgresqlOutputs: Set[Tuple[str, str]] = set() - for protocolId, outputsByName in persistedOutputsByProtocolId.items(): - for outputName in outputsByName.keys(): - postgresqlOutputs.add((normalizeProtocolId(protocolId), str(outputName))) - - runtimeSteps: Set[Tuple[str, int]] = set() - for protocolId, stepsByIndex in runtimeStepsByProtocolId.items(): - for stepIndex in stepsByIndex.keys(): - runtimeSteps.add((protocolId, int(stepIndex))) - - postgresqlSteps: Set[Tuple[str, int]] = set() - for protocolId, stepsByIndex in normalizedPostgresqlStepsByProtocolId.items(): - for stepIndex in stepsByIndex.keys(): - postgresqlSteps.add((protocolId, int(stepIndex))) - - runtimeProtocolIds = set(runtimeStatuses.keys()) - postgresqlProtocolIds = set(postgresqlStatuses.keys()) - runtimeInputRefs = set(runtimeInputRefsByKey.keys()) - postgresqlInputRefsKeys = set(postgresqlInputRefsByKey.keys()) - runtimeDependenciesFromInputRefs: Set[Tuple[str, str]] = set() - - runtimeParams: Set[Tuple[str, str]] = set() - for protocolId, paramsByName in runtimeParamsByProtocolId.items(): - for paramName in paramsByName.keys(): - runtimeParams.add((protocolId, paramName)) - - postgresqlParams: Set[Tuple[str, str]] = set() - for protocolId, paramsByName in postgresqlParamsByProtocolId.items(): - for paramName in paramsByName.keys(): - postgresqlParams.add((protocolId, paramName)) - - for inputRef in runtimeInputRefsByKey.values(): - dependencyKey = dependencyKeyFromInputRef(inputRef) - if dependencyKey is not None: - runtimeDependenciesFromInputRefs.add(dependencyKey) - - postgresqlDependenciesFromInputRefs: Set[Tuple[str, str]] = set() - for inputRef in postgresqlInputRefsByKey.values(): - dependencyKey = dependencyKeyFromInputRef(inputRef) - if dependencyKey is not None: - postgresqlDependenciesFromInputRefs.add(dependencyKey) - - missingProtocolIds = sorted( - runtimeProtocolIds - postgresqlProtocolIds, - key=protocolSortKey, - ) - extraProtocolIds = sorted( - postgresqlProtocolIds - runtimeProtocolIds, - key=protocolSortKey, - ) - - commonProtocolIds = sorted( - runtimeProtocolIds.intersection(postgresqlProtocolIds), - key=protocolSortKey, - ) - - statusMismatches = [] - for protocolId in commonProtocolIds: - runtimeStatus = runtimeStatuses.get(protocolId, "") - postgresqlStatus = postgresqlStatuses.get(protocolId, "") - - if runtimeStatus != postgresqlStatus: - statusMismatches.append({ - "protocolId": protocolId, - "runtimeStatus": runtimeStatus, - "postgresqlStatus": postgresqlStatus, - }) - - protocolClassMismatches = [] - for protocolId in commonProtocolIds: - runtimeClassName = normalizeClassName(runtimeClassNames.get(protocolId)) - postgresqlClassName = normalizeClassName(postgresqlClassNames.get(protocolId)) - - if ( - runtimeClassName - and postgresqlClassName - and runtimeClassName != postgresqlClassName - ): - protocolClassMismatches.append({ - "protocolId": protocolId, - "runtimeClassName": runtimeClassName, - "postgresqlClassName": postgresqlClassName, - }) + from app.backend.api.services.project_consistency_service import ProjectConsistencyService - missingDependencies = sorted( - runtimeDependencies - postgresqlDependencies, - key=dependencySortKey, - ) - extraDependencies = sorted( - postgresqlDependencies - runtimeDependencies, - key=dependencySortKey, - ) - - missingOutputs = sorted( - runtimeOutputs - postgresqlOutputs, - key=dependencySortKey, - ) - extraOutputs = sorted( - postgresqlOutputs - runtimeOutputs, - key=dependencySortKey, - ) - - outputClassMismatches = [] - for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=dependencySortKey): - runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - - runtimeClassName = normalizeOptionalText(runtimeOutput.get("className")) - postgresqlClassName = normalizeOptionalText(postgresqlOutput.get("className")) - - if ( - runtimeClassName is not None - and postgresqlClassName is not None - and runtimeClassName != postgresqlClassName - ): - outputClassMismatches.append({ - "protocolId": protocolId, - "outputName": outputName, - "runtimeClassName": runtimeClassName, - "postgresqlClassName": postgresqlClassName, - "mapperKind": postgresqlOutput.get("mapperKind"), - }) - - outputMapperKindMismatches = [] - for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=dependencySortKey): - runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - - runtimeClassName = normalizeOptionalText(runtimeOutput.get("className")) - expectedMapperKind = expectedOutputMapperKind(runtimeClassName) - postgresqlMapperKind = normalizeOptionalText(postgresqlOutput.get("mapperKind")) - - if ( - expectedMapperKind is not None - and postgresqlMapperKind is not None - and expectedMapperKind != postgresqlMapperKind - ): - outputMapperKindMismatches.append({ - "protocolId": protocolId, - "outputName": outputName, - "className": runtimeClassName, - "expectedMapperKind": expectedMapperKind, - "postgresqlMapperKind": postgresqlMapperKind, - }) - - outputItemsCountMismatches = [] - for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=dependencySortKey): - runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - - runtimeClassName = normalizeOptionalText(runtimeOutput.get("className")) - if expectedOutputMapperKind(runtimeClassName) != "flat_set": - continue - - runtimeItemsCount = toOptionalInt(runtimeOutput.get("itemsCount")) - postgresqlItemsCount = toOptionalInt(postgresqlOutput.get("itemsCount")) - - if runtimeItemsCount is None or postgresqlItemsCount is None: - continue - - if runtimeItemsCount != postgresqlItemsCount: - outputItemsCountMismatches.append({ - "protocolId": protocolId, - "outputName": outputName, - "className": runtimeClassName, - "runtimeItemsCount": runtimeItemsCount, - "postgresqlItemsCount": postgresqlItemsCount, - "mapperKind": postgresqlOutput.get("mapperKind"), - }) - - missingSteps = sorted( - runtimeSteps - postgresqlSteps, - key=stepSortKey, - ) - extraSteps = sorted( - postgresqlSteps - runtimeSteps, - key=stepSortKey, - ) - - stepMismatches = [] - for protocolId, stepIndex in sorted(runtimeSteps.intersection(postgresqlSteps), key=stepSortKey): - runtimeStep = runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) - postgresqlStep = normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) - - runtimeName = str(runtimeStep.get("name") or "") - postgresqlName = str(postgresqlStep.get("name") or "") - runtimeStatus = normalizeStatus(runtimeStep.get("status")) - postgresqlStatus = normalizeStatus(postgresqlStep.get("status")) - - changedFields = [] - if runtimeName != postgresqlName: - changedFields.append("name") - if runtimeStatus != postgresqlStatus: - changedFields.append("status") - - if changedFields: - stepMismatches.append({ - "protocolId": protocolId, - "index": int(stepIndex), - "fields": changedFields, - "runtimeName": runtimeName, - "postgresqlName": postgresqlName, - "runtimeStatus": runtimeStatus, - "postgresqlStatus": postgresqlStatus, - }) - - - missingInputRefs = sorted( - runtimeInputRefs - postgresqlInputRefsKeys, - key=inputRefSortKey, - ) - extraInputRefs = sorted( - postgresqlInputRefsKeys - runtimeInputRefs, - key=inputRefSortKey, - ) - - missingParams = sorted( - runtimeParams - postgresqlParams, - key=paramSortKey, - ) - extraParams = sorted( - postgresqlParams - runtimeParams, - key=paramSortKey, - ) - - paramValueMismatches = [] - for key in sorted(runtimeParams.intersection(postgresqlParams), key=paramSortKey): - protocolId, paramName = key - - runtimeParam = runtimeParamsByProtocolId.get(protocolId, {}).get(paramName, {}) - postgresqlParam = postgresqlParamsByProtocolId.get(protocolId, {}).get(paramName, {}) - - runtimeValue = normalizeParamValue(runtimeParam.get("value")) - postgresqlValue = normalizeParamValue(postgresqlParam.get("value")) - - if runtimeValue != postgresqlValue: - paramValueMismatches.append({ - "protocolId": protocolId, - "paramName": paramName, - "runtimeValue": runtimeValue, - "postgresqlValue": postgresqlValue, - }) - - inputRefMismatches = [] - for key in sorted(runtimeInputRefs.intersection(postgresqlInputRefsKeys), key=inputRefSortKey): - runtimeRef = runtimeInputRefsByKey.get(key, {}) - postgresqlRef = postgresqlInputRefsByKey.get(key, {}) - - changedFields = [] - - runtimeParentProtocolId = normalizeOptionalText(runtimeRef.get("parentProtocolId")) - postgresqlParentProtocolId = normalizeOptionalText(postgresqlRef.get("parentProtocolId")) - - runtimeParentOutputName = normalizeOptionalText(runtimeRef.get("parentOutputName")) - postgresqlParentOutputName = normalizeOptionalText(postgresqlRef.get("parentOutputName")) - - runtimeObjectClassName = normalizeOptionalText(runtimeRef.get("objectClassName")) - postgresqlObjectClassName = normalizeOptionalText(postgresqlRef.get("objectClassName")) - - if runtimeParentProtocolId != postgresqlParentProtocolId: - changedFields.append("parentProtocolId") - - if runtimeParentOutputName != postgresqlParentOutputName: - changedFields.append("parentOutputName") - - if ( - runtimeObjectClassName is not None - and postgresqlObjectClassName is not None - and runtimeObjectClassName != postgresqlObjectClassName - ): - changedFields.append("objectClassName") - - if changedFields: - protocolId, inputName, itemIndex = key - inputRefMismatches.append({ - "protocolId": protocolId, - "inputName": inputName, - "itemIndex": int(itemIndex), - "fields": changedFields, - "runtimeParentProtocolId": runtimeParentProtocolId, - "postgresqlParentProtocolId": postgresqlParentProtocolId, - "runtimeParentOutputName": runtimeParentOutputName, - "postgresqlParentOutputName": postgresqlParentOutputName, - "runtimeObjectClassName": runtimeObjectClassName, - "postgresqlObjectClassName": postgresqlObjectClassName, - }) - - runtimeInputRefDependenciesMissing = sorted( - runtimeDependenciesFromInputRefs - runtimeDependencies, - key=dependencySortKey, - ) - runtimeDependenciesWithoutInputRefs = sorted( - runtimeDependencies - runtimeDependenciesFromInputRefs, - key=dependencySortKey, - ) - - postgresqlInputRefDependenciesMissing = sorted( - postgresqlDependenciesFromInputRefs - postgresqlDependencies, - key=dependencySortKey, - ) - postgresqlDependenciesWithoutInputRefs = sorted( - postgresqlDependencies - postgresqlDependenciesFromInputRefs, - key=dependencySortKey, + return ProjectConsistencyService(self).validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=projectId, + currentUser=currentUser, + refresh=refresh, + checkPid=checkPid, ) - issues = { - "missingProtocols": [ - { - "protocolId": protocolId, - "runtimeStatus": runtimeStatuses.get(protocolId, ""), - } - for protocolId in missingProtocolIds - ], - "extraProtocols": [ - { - "protocolId": protocolId, - "postgresqlStatus": postgresqlStatuses.get(protocolId, ""), - } - for protocolId in extraProtocolIds - ], - "statusMismatches": statusMismatches, - "protocolClassMismatches": protocolClassMismatches, - "missingDependencies": [ - buildDependency(parentId, childId) - for parentId, childId in missingDependencies - ], - "extraDependencies": [ - buildDependency(parentId, childId) - for parentId, childId in extraDependencies - ], - "missingOutputs": [ - { - "protocolId": protocolId, - "outputName": outputName, - "className": runtimeOutputsByProtocolId - .get(protocolId, {}) - .get(outputName, {}) - .get("className"), - } - for protocolId, outputName in missingOutputs - ], - "extraOutputs": [ - { - "protocolId": protocolId, - "outputName": outputName, - "mapperKind": persistedOutputsByProtocolId - .get(protocolId, {}) - .get(outputName, {}) - .get("mapperKind"), - "className": persistedOutputsByProtocolId - .get(protocolId, {}) - .get(outputName, {}) - .get("className"), - } - for protocolId, outputName in extraOutputs - ], - "outputClassMismatches": outputClassMismatches, - "outputMapperKindMismatches": outputMapperKindMismatches, - "outputItemsCountMismatches": outputItemsCountMismatches, - - "missingSteps": [ - buildStep( - protocolId, - stepIndex, - runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}), - ) - for protocolId, stepIndex in missingSteps - ], - "extraSteps": [ - buildStep( - protocolId, - stepIndex, - normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}), - ) - for protocolId, stepIndex in extraSteps - ], - "stepMismatches": stepMismatches, - "missingInputRefs": [ - buildInputRef(key, runtimeInputRefsByKey.get(key, {})) - for key in missingInputRefs - ], - "extraInputRefs": [ - buildInputRef(key, postgresqlInputRefsByKey.get(key, {})) - for key in extraInputRefs - ], - "inputRefMismatches": inputRefMismatches, - "runtimeInputRefDependenciesMissing": [ - buildDependency(parentId, childId) - for parentId, childId in runtimeInputRefDependenciesMissing - ], - "runtimeDependenciesWithoutInputRefs": [ - buildDependency(parentId, childId) - for parentId, childId in runtimeDependenciesWithoutInputRefs - ], - "postgresqlInputRefDependenciesMissing": [ - buildDependency(parentId, childId) - for parentId, childId in postgresqlInputRefDependenciesMissing - ], - "postgresqlDependenciesWithoutInputRefs": [ - buildDependency(parentId, childId) - for parentId, childId in postgresqlDependenciesWithoutInputRefs - ], - "missingParams": [ - buildParamIssue(key, runtimeParamsByProtocolId.get(key[0], {}).get(key[1], {})) - for key in missingParams - ], - "extraParams": [ - buildParamIssue(key, postgresqlParamsByProtocolId.get(key[0], {}).get(key[1], {})) - for key in extraParams - ], - "paramValueMismatches": paramValueMismatches, - } - - issuesCount = sum(len(items) for items in issues.values()) - - return { - "ok": issuesCount == 0, - "projectId": projectId, - "checkedAt": datetime.utcnow().isoformat() + "Z", - "summary": { - "runtimeProtocols": len(runtimeProtocolIds), - "postgresqlProtocols": len(postgresqlProtocolIds), - "runtimeDependencies": len(runtimeDependencies), - "postgresqlDependencies": len(postgresqlDependencies), - "runtimeOutputs": len(runtimeOutputs), - "postgresqlOutputs": len(postgresqlOutputs), - "runtimeSteps": len(runtimeSteps), - "postgresqlSteps": len(postgresqlSteps), - "runtimeInputRefs": len(runtimeInputRefs), - "postgresqlInputRefs": len(postgresqlInputRefsKeys), - "runtimeInputRefDependencies": len(runtimeDependenciesFromInputRefs), - "postgresqlInputRefDependencies": len(postgresqlDependenciesFromInputRefs), - "runtimeParams": len(runtimeParams), - "postgresqlParams": len(postgresqlParams), - "issues": issuesCount, - }, - "issues": issues, - } - def listProjectWorkflows(self, raw: bool = False): """ Return available workflow templates. From b262fcea2fbdb58dc90300bcdf34ba51925df032 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 22:50:40 +0200 Subject: [PATCH 182/278] Move consistency checker helpers to service methods --- .../services/project_consistency_service.py | 724 +++++++++--------- 1 file changed, 369 insertions(+), 355 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 6c4269a5..acbe8b45 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -9,6 +9,19 @@ # * the Free Software Foundation; either version 3 of the License, or # * (at your option) any later version. # * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * # ****************************************************************************** from __future__ import annotations @@ -44,339 +57,341 @@ def currentProject(self, value): def __getattr__(self, name: str) -> Any: return getattr(self.projectService, name) + def normalizeStatus(self, value: Any) -> str: + return str(value or "").strip().lower() - def validateProjectPostgresqlConsistency( - self, - mapper: PostgresqlFlatMapper, - projectId: int, - currentUser: dict, - refresh: bool = True, - checkPid: bool = True, - ) -> Dict[str, Any]: - dbProj = self.getProjectDbRow(mapper, projectId, currentUser) - if not dbProj: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Project not found", - ) - - self.loadProjectForThumbnails(dbProj) + def normalizeClassName(self, value: Any) -> str: + return str(value or "").strip() - def normalizeStatus(value: Any) -> str: - return str(value or "").strip().lower() + def normalizeProtocolId(self, value: Any) -> str: + return str(value).strip() - def normalizeClassName(value: Any) -> str: - return str(value or "").strip() + def normalizeOptionalText(self, value: Any) -> Optional[str]: + if value is None or value == "": + return None - def normalizeProtocolId(value: Any) -> str: - return str(value).strip() + text = str(value).strip() + return text or None - def protocolSortKey(value: Any): - text = normalizeProtocolId(value) - try: - return 0, int(text) - except Exception: - return 1, text + def toOptionalInt(self, value: Any) -> Optional[int]: + try: + if value is None or value == "": + return None + return int(value) + except Exception: + return None - def dependencySortKey(item: Tuple[str, str]): - parentId, childId = item - return protocolSortKey(parentId), protocolSortKey(childId) + def protocolSortKey(self, value: Any): + text = self.normalizeProtocolId(value) + try: + return 0, int(text) + except Exception: + return 1, text - def buildDependency(parentId: Any, childId: Any) -> Dict[str, str]: - return { - "parentId": normalizeProtocolId(parentId), - "childId": normalizeProtocolId(childId), - } + def dependencySortKey(self, item: Tuple[str, str]): + parentId, childId = item + return self.protocolSortKey(parentId), self.protocolSortKey(childId) - def dependencyKeyFromInputRef(payload: Dict[str, Any]) -> Optional[Tuple[str, str]]: - parentProtocolId = normalizeOptionalText(payload.get("parentProtocolId")) - childProtocolId = normalizeOptionalText(payload.get("protocolId")) + def stepSortKey(self, item: Tuple[str, int]): + protocolId, stepIndex = item + return self.protocolSortKey(protocolId), int(stepIndex) - if not parentProtocolId or not childProtocolId: - return None + def inputRefSortKey(self, item: Tuple[str, str, int]): + protocolId, inputName, itemIndex = item + return self.protocolSortKey(protocolId), str(inputName), int(itemIndex) - if parentProtocolId == "PROJECT" or childProtocolId == "PROJECT": - return None + def paramSortKey(self, item: Tuple[str, str]): + protocolId, paramName = item + return self.protocolSortKey(protocolId), str(paramName) - if parentProtocolId == childProtocolId: - return None + def buildDependency(self, parentId: Any, childId: Any) -> Dict[str, str]: + return { + "parentId": self.normalizeProtocolId(parentId), + "childId": self.normalizeProtocolId(childId), + } - return parentProtocolId, childProtocolId + def buildStep(self, protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[str, Any]: + return { + "protocolId": self.normalizeProtocolId(protocolId), + "index": int(stepIndex), + "name": str(payload.get("name") or ""), + "status": self.normalizeStatus(payload.get("status")), + } - def toOptionalInt(value: Any) -> Optional[int]: - try: - if value is None or value == "": - return None - return int(value) - except Exception: - return None + def buildInputRef( + self, + key: Tuple[str, str, int], + payload: Dict[str, Any], + ) -> Dict[str, Any]: + protocolId, inputName, itemIndex = key + return { + "protocolId": self.normalizeProtocolId(protocolId), + "inputName": str(inputName), + "itemIndex": int(itemIndex), + "parentProtocolId": self.normalizeOptionalText(payload.get("parentProtocolId")), + "parentOutputName": self.normalizeOptionalText(payload.get("parentOutputName")), + "objectClassName": self.normalizeOptionalText(payload.get("objectClassName")), + } - def stepSortKey(item: Tuple[str, int]): - protocolId, stepIndex = item - return protocolSortKey(protocolId), int(stepIndex) + def buildParamIssue( + self, + key: Tuple[str, str], + payload: Dict[str, Any], + ) -> Dict[str, Any]: + protocolId, paramName = key + return { + "protocolId": self.normalizeProtocolId(protocolId), + "paramName": str(paramName), + "value": payload.get("value"), + } - def stepValue(step: Any, attrName: str, fallback: Any = None) -> Any: - try: - value = getattr(step, attrName, None) - if hasattr(value, "get"): - return value.get() - return value if value is not None else fallback - except Exception: - return fallback + def expectedOutputMapperKind(self, className: Any) -> Optional[str]: + classNameText = self.normalizeOptionalText(className) + if classNameText is None: + return None - def getStepName(step: Any) -> str: - name = stepValue(step, "funcName", None) - if name: - return str(name) + if classNameText.startswith("SetOf"): + return "flat_set" - className = self._safeCall(step, "getClassName", None) - return str(className or "") + return "tree" - def buildStep(protocolId: Any, stepIndex: Any, payload: Dict[str, Any]) -> Dict[str, Any]: - return { - "protocolId": normalizeProtocolId(protocolId), - "index": int(stepIndex), - "name": str(payload.get("name") or ""), - "status": normalizeStatus(payload.get("status")), - } + def getRuntimeOutputItemsCount(self, outputObj: Any) -> Optional[int]: + if outputObj is None: + return None - def normalizeOptionalText(value: Any) -> Optional[str]: - if value is None or value == "": - return None + for methodName in ("getSize", "getDim", "__len__"): + try: + if methodName == "__len__": + value = len(outputObj) + else: + method = getattr(outputObj, methodName, None) + if method is None: + continue + value = method() - text = str(value).strip() - return text or None + if value is None or value == "": + continue - def inputRefSortKey(item: Tuple[str, str, int]): - protocolId, inputName, itemIndex = item - return protocolSortKey(protocolId), str(inputName), int(itemIndex) + return int(value) + except Exception: + continue - def buildInputRef( - key: Tuple[str, str, int], - payload: Dict[str, Any], - ) -> Dict[str, Any]: - protocolId, inputName, itemIndex = key - return { - "protocolId": normalizeProtocolId(protocolId), - "inputName": str(inputName), - "itemIndex": int(itemIndex), - "parentProtocolId": normalizeOptionalText(payload.get("parentProtocolId")), - "parentOutputName": normalizeOptionalText(payload.get("parentOutputName")), - "objectClassName": normalizeOptionalText(payload.get("objectClassName")), - } + return None - def expectedOutputMapperKind(className: Any) -> Optional[str]: - classNameText = normalizeOptionalText(className) - if classNameText is None: - return None + def iterPointerItems(self, attr: Any) -> List[Tuple[int, Any]]: + try: + if isinstance(attr, PointerList): + return [ + (index, pointer) + for index, pointer in enumerate(attr) + ] + except Exception: + pass - if classNameText.startswith("SetOf"): - return "flat_set" + return [(0, attr)] - return "tree" + def dependencyKeyFromInputRef(self, payload: Dict[str, Any]) -> Optional[Tuple[str, str]]: + parentProtocolId = self.normalizeOptionalText(payload.get("parentProtocolId")) + childProtocolId = self.normalizeOptionalText(payload.get("protocolId")) - def getRuntimeOutputItemsCount(outputObj: Any) -> Optional[int]: - if outputObj is None: - return None + if not parentProtocolId or not childProtocolId: + return None - for methodName in ("getSize", "getDim", "__len__"): - try: - if methodName == "__len__": - value = len(outputObj) - else: - method = getattr(outputObj, methodName, None) - if method is None: - continue - value = method() + if parentProtocolId == "PROJECT" or childProtocolId == "PROJECT": + return None - if value is None or value == "": - continue + if parentProtocolId == childProtocolId: + return None - return int(value) - except Exception: - continue + return parentProtocolId, childProtocolId + def normalizeParamValue(self, value: Any) -> Any: + if value is None: return None - def iterPointerItems(attr: Any) -> List[Tuple[int, Any]]: - try: - if isinstance(attr, PointerList): - return [ - (index, pointer) - for index, pointer in enumerate(attr) - ] - except Exception: - pass + if isinstance(value, bool): + return value - return [(0, attr)] + if isinstance(value, (int, float)): + return value - def extractRuntimeInputRef( - protocolId: str, - inputName: str, - itemIndex: int, - pointer: Any, - ) -> Optional[Dict[str, Any]]: - parentProtocolId = None - parentOutputName = None - objectClassName = None - objectId = None + if isinstance(value, str): + text = value.strip() + if text == "": + return "" + lowerText = text.lower() + if lowerText in ("true", "false"): + return lowerText == "true" try: - parentObj = pointer.getObjValue() - parentProtocolId = normalizeOptionalText( - self._safeCall(parentObj, "getObjId", None) - ) + if "." not in text: + return int(text) except Exception: - parentProtocolId = None + pass try: - parentOutputName = normalizeOptionalText(pointer.getExtended()) + return float(text) except Exception: - parentOutputName = None + return text - try: - targetObj = pointer.get() - if targetObj is not None: - objectClassName = normalizeOptionalText( - self._getScipionClassName(targetObj) - ) - objectId = normalizeOptionalText( - self._safeCall(targetObj, "getObjId", None) - ) - except Exception: - objectClassName = None - objectId = None - - if parentProtocolId is None and parentOutputName is None: - return None + if isinstance(value, (list, tuple)): + return [self.normalizeParamValue(item) for item in value] + if isinstance(value, dict): return { - "protocolId": normalizeProtocolId(protocolId), - "inputName": str(inputName), - "itemIndex": int(itemIndex), - "parentProtocolId": parentProtocolId, - "parentOutputName": parentOutputName, - "objectClassName": objectClassName, - "objectId": objectId, + str(key): self.normalizeParamValue(itemValue) + for key, itemValue in value.items() } - def normalizeParamValue(value: Any) -> Any: - if value is None: - return None - - if isinstance(value, bool): - return value + try: + if hasattr(value, "get"): + return self.normalizeParamValue(value.get()) + except Exception: + pass - if isinstance(value, (int, float)): - return value + return str(value) - if isinstance(value, str): - text = value.strip() - if text == "": - return "" - lowerText = text.lower() - if lowerText in ("true", "false"): - return lowerText == "true" + def isPointerParam(self, param: Any) -> bool: + return isinstance(param, (PointerParam, MultiPointerParam, RelationParam)) - try: - if "." not in text: - return int(text) - except Exception: - pass + def stepValue(self, step: Any, attrName: str, fallback: Any = None) -> Any: + try: + value = getattr(step, attrName, None) + if hasattr(value, "get"): + return value.get() + return value if value is not None else fallback + except Exception: + return fallback + + def getStepName(self, step: Any) -> str: + name = self.stepValue(step, "funcName", None) + if name: + return str(name) + + className = self._safeCall(step, "getClassName", None) + return str(className or "") + + def extractRuntimeInputRef( + self, + protocolId: str, + inputName: str, + itemIndex: int, + pointer: Any, + ) -> Optional[Dict[str, Any]]: + parentProtocolId = None + parentOutputName = None + objectClassName = None + objectId = None - try: - return float(text) - except Exception: - return text + try: + parentObj = pointer.getObjValue() + parentProtocolId = self.normalizeOptionalText( + self._safeCall(parentObj, "getObjId", None) + ) + except Exception: + parentProtocolId = None - if isinstance(value, (list, tuple)): - return [normalizeParamValue(item) for item in value] + try: + parentOutputName = self.normalizeOptionalText(pointer.getExtended()) + except Exception: + parentOutputName = None - if isinstance(value, dict): - return { - str(key): normalizeParamValue(itemValue) - for key, itemValue in value.items() - } + try: + targetObj = pointer.get() + if targetObj is not None: + objectClassName = self.normalizeOptionalText( + self._getScipionClassName(targetObj) + ) + objectId = self.normalizeOptionalText( + self._safeCall(targetObj, "getObjId", None) + ) + except Exception: + objectClassName = None + objectId = None - try: - if hasattr(value, "get"): - return normalizeParamValue(value.get()) - except Exception: - pass + if parentProtocolId is None and parentOutputName is None: + return None - return str(value) + return { + "protocolId": self.normalizeProtocolId(protocolId), + "inputName": str(inputName), + "itemIndex": int(itemIndex), + "parentProtocolId": parentProtocolId, + "parentOutputName": parentOutputName, + "objectClassName": objectClassName, + "objectId": objectId, + } - def paramSortKey(item: Tuple[str, str]): - protocolId, paramName = item - return protocolSortKey(protocolId), str(paramName) + def extractRuntimeParams(self, protocol: Any) -> Dict[str, Dict[str, Any]]: + paramsByName: Dict[str, Dict[str, Any]] = {} - def buildParamIssue( - key: Tuple[str, str], - payload: Dict[str, Any], - ) -> Dict[str, Any]: - protocolId, paramName = key - return { - "protocolId": normalizeProtocolId(protocolId), - "paramName": str(paramName), - "value": payload.get("value"), - } + try: + self.currentProject._fixProtParamsConfiguration(protocol) + except Exception: + pass - def isPointerParam(param: Any) -> bool: - return isinstance(param, (PointerParam, MultiPointerParam, RelationParam)) + try: + for paramName, param in protocol.iterParams(): + paramNameText = str(paramName or "").strip() + if not paramNameText: + continue - def extractRuntimeParams(protocol: Any) -> Dict[str, Dict[str, Any]]: - paramsByName: Dict[str, Dict[str, Any]] = {} + if self.isPointerParam(param): + continue - try: - self.currentProject._fixProtParamsConfiguration(protocol) - except Exception: - pass + rawValue = None + try: + rawValue = protocol.getAttributeValue(paramNameText) + except Exception: + try: + rawValue = getattr(protocol, paramNameText, None) + except Exception: + rawValue = None - try: - for paramName, param in protocol.iterParams(): - paramNameText = str(paramName or "").strip() - if not paramNameText: - continue + paramsByName[paramNameText] = { + "value": self.normalizeParamValue(rawValue), + } + except Exception: + logger.debug( + "Could not inspect runtime protocol params during consistency check.", + exc_info=True, + ) - if isPointerParam(param): - continue + return paramsByName - rawValue = None - try: - rawValue = protocol.getAttributeValue(paramNameText) - except Exception: - try: - rawValue = getattr(protocol, paramNameText, None) - except Exception: - rawValue = None - - paramsByName[paramNameText] = { - "value": normalizeParamValue(rawValue), - } - except Exception: - logger.debug( - "Could not inspect runtime protocol params during consistency check.", - exc_info=True, - ) + def extractPostgresqlParams(self, row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + rawParams = row.get("params") + if not isinstance(rawParams, dict): + return {} - return paramsByName + paramsByName: Dict[str, Dict[str, Any]] = {} + for paramName, rawValue in rawParams.items(): + paramNameText = str(paramName or "").strip() + if not paramNameText: + continue - def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: - rawParams = row.get("params") - if not isinstance(rawParams, dict): - return {} + paramsByName[paramNameText] = { + "value": self.normalizeParamValue(rawValue), + } - paramsByName: Dict[str, Dict[str, Any]] = {} - for paramName, rawValue in rawParams.items(): - paramNameText = str(paramName or "").strip() - if not paramNameText: - continue + return paramsByName - paramsByName[paramNameText] = { - "value": normalizeParamValue(rawValue), - } + def validateProjectPostgresqlConsistency( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser: dict, + refresh: bool = True, + checkPid: bool = True, + ) -> Dict[str, Any]: + dbProj = self.getProjectDbRow(mapper, projectId, currentUser) + if not dbProj: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Project not found", + ) - return paramsByName + self.loadProjectForThumbnails(dbProj) runtimeStatuses: Dict[str, str] = {} runtimeClassNames: Dict[str, str] = {} @@ -400,15 +415,15 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: ) for nodeId, nodeObj in nodesDict.items(): - protocolId = normalizeProtocolId(nodeId) + protocolId = self.normalizeProtocolId(nodeId) if not protocolId or protocolId == "PROJECT": continue protocol = getattr(nodeObj, "run", None) - runtimeStatuses[protocolId] = normalizeStatus( + runtimeStatuses[protocolId] = self.normalizeStatus( self._safeCall(protocol, "getStatus", None) ) - runtimeClassNames[protocolId] = normalizeClassName( + runtimeClassNames[protocolId] = self.normalizeClassName( self._safeCall(protocol, "getClassName", None) ) runtimeOutputsByProtocolId.setdefault(protocolId, {}) @@ -434,8 +449,8 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: runtimeOutputsByProtocolId[protocolId][outputName] = { "outputName": outputName, "className": outputClassName, - "itemsCount": getRuntimeOutputItemsCount(outputObj) - if expectedOutputMapperKind(outputClassName) == "flat_set" + "itemsCount": self.getRuntimeOutputItemsCount(outputObj) + if self.expectedOutputMapperKind(outputClassName) == "flat_set" else None, } except Exception: @@ -450,14 +465,14 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: try: for step in protocol.loadSteps() or []: - stepIndex = toOptionalInt(self._safeCall(step, "getIndex", None)) + stepIndex = self.toOptionalInt(self._safeCall(step, "getIndex", None)) if stepIndex is None: continue runtimeStepsByProtocolId[protocolId][stepIndex] = { "index": stepIndex, - "name": getStepName(step), - "status": normalizeStatus(self._safeCall(step, "getStatus", None)), + "name": self.getStepName(step), + "status": self.normalizeStatus(self._safeCall(step, "getStatus", None)), } except Exception: logger.debug( @@ -473,8 +488,8 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: if not inputNameText: continue - for itemIndex, pointer in iterPointerItems(attr): - inputRef = extractRuntimeInputRef( + for itemIndex, pointer in self.iterPointerItems(attr): + inputRef = self.extractRuntimeInputRef( protocolId=protocolId, inputName=inputNameText, itemIndex=int(itemIndex), @@ -498,13 +513,13 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: exc_info=True, ) - runtimeParamsByProtocolId[protocolId] = extractRuntimeParams(protocol) + runtimeParamsByProtocolId[protocolId] = self.extractRuntimeParams(protocol) for parent in getattr(nodeObj, "_parents", []) or []: try: - parentId = normalizeProtocolId(parent.getName()) + parentId = self.normalizeProtocolId(parent.getName()) except Exception: - parentId = normalizeProtocolId(parent) + parentId = self.normalizeProtocolId(parent) if not parentId or parentId == "PROJECT": continue @@ -528,13 +543,13 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: postgresqlParamsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]] = {} for row in protocolRows: - protocolId = normalizeProtocolId(row.get("protocolId")) + protocolId = self.normalizeProtocolId(row.get("protocolId")) if not protocolId: continue - postgresqlStatuses[protocolId] = normalizeStatus(row.get("status")) - postgresqlClassNames[protocolId] = normalizeClassName(row.get("protocolClassName")) - postgresqlParamsByProtocolId[protocolId] = extractPostgresqlParams(row) + postgresqlStatuses[protocolId] = self.normalizeStatus(row.get("status")) + postgresqlClassNames[protocolId] = self.normalizeClassName(row.get("protocolClassName")) + postgresqlParamsByProtocolId[protocolId] = self.extractPostgresqlParams(row) try: adjacencyMap = mapper.getProjectProtocolAdjacencyMap(projectId) or {} @@ -550,12 +565,12 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: postgresqlDependencies: Set[Tuple[str, str]] = set() for childId, refs in adjacencyMap.items(): - childProtocolId = normalizeProtocolId(childId) + childProtocolId = self.normalizeProtocolId(childId) if not childProtocolId or childProtocolId == "PROJECT": continue for parentIdValue in refs.get("parents") or []: - parentProtocolId = normalizeProtocolId(parentIdValue) + parentProtocolId = self.normalizeProtocolId(parentIdValue) if not parentProtocolId or parentProtocolId == "PROJECT": continue @@ -590,16 +605,16 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: normalizedPostgresqlStepsByProtocolId: Dict[str, Dict[int, Dict[str, Any]]] = {} for protocolId, steps in postgresqlStepsByProtocolId.items(): - protocolIdText = normalizeProtocolId(protocolId) + protocolIdText = self.normalizeProtocolId(protocolId) for step in steps or []: - stepIndex = toOptionalInt(step.get("index")) + stepIndex = self.toOptionalInt(step.get("index")) if stepIndex is None: continue normalizedPostgresqlStepsByProtocolId.setdefault(protocolIdText, {})[stepIndex] = { "index": stepIndex, "name": str(step.get("name") or ""), - "status": normalizeStatus(step.get("status")), + "status": self.normalizeStatus(step.get("status")), } try: @@ -616,9 +631,9 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: postgresqlInputRefsByKey: Dict[Tuple[str, str, int], Dict[str, Any]] = {} for ref in postgresqlInputRefs: - protocolIdText = normalizeProtocolId(ref.get("protocolId")) + protocolIdText = self.normalizeProtocolId(ref.get("protocolId")) inputName = str(ref.get("inputName") or "").strip() - itemIndex = toOptionalInt(ref.get("itemIndex")) + itemIndex = self.toOptionalInt(ref.get("itemIndex")) if not protocolIdText or not inputName: continue @@ -631,10 +646,10 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: "protocolId": protocolIdText, "inputName": inputName, "itemIndex": int(itemIndex), - "parentProtocolId": normalizeOptionalText(ref.get("parentProtocolId")), - "parentOutputName": normalizeOptionalText(ref.get("parentOutputName")), - "objectClassName": normalizeOptionalText(ref.get("objectClassName")), - "objectId": normalizeOptionalText(ref.get("objectId")), + "parentProtocolId": self.normalizeOptionalText(ref.get("parentProtocolId")), + "parentOutputName": self.normalizeOptionalText(ref.get("parentOutputName")), + "objectClassName": self.normalizeOptionalText(ref.get("objectClassName")), + "objectId": self.normalizeOptionalText(ref.get("objectId")), } runtimeOutputs: Set[Tuple[str, str]] = set() @@ -645,7 +660,7 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: postgresqlOutputs: Set[Tuple[str, str]] = set() for protocolId, outputsByName in persistedOutputsByProtocolId.items(): for outputName in outputsByName.keys(): - postgresqlOutputs.add((normalizeProtocolId(protocolId), str(outputName))) + postgresqlOutputs.add((self.normalizeProtocolId(protocolId), str(outputName))) runtimeSteps: Set[Tuple[str, int]] = set() for protocolId, stepsByIndex in runtimeStepsByProtocolId.items(): @@ -674,28 +689,28 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: postgresqlParams.add((protocolId, paramName)) for inputRef in runtimeInputRefsByKey.values(): - dependencyKey = dependencyKeyFromInputRef(inputRef) + dependencyKey = self.dependencyKeyFromInputRef(inputRef) if dependencyKey is not None: runtimeDependenciesFromInputRefs.add(dependencyKey) postgresqlDependenciesFromInputRefs: Set[Tuple[str, str]] = set() for inputRef in postgresqlInputRefsByKey.values(): - dependencyKey = dependencyKeyFromInputRef(inputRef) + dependencyKey = self.dependencyKeyFromInputRef(inputRef) if dependencyKey is not None: postgresqlDependenciesFromInputRefs.add(dependencyKey) missingProtocolIds = sorted( runtimeProtocolIds - postgresqlProtocolIds, - key=protocolSortKey, + key=self.protocolSortKey, ) extraProtocolIds = sorted( postgresqlProtocolIds - runtimeProtocolIds, - key=protocolSortKey, + key=self.protocolSortKey, ) commonProtocolIds = sorted( runtimeProtocolIds.intersection(postgresqlProtocolIds), - key=protocolSortKey, + key=self.protocolSortKey, ) statusMismatches = [] @@ -712,8 +727,8 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: protocolClassMismatches = [] for protocolId in commonProtocolIds: - runtimeClassName = normalizeClassName(runtimeClassNames.get(protocolId)) - postgresqlClassName = normalizeClassName(postgresqlClassNames.get(protocolId)) + runtimeClassName = self.normalizeClassName(runtimeClassNames.get(protocolId)) + postgresqlClassName = self.normalizeClassName(postgresqlClassNames.get(protocolId)) if ( runtimeClassName @@ -728,29 +743,29 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: missingDependencies = sorted( runtimeDependencies - postgresqlDependencies, - key=dependencySortKey, + key=self.dependencySortKey, ) extraDependencies = sorted( postgresqlDependencies - runtimeDependencies, - key=dependencySortKey, + key=self.dependencySortKey, ) missingOutputs = sorted( runtimeOutputs - postgresqlOutputs, - key=dependencySortKey, + key=self.dependencySortKey, ) extraOutputs = sorted( postgresqlOutputs - runtimeOutputs, - key=dependencySortKey, + key=self.dependencySortKey, ) outputClassMismatches = [] - for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=dependencySortKey): + for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=self.dependencySortKey): runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - runtimeClassName = normalizeOptionalText(runtimeOutput.get("className")) - postgresqlClassName = normalizeOptionalText(postgresqlOutput.get("className")) + runtimeClassName = self.normalizeOptionalText(runtimeOutput.get("className")) + postgresqlClassName = self.normalizeOptionalText(postgresqlOutput.get("className")) if ( runtimeClassName is not None @@ -766,13 +781,13 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: }) outputMapperKindMismatches = [] - for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=dependencySortKey): + for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=self.dependencySortKey): runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - runtimeClassName = normalizeOptionalText(runtimeOutput.get("className")) - expectedMapperKind = expectedOutputMapperKind(runtimeClassName) - postgresqlMapperKind = normalizeOptionalText(postgresqlOutput.get("mapperKind")) + runtimeClassName = self.normalizeOptionalText(runtimeOutput.get("className")) + expectedMapperKind = self.expectedOutputMapperKind(runtimeClassName) + postgresqlMapperKind = self.normalizeOptionalText(postgresqlOutput.get("mapperKind")) if ( expectedMapperKind is not None @@ -788,16 +803,16 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: }) outputItemsCountMismatches = [] - for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=dependencySortKey): + for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=self.dependencySortKey): runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - runtimeClassName = normalizeOptionalText(runtimeOutput.get("className")) - if expectedOutputMapperKind(runtimeClassName) != "flat_set": + runtimeClassName = self.normalizeOptionalText(runtimeOutput.get("className")) + if self.expectedOutputMapperKind(runtimeClassName) != "flat_set": continue - runtimeItemsCount = toOptionalInt(runtimeOutput.get("itemsCount")) - postgresqlItemsCount = toOptionalInt(postgresqlOutput.get("itemsCount")) + runtimeItemsCount = self.toOptionalInt(runtimeOutput.get("itemsCount")) + postgresqlItemsCount = self.toOptionalInt(postgresqlOutput.get("itemsCount")) if runtimeItemsCount is None or postgresqlItemsCount is None: continue @@ -814,22 +829,22 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: missingSteps = sorted( runtimeSteps - postgresqlSteps, - key=stepSortKey, + key=self.stepSortKey, ) extraSteps = sorted( postgresqlSteps - runtimeSteps, - key=stepSortKey, + key=self.stepSortKey, ) stepMismatches = [] - for protocolId, stepIndex in sorted(runtimeSteps.intersection(postgresqlSteps), key=stepSortKey): + for protocolId, stepIndex in sorted(runtimeSteps.intersection(postgresqlSteps), key=self.stepSortKey): runtimeStep = runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) postgresqlStep = normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) runtimeName = str(runtimeStep.get("name") or "") postgresqlName = str(postgresqlStep.get("name") or "") - runtimeStatus = normalizeStatus(runtimeStep.get("status")) - postgresqlStatus = normalizeStatus(postgresqlStep.get("status")) + runtimeStatus = self.normalizeStatus(runtimeStep.get("status")) + postgresqlStatus = self.normalizeStatus(postgresqlStep.get("status")) changedFields = [] if runtimeName != postgresqlName: @@ -851,31 +866,31 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: missingInputRefs = sorted( runtimeInputRefs - postgresqlInputRefsKeys, - key=inputRefSortKey, + key=self.inputRefSortKey, ) extraInputRefs = sorted( postgresqlInputRefsKeys - runtimeInputRefs, - key=inputRefSortKey, + key=self.inputRefSortKey, ) missingParams = sorted( runtimeParams - postgresqlParams, - key=paramSortKey, + key=self.paramSortKey, ) extraParams = sorted( postgresqlParams - runtimeParams, - key=paramSortKey, + key=self.paramSortKey, ) paramValueMismatches = [] - for key in sorted(runtimeParams.intersection(postgresqlParams), key=paramSortKey): + for key in sorted(runtimeParams.intersection(postgresqlParams), key=self.paramSortKey): protocolId, paramName = key runtimeParam = runtimeParamsByProtocolId.get(protocolId, {}).get(paramName, {}) postgresqlParam = postgresqlParamsByProtocolId.get(protocolId, {}).get(paramName, {}) - runtimeValue = normalizeParamValue(runtimeParam.get("value")) - postgresqlValue = normalizeParamValue(postgresqlParam.get("value")) + runtimeValue = self.normalizeParamValue(runtimeParam.get("value")) + postgresqlValue = self.normalizeParamValue(postgresqlParam.get("value")) if runtimeValue != postgresqlValue: paramValueMismatches.append({ @@ -886,20 +901,20 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: }) inputRefMismatches = [] - for key in sorted(runtimeInputRefs.intersection(postgresqlInputRefsKeys), key=inputRefSortKey): + for key in sorted(runtimeInputRefs.intersection(postgresqlInputRefsKeys), key=self.inputRefSortKey): runtimeRef = runtimeInputRefsByKey.get(key, {}) postgresqlRef = postgresqlInputRefsByKey.get(key, {}) changedFields = [] - runtimeParentProtocolId = normalizeOptionalText(runtimeRef.get("parentProtocolId")) - postgresqlParentProtocolId = normalizeOptionalText(postgresqlRef.get("parentProtocolId")) + runtimeParentProtocolId = self.normalizeOptionalText(runtimeRef.get("parentProtocolId")) + postgresqlParentProtocolId = self.normalizeOptionalText(postgresqlRef.get("parentProtocolId")) - runtimeParentOutputName = normalizeOptionalText(runtimeRef.get("parentOutputName")) - postgresqlParentOutputName = normalizeOptionalText(postgresqlRef.get("parentOutputName")) + runtimeParentOutputName = self.normalizeOptionalText(runtimeRef.get("parentOutputName")) + postgresqlParentOutputName = self.normalizeOptionalText(postgresqlRef.get("parentOutputName")) - runtimeObjectClassName = normalizeOptionalText(runtimeRef.get("objectClassName")) - postgresqlObjectClassName = normalizeOptionalText(postgresqlRef.get("objectClassName")) + runtimeObjectClassName = self.normalizeOptionalText(runtimeRef.get("objectClassName")) + postgresqlObjectClassName = self.normalizeOptionalText(postgresqlRef.get("objectClassName")) if runtimeParentProtocolId != postgresqlParentProtocolId: changedFields.append("parentProtocolId") @@ -931,20 +946,20 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: runtimeInputRefDependenciesMissing = sorted( runtimeDependenciesFromInputRefs - runtimeDependencies, - key=dependencySortKey, + key=self.dependencySortKey, ) runtimeDependenciesWithoutInputRefs = sorted( runtimeDependencies - runtimeDependenciesFromInputRefs, - key=dependencySortKey, + key=self.dependencySortKey, ) postgresqlInputRefDependenciesMissing = sorted( postgresqlDependenciesFromInputRefs - postgresqlDependencies, - key=dependencySortKey, + key=self.dependencySortKey, ) postgresqlDependenciesWithoutInputRefs = sorted( postgresqlDependencies - postgresqlDependenciesFromInputRefs, - key=dependencySortKey, + key=self.dependencySortKey, ) issues = { @@ -965,11 +980,11 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: "statusMismatches": statusMismatches, "protocolClassMismatches": protocolClassMismatches, "missingDependencies": [ - buildDependency(parentId, childId) + self.buildDependency(parentId, childId) for parentId, childId in missingDependencies ], "extraDependencies": [ - buildDependency(parentId, childId) + self.buildDependency(parentId, childId) for parentId, childId in extraDependencies ], "missingOutputs": [ @@ -1003,7 +1018,7 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: "outputItemsCountMismatches": outputItemsCountMismatches, "missingSteps": [ - buildStep( + self.buildStep( protocolId, stepIndex, runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}), @@ -1011,7 +1026,7 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: for protocolId, stepIndex in missingSteps ], "extraSteps": [ - buildStep( + self.buildStep( protocolId, stepIndex, normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}), @@ -1020,36 +1035,36 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: ], "stepMismatches": stepMismatches, "missingInputRefs": [ - buildInputRef(key, runtimeInputRefsByKey.get(key, {})) + self.buildInputRef(key, runtimeInputRefsByKey.get(key, {})) for key in missingInputRefs ], "extraInputRefs": [ - buildInputRef(key, postgresqlInputRefsByKey.get(key, {})) + self.buildInputRef(key, postgresqlInputRefsByKey.get(key, {})) for key in extraInputRefs ], "inputRefMismatches": inputRefMismatches, "runtimeInputRefDependenciesMissing": [ - buildDependency(parentId, childId) + self.buildDependency(parentId, childId) for parentId, childId in runtimeInputRefDependenciesMissing ], "runtimeDependenciesWithoutInputRefs": [ - buildDependency(parentId, childId) + self.buildDependency(parentId, childId) for parentId, childId in runtimeDependenciesWithoutInputRefs ], "postgresqlInputRefDependenciesMissing": [ - buildDependency(parentId, childId) + self.buildDependency(parentId, childId) for parentId, childId in postgresqlInputRefDependenciesMissing ], "postgresqlDependenciesWithoutInputRefs": [ - buildDependency(parentId, childId) + self.buildDependency(parentId, childId) for parentId, childId in postgresqlDependenciesWithoutInputRefs ], "missingParams": [ - buildParamIssue(key, runtimeParamsByProtocolId.get(key[0], {}).get(key[1], {})) + self.buildParamIssue(key, runtimeParamsByProtocolId.get(key[0], {}).get(key[1], {})) for key in missingParams ], "extraParams": [ - buildParamIssue(key, postgresqlParamsByProtocolId.get(key[0], {}).get(key[1], {})) + self.buildParamIssue(key, postgresqlParamsByProtocolId.get(key[0], {}).get(key[1], {})) for key in extraParams ], "paramValueMismatches": paramValueMismatches, @@ -1080,4 +1095,3 @@ def extractPostgresqlParams(row: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: }, "issues": issues, } - From fcbfc4be08984e2a1da43f19da9bf2a0c8f5344c Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 23:01:18 +0200 Subject: [PATCH 183/278] Extract runtime snapshot collection --- .../services/project_consistency_service.py | 60 ++++++++++++++----- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index acbe8b45..39e39172 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -376,23 +376,12 @@ def extractPostgresqlParams(self, row: Dict[str, Any]) -> Dict[str, Dict[str, An return paramsByName - def validateProjectPostgresqlConsistency( + def collectRuntimeSnapshot( self, - mapper: PostgresqlFlatMapper, projectId: int, - currentUser: dict, - refresh: bool = True, - checkPid: bool = True, + refresh: bool, + checkPid: bool, ) -> Dict[str, Any]: - dbProj = self.getProjectDbRow(mapper, projectId, currentUser) - if not dbProj: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Project not found", - ) - - self.loadProjectForThumbnails(dbProj) - runtimeStatuses: Dict[str, str] = {} runtimeClassNames: Dict[str, str] = {} runtimeDependencies: Set[Tuple[str, str]] = set() @@ -461,6 +450,7 @@ def validateProjectPostgresqlConsistency( protocolId, exc_info=True, ) + runtimeStepsByProtocolId.setdefault(protocolId, {}) try: @@ -482,6 +472,7 @@ def validateProjectPostgresqlConsistency( protocolId, exc_info=True, ) + try: for inputName, attr in protocol.iterInputAttributes(): inputNameText = str(inputName or "").strip() @@ -526,6 +517,47 @@ def validateProjectPostgresqlConsistency( runtimeDependencies.add((parentId, protocolId)) + return { + "statuses": runtimeStatuses, + "classNames": runtimeClassNames, + "dependencies": runtimeDependencies, + "outputsByProtocolId": runtimeOutputsByProtocolId, + "stepsByProtocolId": runtimeStepsByProtocolId, + "inputRefsByKey": runtimeInputRefsByKey, + "paramsByProtocolId": runtimeParamsByProtocolId, + } + + def validateProjectPostgresqlConsistency( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser: dict, + refresh: bool = True, + checkPid: bool = True, + ) -> Dict[str, Any]: + dbProj = self.getProjectDbRow(mapper, projectId, currentUser) + if not dbProj: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Project not found", + ) + + self.loadProjectForThumbnails(dbProj) + + runtimeSnapshot = self.collectRuntimeSnapshot( + projectId=projectId, + refresh=refresh, + checkPid=checkPid, + ) + + runtimeStatuses = runtimeSnapshot["statuses"] + runtimeClassNames = runtimeSnapshot["classNames"] + runtimeDependencies = runtimeSnapshot["dependencies"] + runtimeOutputsByProtocolId = runtimeSnapshot["outputsByProtocolId"] + runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] + runtimeInputRefsByKey = runtimeSnapshot["inputRefsByKey"] + runtimeParamsByProtocolId = runtimeSnapshot["paramsByProtocolId"] + try: protocolRows = mapper.getProtocols(projectId) or [] except Exception as e: From e9b7661d417aa1c5efbc146f90b277441db274b4 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Thu, 25 Jun 2026 23:08:37 +0200 Subject: [PATCH 184/278] Extract PostgreSQL snapshot collection --- .../services/project_consistency_service.py | 82 +++++++++++++------ 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 39e39172..cb81da04 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -527,37 +527,11 @@ def collectRuntimeSnapshot( "paramsByProtocolId": runtimeParamsByProtocolId, } - def validateProjectPostgresqlConsistency( + def collectPostgresqlSnapshot( self, mapper: PostgresqlFlatMapper, projectId: int, - currentUser: dict, - refresh: bool = True, - checkPid: bool = True, ) -> Dict[str, Any]: - dbProj = self.getProjectDbRow(mapper, projectId, currentUser) - if not dbProj: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Project not found", - ) - - self.loadProjectForThumbnails(dbProj) - - runtimeSnapshot = self.collectRuntimeSnapshot( - projectId=projectId, - refresh=refresh, - checkPid=checkPid, - ) - - runtimeStatuses = runtimeSnapshot["statuses"] - runtimeClassNames = runtimeSnapshot["classNames"] - runtimeDependencies = runtimeSnapshot["dependencies"] - runtimeOutputsByProtocolId = runtimeSnapshot["outputsByProtocolId"] - runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] - runtimeInputRefsByKey = runtimeSnapshot["inputRefsByKey"] - runtimeParamsByProtocolId = runtimeSnapshot["paramsByProtocolId"] - try: protocolRows = mapper.getProtocols(projectId) or [] except Exception as e: @@ -684,6 +658,60 @@ def validateProjectPostgresqlConsistency( "objectId": self.normalizeOptionalText(ref.get("objectId")), } + return { + "statuses": postgresqlStatuses, + "classNames": postgresqlClassNames, + "dependencies": postgresqlDependencies, + "outputsByProtocolId": persistedOutputsByProtocolId, + "stepsByProtocolId": normalizedPostgresqlStepsByProtocolId, + "inputRefsByKey": postgresqlInputRefsByKey, + "paramsByProtocolId": postgresqlParamsByProtocolId, + } + + def validateProjectPostgresqlConsistency( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser: dict, + refresh: bool = True, + checkPid: bool = True, + ) -> Dict[str, Any]: + dbProj = self.getProjectDbRow(mapper, projectId, currentUser) + if not dbProj: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Project not found", + ) + + self.loadProjectForThumbnails(dbProj) + + runtimeSnapshot = self.collectRuntimeSnapshot( + projectId=projectId, + refresh=refresh, + checkPid=checkPid, + ) + + runtimeStatuses = runtimeSnapshot["statuses"] + runtimeClassNames = runtimeSnapshot["classNames"] + runtimeDependencies = runtimeSnapshot["dependencies"] + runtimeOutputsByProtocolId = runtimeSnapshot["outputsByProtocolId"] + runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] + runtimeInputRefsByKey = runtimeSnapshot["inputRefsByKey"] + runtimeParamsByProtocolId = runtimeSnapshot["paramsByProtocolId"] + + postgresqlSnapshot = self.collectPostgresqlSnapshot( + mapper=mapper, + projectId=projectId, + ) + + postgresqlStatuses = postgresqlSnapshot["statuses"] + postgresqlClassNames = postgresqlSnapshot["classNames"] + postgresqlDependencies = postgresqlSnapshot["dependencies"] + persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] + normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] + postgresqlInputRefsByKey = postgresqlSnapshot["inputRefsByKey"] + postgresqlParamsByProtocolId = postgresqlSnapshot["paramsByProtocolId"] + runtimeOutputs: Set[Tuple[str, str]] = set() for protocolId, outputsByName in runtimeOutputsByProtocolId.items(): for outputName in outputsByName.keys(): From 52378217932f994f85b627390b028febec196499 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 00:17:51 +0200 Subject: [PATCH 185/278] Extract consistency derived sets builder --- .../services/project_consistency_service.py | 112 +++++++++++++----- 1 file changed, 82 insertions(+), 30 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index cb81da04..0450db0d 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -668,44 +668,19 @@ def collectPostgresqlSnapshot( "paramsByProtocolId": postgresqlParamsByProtocolId, } - def validateProjectPostgresqlConsistency( + def buildDerivedSets( self, - mapper: PostgresqlFlatMapper, - projectId: int, - currentUser: dict, - refresh: bool = True, - checkPid: bool = True, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], ) -> Dict[str, Any]: - dbProj = self.getProjectDbRow(mapper, projectId, currentUser) - if not dbProj: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Project not found", - ) - - self.loadProjectForThumbnails(dbProj) - - runtimeSnapshot = self.collectRuntimeSnapshot( - projectId=projectId, - refresh=refresh, - checkPid=checkPid, - ) - runtimeStatuses = runtimeSnapshot["statuses"] - runtimeClassNames = runtimeSnapshot["classNames"] runtimeDependencies = runtimeSnapshot["dependencies"] runtimeOutputsByProtocolId = runtimeSnapshot["outputsByProtocolId"] runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] runtimeInputRefsByKey = runtimeSnapshot["inputRefsByKey"] runtimeParamsByProtocolId = runtimeSnapshot["paramsByProtocolId"] - postgresqlSnapshot = self.collectPostgresqlSnapshot( - mapper=mapper, - projectId=projectId, - ) - postgresqlStatuses = postgresqlSnapshot["statuses"] - postgresqlClassNames = postgresqlSnapshot["classNames"] postgresqlDependencies = postgresqlSnapshot["dependencies"] persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] @@ -736,7 +711,6 @@ def validateProjectPostgresqlConsistency( postgresqlProtocolIds = set(postgresqlStatuses.keys()) runtimeInputRefs = set(runtimeInputRefsByKey.keys()) postgresqlInputRefsKeys = set(postgresqlInputRefsByKey.keys()) - runtimeDependenciesFromInputRefs: Set[Tuple[str, str]] = set() runtimeParams: Set[Tuple[str, str]] = set() for protocolId, paramsByName in runtimeParamsByProtocolId.items(): @@ -748,6 +722,7 @@ def validateProjectPostgresqlConsistency( for paramName in paramsByName.keys(): postgresqlParams.add((protocolId, paramName)) + runtimeDependenciesFromInputRefs: Set[Tuple[str, str]] = set() for inputRef in runtimeInputRefsByKey.values(): dependencyKey = self.dependencyKeyFromInputRef(inputRef) if dependencyKey is not None: @@ -759,6 +734,83 @@ def validateProjectPostgresqlConsistency( if dependencyKey is not None: postgresqlDependenciesFromInputRefs.add(dependencyKey) + return { + "runtimeOutputs": runtimeOutputs, + "postgresqlOutputs": postgresqlOutputs, + "runtimeSteps": runtimeSteps, + "postgresqlSteps": postgresqlSteps, + "runtimeProtocolIds": runtimeProtocolIds, + "postgresqlProtocolIds": postgresqlProtocolIds, + "runtimeInputRefs": runtimeInputRefs, + "postgresqlInputRefsKeys": postgresqlInputRefsKeys, + "runtimeParams": runtimeParams, + "postgresqlParams": postgresqlParams, + "runtimeDependenciesFromInputRefs": runtimeDependenciesFromInputRefs, + "postgresqlDependenciesFromInputRefs": postgresqlDependenciesFromInputRefs, + } + + def validateProjectPostgresqlConsistency( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser: dict, + refresh: bool = True, + checkPid: bool = True, + ) -> Dict[str, Any]: + dbProj = self.getProjectDbRow(mapper, projectId, currentUser) + if not dbProj: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Project not found", + ) + + self.loadProjectForThumbnails(dbProj) + + runtimeSnapshot = self.collectRuntimeSnapshot( + projectId=projectId, + refresh=refresh, + checkPid=checkPid, + ) + + runtimeStatuses = runtimeSnapshot["statuses"] + runtimeClassNames = runtimeSnapshot["classNames"] + runtimeDependencies = runtimeSnapshot["dependencies"] + runtimeOutputsByProtocolId = runtimeSnapshot["outputsByProtocolId"] + runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] + runtimeInputRefsByKey = runtimeSnapshot["inputRefsByKey"] + runtimeParamsByProtocolId = runtimeSnapshot["paramsByProtocolId"] + + postgresqlSnapshot = self.collectPostgresqlSnapshot( + mapper=mapper, + projectId=projectId, + ) + + postgresqlStatuses = postgresqlSnapshot["statuses"] + postgresqlClassNames = postgresqlSnapshot["classNames"] + postgresqlDependencies = postgresqlSnapshot["dependencies"] + persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] + normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] + postgresqlInputRefsByKey = postgresqlSnapshot["inputRefsByKey"] + postgresqlParamsByProtocolId = postgresqlSnapshot["paramsByProtocolId"] + + derivedSets = self.buildDerivedSets( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + ) + + runtimeOutputs = derivedSets["runtimeOutputs"] + postgresqlOutputs = derivedSets["postgresqlOutputs"] + runtimeSteps = derivedSets["runtimeSteps"] + postgresqlSteps = derivedSets["postgresqlSteps"] + runtimeProtocolIds = derivedSets["runtimeProtocolIds"] + postgresqlProtocolIds = derivedSets["postgresqlProtocolIds"] + runtimeInputRefs = derivedSets["runtimeInputRefs"] + postgresqlInputRefsKeys = derivedSets["postgresqlInputRefsKeys"] + runtimeParams = derivedSets["runtimeParams"] + postgresqlParams = derivedSets["postgresqlParams"] + runtimeDependenciesFromInputRefs = derivedSets["runtimeDependenciesFromInputRefs"] + postgresqlDependenciesFromInputRefs = derivedSets["postgresqlDependenciesFromInputRefs"] + missingProtocolIds = sorted( runtimeProtocolIds - postgresqlProtocolIds, key=self.protocolSortKey, @@ -1154,4 +1206,4 @@ def validateProjectPostgresqlConsistency( "issues": issuesCount, }, "issues": issues, - } + } \ No newline at end of file From 12e729674cd631226f677532c2d78546e019cc1a Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 00:27:22 +0200 Subject: [PATCH 186/278] Extract protocol consistency comparison --- .../services/project_consistency_service.py | 114 ++++++++++++------ 1 file changed, 74 insertions(+), 40 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 0450db0d..74da7e1f 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -749,6 +749,71 @@ def buildDerivedSets( "postgresqlDependenciesFromInputRefs": postgresqlDependenciesFromInputRefs, } + def compareProtocols( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeStatuses = runtimeSnapshot["statuses"] + runtimeClassNames = runtimeSnapshot["classNames"] + + postgresqlStatuses = postgresqlSnapshot["statuses"] + postgresqlClassNames = postgresqlSnapshot["classNames"] + + runtimeProtocolIds = derivedSets["runtimeProtocolIds"] + postgresqlProtocolIds = derivedSets["postgresqlProtocolIds"] + + missingProtocolIds = sorted( + runtimeProtocolIds - postgresqlProtocolIds, + key=self.protocolSortKey, + ) + extraProtocolIds = sorted( + postgresqlProtocolIds - runtimeProtocolIds, + key=self.protocolSortKey, + ) + + commonProtocolIds = sorted( + runtimeProtocolIds.intersection(postgresqlProtocolIds), + key=self.protocolSortKey, + ) + + statusMismatches = [] + for protocolId in commonProtocolIds: + runtimeStatus = runtimeStatuses.get(protocolId, "") + postgresqlStatus = postgresqlStatuses.get(protocolId, "") + + if runtimeStatus != postgresqlStatus: + statusMismatches.append({ + "protocolId": protocolId, + "runtimeStatus": runtimeStatus, + "postgresqlStatus": postgresqlStatus, + }) + + protocolClassMismatches = [] + for protocolId in commonProtocolIds: + runtimeClassName = self.normalizeClassName(runtimeClassNames.get(protocolId)) + postgresqlClassName = self.normalizeClassName(postgresqlClassNames.get(protocolId)) + + if ( + runtimeClassName + and postgresqlClassName + and runtimeClassName != postgresqlClassName + ): + protocolClassMismatches.append({ + "protocolId": protocolId, + "runtimeClassName": runtimeClassName, + "postgresqlClassName": postgresqlClassName, + }) + + return { + "missingProtocolIds": missingProtocolIds, + "extraProtocolIds": extraProtocolIds, + "commonProtocolIds": commonProtocolIds, + "statusMismatches": statusMismatches, + "protocolClassMismatches": protocolClassMismatches, + } + def validateProjectPostgresqlConsistency( self, mapper: PostgresqlFlatMapper, @@ -811,47 +876,17 @@ def validateProjectPostgresqlConsistency( runtimeDependenciesFromInputRefs = derivedSets["runtimeDependenciesFromInputRefs"] postgresqlDependenciesFromInputRefs = derivedSets["postgresqlDependenciesFromInputRefs"] - missingProtocolIds = sorted( - runtimeProtocolIds - postgresqlProtocolIds, - key=self.protocolSortKey, - ) - extraProtocolIds = sorted( - postgresqlProtocolIds - runtimeProtocolIds, - key=self.protocolSortKey, + protocolComparison = self.compareProtocols( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, ) - commonProtocolIds = sorted( - runtimeProtocolIds.intersection(postgresqlProtocolIds), - key=self.protocolSortKey, - ) - - statusMismatches = [] - for protocolId in commonProtocolIds: - runtimeStatus = runtimeStatuses.get(protocolId, "") - postgresqlStatus = postgresqlStatuses.get(protocolId, "") - - if runtimeStatus != postgresqlStatus: - statusMismatches.append({ - "protocolId": protocolId, - "runtimeStatus": runtimeStatus, - "postgresqlStatus": postgresqlStatus, - }) - - protocolClassMismatches = [] - for protocolId in commonProtocolIds: - runtimeClassName = self.normalizeClassName(runtimeClassNames.get(protocolId)) - postgresqlClassName = self.normalizeClassName(postgresqlClassNames.get(protocolId)) - - if ( - runtimeClassName - and postgresqlClassName - and runtimeClassName != postgresqlClassName - ): - protocolClassMismatches.append({ - "protocolId": protocolId, - "runtimeClassName": runtimeClassName, - "postgresqlClassName": postgresqlClassName, - }) + missingProtocolIds = protocolComparison["missingProtocolIds"] + extraProtocolIds = protocolComparison["extraProtocolIds"] + commonProtocolIds = protocolComparison["commonProtocolIds"] + statusMismatches = protocolComparison["statusMismatches"] + protocolClassMismatches = protocolComparison["protocolClassMismatches"] missingDependencies = sorted( runtimeDependencies - postgresqlDependencies, @@ -975,7 +1010,6 @@ def validateProjectPostgresqlConsistency( "postgresqlStatus": postgresqlStatus, }) - missingInputRefs = sorted( runtimeInputRefs - postgresqlInputRefsKeys, key=self.inputRefSortKey, From 97027b8c017f884ee48e0ccee92d08d41c48db33 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 00:33:05 +0200 Subject: [PATCH 187/278] Extract dependency consistency comparison --- .../services/project_consistency_service.py | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 74da7e1f..c1b9602e 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -814,6 +814,28 @@ def compareProtocols( "protocolClassMismatches": protocolClassMismatches, } + def compareDependencies( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeDependencies = runtimeSnapshot["dependencies"] + postgresqlDependencies = postgresqlSnapshot["dependencies"] + + missingDependencies = sorted( + runtimeDependencies - postgresqlDependencies, + key=self.dependencySortKey, + ) + extraDependencies = sorted( + postgresqlDependencies - runtimeDependencies, + key=self.dependencySortKey, + ) + + return { + "missingDependencies": missingDependencies, + "extraDependencies": extraDependencies, + } + def validateProjectPostgresqlConsistency( self, mapper: PostgresqlFlatMapper, @@ -888,15 +910,14 @@ def validateProjectPostgresqlConsistency( statusMismatches = protocolComparison["statusMismatches"] protocolClassMismatches = protocolComparison["protocolClassMismatches"] - missingDependencies = sorted( - runtimeDependencies - postgresqlDependencies, - key=self.dependencySortKey, - ) - extraDependencies = sorted( - postgresqlDependencies - runtimeDependencies, - key=self.dependencySortKey, + dependencyComparison = self.compareDependencies( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, ) + missingDependencies = dependencyComparison["missingDependencies"] + extraDependencies = dependencyComparison["extraDependencies"] + missingOutputs = sorted( runtimeOutputs - postgresqlOutputs, key=self.dependencySortKey, From d75e9b76fcedc5b64e1d6cb4ac96d648f432a987 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 00:45:36 +0200 Subject: [PATCH 188/278] Extract output issue builders --- .../services/project_consistency_service.py | 72 +++++++++++++------ 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index c1b9602e..b2f58e45 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -145,6 +145,46 @@ def buildParamIssue( "value": payload.get("value"), } + def buildMissingOutputIssue( + self, + protocolId: Any, + outputName: Any, + runtimeOutputsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]], + ) -> Dict[str, Any]: + protocolIdText = self.normalizeProtocolId(protocolId) + outputNameText = str(outputName) + + return { + "protocolId": protocolIdText, + "outputName": outputNameText, + "className": runtimeOutputsByProtocolId + .get(protocolIdText, {}) + .get(outputNameText, {}) + .get("className"), + } + + def buildExtraOutputIssue( + self, + protocolId: Any, + outputName: Any, + persistedOutputsByProtocolId: Dict[str, Dict[str, Dict[str, Any]]], + ) -> Dict[str, Any]: + protocolIdText = self.normalizeProtocolId(protocolId) + outputNameText = str(outputName) + + return { + "protocolId": protocolIdText, + "outputName": outputNameText, + "mapperKind": persistedOutputsByProtocolId + .get(protocolIdText, {}) + .get(outputNameText, {}) + .get("mapperKind"), + "className": persistedOutputsByProtocolId + .get(protocolIdText, {}) + .get(outputNameText, {}) + .get("className"), + } + def expectedOutputMapperKind(self, className: Any) -> Optional[str]: classNameText = self.normalizeOptionalText(className) if classNameText is None: @@ -1154,30 +1194,20 @@ def validateProjectPostgresqlConsistency( self.buildDependency(parentId, childId) for parentId, childId in extraDependencies ], - "missingOutputs": [ - { - "protocolId": protocolId, - "outputName": outputName, - "className": runtimeOutputsByProtocolId - .get(protocolId, {}) - .get(outputName, {}) - .get("className"), - } + "missingOutputs": [ + self.buildMissingOutputIssue( + protocolId, + outputName, + runtimeOutputsByProtocolId, + ) for protocolId, outputName in missingOutputs ], "extraOutputs": [ - { - "protocolId": protocolId, - "outputName": outputName, - "mapperKind": persistedOutputsByProtocolId - .get(protocolId, {}) - .get(outputName, {}) - .get("mapperKind"), - "className": persistedOutputsByProtocolId - .get(protocolId, {}) - .get(outputName, {}) - .get("className"), - } + self.buildExtraOutputIssue( + protocolId, + outputName, + persistedOutputsByProtocolId, + ) for protocolId, outputName in extraOutputs ], "outputClassMismatches": outputClassMismatches, From de3bec143b879260efda511bc52fb6cd0ad751de Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 00:53:43 +0200 Subject: [PATCH 189/278] Extract output consistency comparison --- .../services/project_consistency_service.py | 185 +++++++++++------- 1 file changed, 111 insertions(+), 74 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index b2f58e45..5a91c0f4 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -876,6 +876,108 @@ def compareDependencies( "extraDependencies": extraDependencies, } + def compareOutputs( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeOutputsByProtocolId = runtimeSnapshot["outputsByProtocolId"] + persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] + + runtimeOutputs = derivedSets["runtimeOutputs"] + postgresqlOutputs = derivedSets["postgresqlOutputs"] + + commonOutputs = sorted( + runtimeOutputs.intersection(postgresqlOutputs), + key=self.dependencySortKey, + ) + + missingOutputs = sorted( + runtimeOutputs - postgresqlOutputs, + key=self.dependencySortKey, + ) + extraOutputs = sorted( + postgresqlOutputs - runtimeOutputs, + key=self.dependencySortKey, + ) + + outputClassMismatches = [] + for protocolId, outputName in commonOutputs: + runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + + runtimeClassName = self.normalizeOptionalText(runtimeOutput.get("className")) + postgresqlClassName = self.normalizeOptionalText(postgresqlOutput.get("className")) + + if ( + runtimeClassName is not None + and postgresqlClassName is not None + and runtimeClassName != postgresqlClassName + ): + outputClassMismatches.append({ + "protocolId": protocolId, + "outputName": outputName, + "runtimeClassName": runtimeClassName, + "postgresqlClassName": postgresqlClassName, + "mapperKind": postgresqlOutput.get("mapperKind"), + }) + + outputMapperKindMismatches = [] + for protocolId, outputName in commonOutputs: + runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + + runtimeClassName = self.normalizeOptionalText(runtimeOutput.get("className")) + expectedMapperKind = self.expectedOutputMapperKind(runtimeClassName) + postgresqlMapperKind = self.normalizeOptionalText(postgresqlOutput.get("mapperKind")) + + if ( + expectedMapperKind is not None + and postgresqlMapperKind is not None + and expectedMapperKind != postgresqlMapperKind + ): + outputMapperKindMismatches.append({ + "protocolId": protocolId, + "outputName": outputName, + "className": runtimeClassName, + "expectedMapperKind": expectedMapperKind, + "postgresqlMapperKind": postgresqlMapperKind, + }) + + outputItemsCountMismatches = [] + for protocolId, outputName in commonOutputs: + runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) + + runtimeClassName = self.normalizeOptionalText(runtimeOutput.get("className")) + if self.expectedOutputMapperKind(runtimeClassName) != "flat_set": + continue + + runtimeItemsCount = self.toOptionalInt(runtimeOutput.get("itemsCount")) + postgresqlItemsCount = self.toOptionalInt(postgresqlOutput.get("itemsCount")) + + if runtimeItemsCount is None or postgresqlItemsCount is None: + continue + + if runtimeItemsCount != postgresqlItemsCount: + outputItemsCountMismatches.append({ + "protocolId": protocolId, + "outputName": outputName, + "className": runtimeClassName, + "runtimeItemsCount": runtimeItemsCount, + "postgresqlItemsCount": postgresqlItemsCount, + "mapperKind": postgresqlOutput.get("mapperKind"), + }) + + return { + "missingOutputs": missingOutputs, + "extraOutputs": extraOutputs, + "outputClassMismatches": outputClassMismatches, + "outputMapperKindMismatches": outputMapperKindMismatches, + "outputItemsCountMismatches": outputItemsCountMismatches, + } + def validateProjectPostgresqlConsistency( self, mapper: PostgresqlFlatMapper, @@ -958,82 +1060,17 @@ def validateProjectPostgresqlConsistency( missingDependencies = dependencyComparison["missingDependencies"] extraDependencies = dependencyComparison["extraDependencies"] - missingOutputs = sorted( - runtimeOutputs - postgresqlOutputs, - key=self.dependencySortKey, - ) - extraOutputs = sorted( - postgresqlOutputs - runtimeOutputs, - key=self.dependencySortKey, + outputComparison = self.compareOutputs( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, ) - outputClassMismatches = [] - for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=self.dependencySortKey): - runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - - runtimeClassName = self.normalizeOptionalText(runtimeOutput.get("className")) - postgresqlClassName = self.normalizeOptionalText(postgresqlOutput.get("className")) - - if ( - runtimeClassName is not None - and postgresqlClassName is not None - and runtimeClassName != postgresqlClassName - ): - outputClassMismatches.append({ - "protocolId": protocolId, - "outputName": outputName, - "runtimeClassName": runtimeClassName, - "postgresqlClassName": postgresqlClassName, - "mapperKind": postgresqlOutput.get("mapperKind"), - }) - - outputMapperKindMismatches = [] - for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=self.dependencySortKey): - runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - - runtimeClassName = self.normalizeOptionalText(runtimeOutput.get("className")) - expectedMapperKind = self.expectedOutputMapperKind(runtimeClassName) - postgresqlMapperKind = self.normalizeOptionalText(postgresqlOutput.get("mapperKind")) - - if ( - expectedMapperKind is not None - and postgresqlMapperKind is not None - and expectedMapperKind != postgresqlMapperKind - ): - outputMapperKindMismatches.append({ - "protocolId": protocolId, - "outputName": outputName, - "className": runtimeClassName, - "expectedMapperKind": expectedMapperKind, - "postgresqlMapperKind": postgresqlMapperKind, - }) - - outputItemsCountMismatches = [] - for protocolId, outputName in sorted(runtimeOutputs.intersection(postgresqlOutputs), key=self.dependencySortKey): - runtimeOutput = runtimeOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - postgresqlOutput = persistedOutputsByProtocolId.get(protocolId, {}).get(outputName, {}) - - runtimeClassName = self.normalizeOptionalText(runtimeOutput.get("className")) - if self.expectedOutputMapperKind(runtimeClassName) != "flat_set": - continue - - runtimeItemsCount = self.toOptionalInt(runtimeOutput.get("itemsCount")) - postgresqlItemsCount = self.toOptionalInt(postgresqlOutput.get("itemsCount")) - - if runtimeItemsCount is None or postgresqlItemsCount is None: - continue - - if runtimeItemsCount != postgresqlItemsCount: - outputItemsCountMismatches.append({ - "protocolId": protocolId, - "outputName": outputName, - "className": runtimeClassName, - "runtimeItemsCount": runtimeItemsCount, - "postgresqlItemsCount": postgresqlItemsCount, - "mapperKind": postgresqlOutput.get("mapperKind"), - }) + missingOutputs = outputComparison["missingOutputs"] + extraOutputs = outputComparison["extraOutputs"] + outputClassMismatches = outputComparison["outputClassMismatches"] + outputMapperKindMismatches = outputComparison["outputMapperKindMismatches"] + outputItemsCountMismatches = outputComparison["outputItemsCountMismatches"] missingSteps = sorted( runtimeSteps - postgresqlSteps, From 8f09ce88e1643379aa1b86822f04bfe259e9afe7 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 01:17:38 +0200 Subject: [PATCH 190/278] Extract step consistency comparison --- .../services/project_consistency_service.py | 94 ++++++++++++------- 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 5a91c0f4..28fe52da 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -978,6 +978,60 @@ def compareOutputs( "outputItemsCountMismatches": outputItemsCountMismatches, } + def compareSteps( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] + normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] + + runtimeSteps = derivedSets["runtimeSteps"] + postgresqlSteps = derivedSets["postgresqlSteps"] + + missingSteps = sorted( + runtimeSteps - postgresqlSteps, + key=self.stepSortKey, + ) + extraSteps = sorted( + postgresqlSteps - runtimeSteps, + key=self.stepSortKey, + ) + + stepMismatches = [] + for protocolId, stepIndex in sorted(runtimeSteps.intersection(postgresqlSteps), key=self.stepSortKey): + runtimeStep = runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) + postgresqlStep = normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) + + runtimeName = str(runtimeStep.get("name") or "") + postgresqlName = str(postgresqlStep.get("name") or "") + runtimeStatus = self.normalizeStatus(runtimeStep.get("status")) + postgresqlStatus = self.normalizeStatus(postgresqlStep.get("status")) + + changedFields = [] + if runtimeName != postgresqlName: + changedFields.append("name") + if runtimeStatus != postgresqlStatus: + changedFields.append("status") + + if changedFields: + stepMismatches.append({ + "protocolId": protocolId, + "index": int(stepIndex), + "fields": changedFields, + "runtimeName": runtimeName, + "postgresqlName": postgresqlName, + "runtimeStatus": runtimeStatus, + "postgresqlStatus": postgresqlStatus, + }) + + return { + "missingSteps": missingSteps, + "extraSteps": extraSteps, + "stepMismatches": stepMismatches, + } + def validateProjectPostgresqlConsistency( self, mapper: PostgresqlFlatMapper, @@ -1072,41 +1126,15 @@ def validateProjectPostgresqlConsistency( outputMapperKindMismatches = outputComparison["outputMapperKindMismatches"] outputItemsCountMismatches = outputComparison["outputItemsCountMismatches"] - missingSteps = sorted( - runtimeSteps - postgresqlSteps, - key=self.stepSortKey, - ) - extraSteps = sorted( - postgresqlSteps - runtimeSteps, - key=self.stepSortKey, + stepComparison = self.compareSteps( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, ) - stepMismatches = [] - for protocolId, stepIndex in sorted(runtimeSteps.intersection(postgresqlSteps), key=self.stepSortKey): - runtimeStep = runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) - postgresqlStep = normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) - - runtimeName = str(runtimeStep.get("name") or "") - postgresqlName = str(postgresqlStep.get("name") or "") - runtimeStatus = self.normalizeStatus(runtimeStep.get("status")) - postgresqlStatus = self.normalizeStatus(postgresqlStep.get("status")) - - changedFields = [] - if runtimeName != postgresqlName: - changedFields.append("name") - if runtimeStatus != postgresqlStatus: - changedFields.append("status") - - if changedFields: - stepMismatches.append({ - "protocolId": protocolId, - "index": int(stepIndex), - "fields": changedFields, - "runtimeName": runtimeName, - "postgresqlName": postgresqlName, - "runtimeStatus": runtimeStatus, - "postgresqlStatus": postgresqlStatus, - }) + missingSteps = stepComparison["missingSteps"] + extraSteps = stepComparison["extraSteps"] + stepMismatches = stepComparison["stepMismatches"] missingInputRefs = sorted( runtimeInputRefs - postgresqlInputRefsKeys, From 3f4725a9e30b0662c84478982334ec0c63b82f16 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 01:25:44 +0200 Subject: [PATCH 191/278] Extract parameter consistency comparison --- .../services/project_consistency_service.py | 76 +++++++++++++------ 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 28fe52da..4ac5df37 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1032,6 +1032,51 @@ def compareSteps( "stepMismatches": stepMismatches, } + def compareParams( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeParamsByProtocolId = runtimeSnapshot["paramsByProtocolId"] + postgresqlParamsByProtocolId = postgresqlSnapshot["paramsByProtocolId"] + + runtimeParams = derivedSets["runtimeParams"] + postgresqlParams = derivedSets["postgresqlParams"] + + missingParams = sorted( + runtimeParams - postgresqlParams, + key=self.paramSortKey, + ) + extraParams = sorted( + postgresqlParams - runtimeParams, + key=self.paramSortKey, + ) + + paramValueMismatches = [] + for key in sorted(runtimeParams.intersection(postgresqlParams), key=self.paramSortKey): + protocolId, paramName = key + + runtimeParam = runtimeParamsByProtocolId.get(protocolId, {}).get(paramName, {}) + postgresqlParam = postgresqlParamsByProtocolId.get(protocolId, {}).get(paramName, {}) + + runtimeValue = self.normalizeParamValue(runtimeParam.get("value")) + postgresqlValue = self.normalizeParamValue(postgresqlParam.get("value")) + + if runtimeValue != postgresqlValue: + paramValueMismatches.append({ + "protocolId": protocolId, + "paramName": paramName, + "runtimeValue": runtimeValue, + "postgresqlValue": postgresqlValue, + }) + + return { + "missingParams": missingParams, + "extraParams": extraParams, + "paramValueMismatches": paramValueMismatches, + } + def validateProjectPostgresqlConsistency( self, mapper: PostgresqlFlatMapper, @@ -1145,32 +1190,15 @@ def validateProjectPostgresqlConsistency( key=self.inputRefSortKey, ) - missingParams = sorted( - runtimeParams - postgresqlParams, - key=self.paramSortKey, - ) - extraParams = sorted( - postgresqlParams - runtimeParams, - key=self.paramSortKey, + paramComparison = self.compareParams( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, ) - paramValueMismatches = [] - for key in sorted(runtimeParams.intersection(postgresqlParams), key=self.paramSortKey): - protocolId, paramName = key - - runtimeParam = runtimeParamsByProtocolId.get(protocolId, {}).get(paramName, {}) - postgresqlParam = postgresqlParamsByProtocolId.get(protocolId, {}).get(paramName, {}) - - runtimeValue = self.normalizeParamValue(runtimeParam.get("value")) - postgresqlValue = self.normalizeParamValue(postgresqlParam.get("value")) - - if runtimeValue != postgresqlValue: - paramValueMismatches.append({ - "protocolId": protocolId, - "paramName": paramName, - "runtimeValue": runtimeValue, - "postgresqlValue": postgresqlValue, - }) + missingParams = paramComparison["missingParams"] + extraParams = paramComparison["extraParams"] + paramValueMismatches = paramComparison["paramValueMismatches"] inputRefMismatches = [] for key in sorted(runtimeInputRefs.intersection(postgresqlInputRefsKeys), key=self.inputRefSortKey): From 87fa20056bf9bcb07d30f88c67d00473f4f24095 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 01:53:57 +0200 Subject: [PATCH 192/278] Extract input reference consistency comparison --- .../services/project_consistency_service.py | 130 +++++++++++------- 1 file changed, 79 insertions(+), 51 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 4ac5df37..4004abdd 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1077,6 +1077,77 @@ def compareParams( "paramValueMismatches": paramValueMismatches, } + def compareInputRefs( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeInputRefsByKey = runtimeSnapshot["inputRefsByKey"] + postgresqlInputRefsByKey = postgresqlSnapshot["inputRefsByKey"] + + runtimeInputRefs = derivedSets["runtimeInputRefs"] + postgresqlInputRefsKeys = derivedSets["postgresqlInputRefsKeys"] + + missingInputRefs = sorted( + runtimeInputRefs - postgresqlInputRefsKeys, + key=self.inputRefSortKey, + ) + extraInputRefs = sorted( + postgresqlInputRefsKeys - runtimeInputRefs, + key=self.inputRefSortKey, + ) + + inputRefMismatches = [] + for key in sorted(runtimeInputRefs.intersection(postgresqlInputRefsKeys), key=self.inputRefSortKey): + runtimeRef = runtimeInputRefsByKey.get(key, {}) + postgresqlRef = postgresqlInputRefsByKey.get(key, {}) + + changedFields = [] + + runtimeParentProtocolId = self.normalizeOptionalText(runtimeRef.get("parentProtocolId")) + postgresqlParentProtocolId = self.normalizeOptionalText(postgresqlRef.get("parentProtocolId")) + + runtimeParentOutputName = self.normalizeOptionalText(runtimeRef.get("parentOutputName")) + postgresqlParentOutputName = self.normalizeOptionalText(postgresqlRef.get("parentOutputName")) + + runtimeObjectClassName = self.normalizeOptionalText(runtimeRef.get("objectClassName")) + postgresqlObjectClassName = self.normalizeOptionalText(postgresqlRef.get("objectClassName")) + + if runtimeParentProtocolId != postgresqlParentProtocolId: + changedFields.append("parentProtocolId") + + if runtimeParentOutputName != postgresqlParentOutputName: + changedFields.append("parentOutputName") + + if ( + runtimeObjectClassName is not None + and postgresqlObjectClassName is not None + and runtimeObjectClassName != postgresqlObjectClassName + ): + changedFields.append("objectClassName") + + if changedFields: + protocolId, inputName, itemIndex = key + inputRefMismatches.append({ + "protocolId": protocolId, + "inputName": inputName, + "itemIndex": int(itemIndex), + "fields": changedFields, + "runtimeParentProtocolId": runtimeParentProtocolId, + "postgresqlParentProtocolId": postgresqlParentProtocolId, + "runtimeParentOutputName": runtimeParentOutputName, + "postgresqlParentOutputName": postgresqlParentOutputName, + "runtimeObjectClassName": runtimeObjectClassName, + "postgresqlObjectClassName": postgresqlObjectClassName, + }) + + return { + "missingInputRefs": missingInputRefs, + "extraInputRefs": extraInputRefs, + "inputRefMismatches": inputRefMismatches, + } + def validateProjectPostgresqlConsistency( self, mapper: PostgresqlFlatMapper, @@ -1181,15 +1252,6 @@ def validateProjectPostgresqlConsistency( extraSteps = stepComparison["extraSteps"] stepMismatches = stepComparison["stepMismatches"] - missingInputRefs = sorted( - runtimeInputRefs - postgresqlInputRefsKeys, - key=self.inputRefSortKey, - ) - extraInputRefs = sorted( - postgresqlInputRefsKeys - runtimeInputRefs, - key=self.inputRefSortKey, - ) - paramComparison = self.compareParams( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, @@ -1200,49 +1262,15 @@ def validateProjectPostgresqlConsistency( extraParams = paramComparison["extraParams"] paramValueMismatches = paramComparison["paramValueMismatches"] - inputRefMismatches = [] - for key in sorted(runtimeInputRefs.intersection(postgresqlInputRefsKeys), key=self.inputRefSortKey): - runtimeRef = runtimeInputRefsByKey.get(key, {}) - postgresqlRef = postgresqlInputRefsByKey.get(key, {}) - - changedFields = [] - - runtimeParentProtocolId = self.normalizeOptionalText(runtimeRef.get("parentProtocolId")) - postgresqlParentProtocolId = self.normalizeOptionalText(postgresqlRef.get("parentProtocolId")) - - runtimeParentOutputName = self.normalizeOptionalText(runtimeRef.get("parentOutputName")) - postgresqlParentOutputName = self.normalizeOptionalText(postgresqlRef.get("parentOutputName")) - - runtimeObjectClassName = self.normalizeOptionalText(runtimeRef.get("objectClassName")) - postgresqlObjectClassName = self.normalizeOptionalText(postgresqlRef.get("objectClassName")) - - if runtimeParentProtocolId != postgresqlParentProtocolId: - changedFields.append("parentProtocolId") - - if runtimeParentOutputName != postgresqlParentOutputName: - changedFields.append("parentOutputName") - - if ( - runtimeObjectClassName is not None - and postgresqlObjectClassName is not None - and runtimeObjectClassName != postgresqlObjectClassName - ): - changedFields.append("objectClassName") + inputRefComparison = self.compareInputRefs( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) - if changedFields: - protocolId, inputName, itemIndex = key - inputRefMismatches.append({ - "protocolId": protocolId, - "inputName": inputName, - "itemIndex": int(itemIndex), - "fields": changedFields, - "runtimeParentProtocolId": runtimeParentProtocolId, - "postgresqlParentProtocolId": postgresqlParentProtocolId, - "runtimeParentOutputName": runtimeParentOutputName, - "postgresqlParentOutputName": postgresqlParentOutputName, - "runtimeObjectClassName": runtimeObjectClassName, - "postgresqlObjectClassName": postgresqlObjectClassName, - }) + missingInputRefs = inputRefComparison["missingInputRefs"] + extraInputRefs = inputRefComparison["extraInputRefs"] + inputRefMismatches = inputRefComparison["inputRefMismatches"] runtimeInputRefDependenciesMissing = sorted( runtimeDependenciesFromInputRefs - runtimeDependencies, From eb3e89fc9c04dca17eed92ab459a617719552079 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 02:15:56 +0200 Subject: [PATCH 193/278] Extract input reference dependency comparison --- .../services/project_consistency_service.py | 60 ++++++++++++++----- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 4004abdd..7bb8a834 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1148,6 +1148,43 @@ def compareInputRefs( "inputRefMismatches": inputRefMismatches, } + def compareInputRefDependencies( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeDependencies = runtimeSnapshot["dependencies"] + postgresqlDependencies = postgresqlSnapshot["dependencies"] + + runtimeDependenciesFromInputRefs = derivedSets["runtimeDependenciesFromInputRefs"] + postgresqlDependenciesFromInputRefs = derivedSets["postgresqlDependenciesFromInputRefs"] + + runtimeInputRefDependenciesMissing = sorted( + runtimeDependenciesFromInputRefs - runtimeDependencies, + key=self.dependencySortKey, + ) + runtimeDependenciesWithoutInputRefs = sorted( + runtimeDependencies - runtimeDependenciesFromInputRefs, + key=self.dependencySortKey, + ) + + postgresqlInputRefDependenciesMissing = sorted( + postgresqlDependenciesFromInputRefs - postgresqlDependencies, + key=self.dependencySortKey, + ) + postgresqlDependenciesWithoutInputRefs = sorted( + postgresqlDependencies - postgresqlDependenciesFromInputRefs, + key=self.dependencySortKey, + ) + + return { + "runtimeInputRefDependenciesMissing": runtimeInputRefDependenciesMissing, + "runtimeDependenciesWithoutInputRefs": runtimeDependenciesWithoutInputRefs, + "postgresqlInputRefDependenciesMissing": postgresqlInputRefDependenciesMissing, + "postgresqlDependenciesWithoutInputRefs": postgresqlDependenciesWithoutInputRefs, + } + def validateProjectPostgresqlConsistency( self, mapper: PostgresqlFlatMapper, @@ -1272,23 +1309,16 @@ def validateProjectPostgresqlConsistency( extraInputRefs = inputRefComparison["extraInputRefs"] inputRefMismatches = inputRefComparison["inputRefMismatches"] - runtimeInputRefDependenciesMissing = sorted( - runtimeDependenciesFromInputRefs - runtimeDependencies, - key=self.dependencySortKey, - ) - runtimeDependenciesWithoutInputRefs = sorted( - runtimeDependencies - runtimeDependenciesFromInputRefs, - key=self.dependencySortKey, + inputRefDependencyComparison = self.compareInputRefDependencies( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, ) - postgresqlInputRefDependenciesMissing = sorted( - postgresqlDependenciesFromInputRefs - postgresqlDependencies, - key=self.dependencySortKey, - ) - postgresqlDependenciesWithoutInputRefs = sorted( - postgresqlDependencies - postgresqlDependenciesFromInputRefs, - key=self.dependencySortKey, - ) + runtimeInputRefDependenciesMissing = inputRefDependencyComparison["runtimeInputRefDependenciesMissing"] + runtimeDependenciesWithoutInputRefs = inputRefDependencyComparison["runtimeDependenciesWithoutInputRefs"] + postgresqlInputRefDependenciesMissing = inputRefDependencyComparison["postgresqlInputRefDependenciesMissing"] + postgresqlDependenciesWithoutInputRefs = inputRefDependencyComparison["postgresqlDependenciesWithoutInputRefs"] issues = { "missingProtocols": [ From 8a1edd48186e0fc8119d3f8e66cb1630f7c6fe6b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 02:26:26 +0200 Subject: [PATCH 194/278] Extract consistency issues builder --- .../services/project_consistency_service.py | 254 +++++++++++------- 1 file changed, 160 insertions(+), 94 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 7bb8a834..d12da788 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1185,142 +1185,62 @@ def compareInputRefDependencies( "postgresqlDependenciesWithoutInputRefs": postgresqlDependenciesWithoutInputRefs, } - def validateProjectPostgresqlConsistency( + def buildIssues( self, - mapper: PostgresqlFlatMapper, - projectId: int, - currentUser: dict, - refresh: bool = True, - checkPid: bool = True, - ) -> Dict[str, Any]: - dbProj = self.getProjectDbRow(mapper, projectId, currentUser) - if not dbProj: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Project not found", - ) - - self.loadProjectForThumbnails(dbProj) - - runtimeSnapshot = self.collectRuntimeSnapshot( - projectId=projectId, - refresh=refresh, - checkPid=checkPid, - ) - + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + protocolComparison: Dict[str, Any], + dependencyComparison: Dict[str, Any], + outputComparison: Dict[str, Any], + stepComparison: Dict[str, Any], + paramComparison: Dict[str, Any], + inputRefComparison: Dict[str, Any], + inputRefDependencyComparison: Dict[str, Any], + ) -> Dict[str, List[Dict[str, Any]]]: runtimeStatuses = runtimeSnapshot["statuses"] - runtimeClassNames = runtimeSnapshot["classNames"] - runtimeDependencies = runtimeSnapshot["dependencies"] runtimeOutputsByProtocolId = runtimeSnapshot["outputsByProtocolId"] runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] runtimeInputRefsByKey = runtimeSnapshot["inputRefsByKey"] runtimeParamsByProtocolId = runtimeSnapshot["paramsByProtocolId"] - postgresqlSnapshot = self.collectPostgresqlSnapshot( - mapper=mapper, - projectId=projectId, - ) - postgresqlStatuses = postgresqlSnapshot["statuses"] - postgresqlClassNames = postgresqlSnapshot["classNames"] - postgresqlDependencies = postgresqlSnapshot["dependencies"] persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] postgresqlInputRefsByKey = postgresqlSnapshot["inputRefsByKey"] postgresqlParamsByProtocolId = postgresqlSnapshot["paramsByProtocolId"] - derivedSets = self.buildDerivedSets( - runtimeSnapshot=runtimeSnapshot, - postgresqlSnapshot=postgresqlSnapshot, - ) - - runtimeOutputs = derivedSets["runtimeOutputs"] - postgresqlOutputs = derivedSets["postgresqlOutputs"] - runtimeSteps = derivedSets["runtimeSteps"] - postgresqlSteps = derivedSets["postgresqlSteps"] - runtimeProtocolIds = derivedSets["runtimeProtocolIds"] - postgresqlProtocolIds = derivedSets["postgresqlProtocolIds"] - runtimeInputRefs = derivedSets["runtimeInputRefs"] - postgresqlInputRefsKeys = derivedSets["postgresqlInputRefsKeys"] - runtimeParams = derivedSets["runtimeParams"] - postgresqlParams = derivedSets["postgresqlParams"] - runtimeDependenciesFromInputRefs = derivedSets["runtimeDependenciesFromInputRefs"] - postgresqlDependenciesFromInputRefs = derivedSets["postgresqlDependenciesFromInputRefs"] - - protocolComparison = self.compareProtocols( - runtimeSnapshot=runtimeSnapshot, - postgresqlSnapshot=postgresqlSnapshot, - derivedSets=derivedSets, - ) - missingProtocolIds = protocolComparison["missingProtocolIds"] extraProtocolIds = protocolComparison["extraProtocolIds"] - commonProtocolIds = protocolComparison["commonProtocolIds"] statusMismatches = protocolComparison["statusMismatches"] protocolClassMismatches = protocolComparison["protocolClassMismatches"] - dependencyComparison = self.compareDependencies( - runtimeSnapshot=runtimeSnapshot, - postgresqlSnapshot=postgresqlSnapshot, - ) - missingDependencies = dependencyComparison["missingDependencies"] extraDependencies = dependencyComparison["extraDependencies"] - outputComparison = self.compareOutputs( - runtimeSnapshot=runtimeSnapshot, - postgresqlSnapshot=postgresqlSnapshot, - derivedSets=derivedSets, - ) - missingOutputs = outputComparison["missingOutputs"] extraOutputs = outputComparison["extraOutputs"] outputClassMismatches = outputComparison["outputClassMismatches"] outputMapperKindMismatches = outputComparison["outputMapperKindMismatches"] outputItemsCountMismatches = outputComparison["outputItemsCountMismatches"] - stepComparison = self.compareSteps( - runtimeSnapshot=runtimeSnapshot, - postgresqlSnapshot=postgresqlSnapshot, - derivedSets=derivedSets, - ) - missingSteps = stepComparison["missingSteps"] extraSteps = stepComparison["extraSteps"] stepMismatches = stepComparison["stepMismatches"] - paramComparison = self.compareParams( - runtimeSnapshot=runtimeSnapshot, - postgresqlSnapshot=postgresqlSnapshot, - derivedSets=derivedSets, - ) - missingParams = paramComparison["missingParams"] extraParams = paramComparison["extraParams"] paramValueMismatches = paramComparison["paramValueMismatches"] - inputRefComparison = self.compareInputRefs( - runtimeSnapshot=runtimeSnapshot, - postgresqlSnapshot=postgresqlSnapshot, - derivedSets=derivedSets, - ) - missingInputRefs = inputRefComparison["missingInputRefs"] extraInputRefs = inputRefComparison["extraInputRefs"] inputRefMismatches = inputRefComparison["inputRefMismatches"] - inputRefDependencyComparison = self.compareInputRefDependencies( - runtimeSnapshot=runtimeSnapshot, - postgresqlSnapshot=postgresqlSnapshot, - derivedSets=derivedSets, - ) - runtimeInputRefDependenciesMissing = inputRefDependencyComparison["runtimeInputRefDependenciesMissing"] runtimeDependenciesWithoutInputRefs = inputRefDependencyComparison["runtimeDependenciesWithoutInputRefs"] postgresqlInputRefDependenciesMissing = inputRefDependencyComparison["postgresqlInputRefDependenciesMissing"] postgresqlDependenciesWithoutInputRefs = inputRefDependencyComparison["postgresqlDependenciesWithoutInputRefs"] - issues = { + return { "missingProtocols": [ { "protocolId": protocolId, @@ -1345,7 +1265,7 @@ def validateProjectPostgresqlConsistency( self.buildDependency(parentId, childId) for parentId, childId in extraDependencies ], - "missingOutputs": [ + "missingOutputs": [ self.buildMissingOutputIssue( protocolId, outputName, @@ -1364,7 +1284,6 @@ def validateProjectPostgresqlConsistency( "outputClassMismatches": outputClassMismatches, "outputMapperKindMismatches": outputMapperKindMismatches, "outputItemsCountMismatches": outputItemsCountMismatches, - "missingSteps": [ self.buildStep( protocolId, @@ -1418,6 +1337,153 @@ def validateProjectPostgresqlConsistency( "paramValueMismatches": paramValueMismatches, } + def validateProjectPostgresqlConsistency( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser: dict, + refresh: bool = True, + checkPid: bool = True, + ) -> Dict[str, Any]: + dbProj = self.getProjectDbRow(mapper, projectId, currentUser) + if not dbProj: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Project not found", + ) + + self.loadProjectForThumbnails(dbProj) + + runtimeSnapshot = self.collectRuntimeSnapshot( + projectId=projectId, + refresh=refresh, + checkPid=checkPid, + ) + + runtimeStatuses = runtimeSnapshot["statuses"] + runtimeClassNames = runtimeSnapshot["classNames"] + runtimeDependencies = runtimeSnapshot["dependencies"] + runtimeOutputsByProtocolId = runtimeSnapshot["outputsByProtocolId"] + runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] + runtimeInputRefsByKey = runtimeSnapshot["inputRefsByKey"] + runtimeParamsByProtocolId = runtimeSnapshot["paramsByProtocolId"] + + postgresqlSnapshot = self.collectPostgresqlSnapshot( + mapper=mapper, + projectId=projectId, + ) + + postgresqlStatuses = postgresqlSnapshot["statuses"] + postgresqlClassNames = postgresqlSnapshot["classNames"] + postgresqlDependencies = postgresqlSnapshot["dependencies"] + persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] + normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] + postgresqlInputRefsByKey = postgresqlSnapshot["inputRefsByKey"] + postgresqlParamsByProtocolId = postgresqlSnapshot["paramsByProtocolId"] + + derivedSets = self.buildDerivedSets( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + ) + + runtimeOutputs = derivedSets["runtimeOutputs"] + postgresqlOutputs = derivedSets["postgresqlOutputs"] + runtimeSteps = derivedSets["runtimeSteps"] + postgresqlSteps = derivedSets["postgresqlSteps"] + runtimeProtocolIds = derivedSets["runtimeProtocolIds"] + postgresqlProtocolIds = derivedSets["postgresqlProtocolIds"] + runtimeInputRefs = derivedSets["runtimeInputRefs"] + postgresqlInputRefsKeys = derivedSets["postgresqlInputRefsKeys"] + runtimeParams = derivedSets["runtimeParams"] + postgresqlParams = derivedSets["postgresqlParams"] + runtimeDependenciesFromInputRefs = derivedSets["runtimeDependenciesFromInputRefs"] + postgresqlDependenciesFromInputRefs = derivedSets["postgresqlDependenciesFromInputRefs"] + + protocolComparison = self.compareProtocols( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + + missingProtocolIds = protocolComparison["missingProtocolIds"] + extraProtocolIds = protocolComparison["extraProtocolIds"] + commonProtocolIds = protocolComparison["commonProtocolIds"] + statusMismatches = protocolComparison["statusMismatches"] + protocolClassMismatches = protocolComparison["protocolClassMismatches"] + + dependencyComparison = self.compareDependencies( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + ) + + missingDependencies = dependencyComparison["missingDependencies"] + extraDependencies = dependencyComparison["extraDependencies"] + + outputComparison = self.compareOutputs( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + + missingOutputs = outputComparison["missingOutputs"] + extraOutputs = outputComparison["extraOutputs"] + outputClassMismatches = outputComparison["outputClassMismatches"] + outputMapperKindMismatches = outputComparison["outputMapperKindMismatches"] + outputItemsCountMismatches = outputComparison["outputItemsCountMismatches"] + + stepComparison = self.compareSteps( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + + missingSteps = stepComparison["missingSteps"] + extraSteps = stepComparison["extraSteps"] + stepMismatches = stepComparison["stepMismatches"] + + paramComparison = self.compareParams( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + + missingParams = paramComparison["missingParams"] + extraParams = paramComparison["extraParams"] + paramValueMismatches = paramComparison["paramValueMismatches"] + + inputRefComparison = self.compareInputRefs( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + + missingInputRefs = inputRefComparison["missingInputRefs"] + extraInputRefs = inputRefComparison["extraInputRefs"] + inputRefMismatches = inputRefComparison["inputRefMismatches"] + + inputRefDependencyComparison = self.compareInputRefDependencies( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + + runtimeInputRefDependenciesMissing = inputRefDependencyComparison["runtimeInputRefDependenciesMissing"] + runtimeDependenciesWithoutInputRefs = inputRefDependencyComparison["runtimeDependenciesWithoutInputRefs"] + postgresqlInputRefDependenciesMissing = inputRefDependencyComparison["postgresqlInputRefDependenciesMissing"] + postgresqlDependenciesWithoutInputRefs = inputRefDependencyComparison["postgresqlDependenciesWithoutInputRefs"] + + issues = self.buildIssues( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + protocolComparison=protocolComparison, + dependencyComparison=dependencyComparison, + outputComparison=outputComparison, + stepComparison=stepComparison, + paramComparison=paramComparison, + inputRefComparison=inputRefComparison, + inputRefDependencyComparison=inputRefDependencyComparison, + ) + issuesCount = sum(len(items) for items in issues.values()) return { From 021cc0938cd0a40787d815458d4cd12839724a96 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 02:49:22 +0200 Subject: [PATCH 195/278] Extract consistency summary builder --- .../services/project_consistency_service.py | 111 ++++++------------ 1 file changed, 33 insertions(+), 78 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index d12da788..3c0e4a61 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1337,6 +1337,31 @@ def buildIssues( "paramValueMismatches": paramValueMismatches, } + def buildSummary( + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + issuesCount: int, + ) -> Dict[str, int]: + return { + "runtimeProtocols": len(derivedSets["runtimeProtocolIds"]), + "postgresqlProtocols": len(derivedSets["postgresqlProtocolIds"]), + "runtimeDependencies": len(runtimeSnapshot["dependencies"]), + "postgresqlDependencies": len(postgresqlSnapshot["dependencies"]), + "runtimeOutputs": len(derivedSets["runtimeOutputs"]), + "postgresqlOutputs": len(derivedSets["postgresqlOutputs"]), + "runtimeSteps": len(derivedSets["runtimeSteps"]), + "postgresqlSteps": len(derivedSets["postgresqlSteps"]), + "runtimeInputRefs": len(derivedSets["runtimeInputRefs"]), + "postgresqlInputRefs": len(derivedSets["postgresqlInputRefsKeys"]), + "runtimeInputRefDependencies": len(derivedSets["runtimeDependenciesFromInputRefs"]), + "postgresqlInputRefDependencies": len(derivedSets["postgresqlDependenciesFromInputRefs"]), + "runtimeParams": len(derivedSets["runtimeParams"]), + "postgresqlParams": len(derivedSets["postgresqlParams"]), + "issues": issuesCount, + } + def validateProjectPostgresqlConsistency( self, mapper: PostgresqlFlatMapper, @@ -1360,118 +1385,57 @@ def validateProjectPostgresqlConsistency( checkPid=checkPid, ) - runtimeStatuses = runtimeSnapshot["statuses"] - runtimeClassNames = runtimeSnapshot["classNames"] - runtimeDependencies = runtimeSnapshot["dependencies"] - runtimeOutputsByProtocolId = runtimeSnapshot["outputsByProtocolId"] - runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] - runtimeInputRefsByKey = runtimeSnapshot["inputRefsByKey"] - runtimeParamsByProtocolId = runtimeSnapshot["paramsByProtocolId"] - postgresqlSnapshot = self.collectPostgresqlSnapshot( mapper=mapper, projectId=projectId, ) - postgresqlStatuses = postgresqlSnapshot["statuses"] - postgresqlClassNames = postgresqlSnapshot["classNames"] - postgresqlDependencies = postgresqlSnapshot["dependencies"] - persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] - normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] - postgresqlInputRefsByKey = postgresqlSnapshot["inputRefsByKey"] - postgresqlParamsByProtocolId = postgresqlSnapshot["paramsByProtocolId"] - derivedSets = self.buildDerivedSets( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, ) - runtimeOutputs = derivedSets["runtimeOutputs"] - postgresqlOutputs = derivedSets["postgresqlOutputs"] - runtimeSteps = derivedSets["runtimeSteps"] - postgresqlSteps = derivedSets["postgresqlSteps"] - runtimeProtocolIds = derivedSets["runtimeProtocolIds"] - postgresqlProtocolIds = derivedSets["postgresqlProtocolIds"] - runtimeInputRefs = derivedSets["runtimeInputRefs"] - postgresqlInputRefsKeys = derivedSets["postgresqlInputRefsKeys"] - runtimeParams = derivedSets["runtimeParams"] - postgresqlParams = derivedSets["postgresqlParams"] - runtimeDependenciesFromInputRefs = derivedSets["runtimeDependenciesFromInputRefs"] - postgresqlDependenciesFromInputRefs = derivedSets["postgresqlDependenciesFromInputRefs"] - protocolComparison = self.compareProtocols( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, derivedSets=derivedSets, ) - missingProtocolIds = protocolComparison["missingProtocolIds"] - extraProtocolIds = protocolComparison["extraProtocolIds"] - commonProtocolIds = protocolComparison["commonProtocolIds"] - statusMismatches = protocolComparison["statusMismatches"] - protocolClassMismatches = protocolComparison["protocolClassMismatches"] - dependencyComparison = self.compareDependencies( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, ) - missingDependencies = dependencyComparison["missingDependencies"] - extraDependencies = dependencyComparison["extraDependencies"] - outputComparison = self.compareOutputs( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, derivedSets=derivedSets, ) - missingOutputs = outputComparison["missingOutputs"] - extraOutputs = outputComparison["extraOutputs"] - outputClassMismatches = outputComparison["outputClassMismatches"] - outputMapperKindMismatches = outputComparison["outputMapperKindMismatches"] - outputItemsCountMismatches = outputComparison["outputItemsCountMismatches"] - stepComparison = self.compareSteps( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, derivedSets=derivedSets, ) - missingSteps = stepComparison["missingSteps"] - extraSteps = stepComparison["extraSteps"] - stepMismatches = stepComparison["stepMismatches"] - paramComparison = self.compareParams( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, derivedSets=derivedSets, ) - missingParams = paramComparison["missingParams"] - extraParams = paramComparison["extraParams"] - paramValueMismatches = paramComparison["paramValueMismatches"] - inputRefComparison = self.compareInputRefs( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, derivedSets=derivedSets, ) - missingInputRefs = inputRefComparison["missingInputRefs"] - extraInputRefs = inputRefComparison["extraInputRefs"] - inputRefMismatches = inputRefComparison["inputRefMismatches"] - inputRefDependencyComparison = self.compareInputRefDependencies( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, derivedSets=derivedSets, ) - runtimeInputRefDependenciesMissing = inputRefDependencyComparison["runtimeInputRefDependenciesMissing"] - runtimeDependenciesWithoutInputRefs = inputRefDependencyComparison["runtimeDependenciesWithoutInputRefs"] - postgresqlInputRefDependenciesMissing = inputRefDependencyComparison["postgresqlInputRefDependenciesMissing"] - postgresqlDependenciesWithoutInputRefs = inputRefDependencyComparison["postgresqlDependenciesWithoutInputRefs"] - issues = self.buildIssues( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, @@ -1486,26 +1450,17 @@ def validateProjectPostgresqlConsistency( issuesCount = sum(len(items) for items in issues.values()) + summary = self.buildSummary( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + issuesCount=issuesCount, + ) + return { "ok": issuesCount == 0, "projectId": projectId, "checkedAt": datetime.utcnow().isoformat() + "Z", - "summary": { - "runtimeProtocols": len(runtimeProtocolIds), - "postgresqlProtocols": len(postgresqlProtocolIds), - "runtimeDependencies": len(runtimeDependencies), - "postgresqlDependencies": len(postgresqlDependencies), - "runtimeOutputs": len(runtimeOutputs), - "postgresqlOutputs": len(postgresqlOutputs), - "runtimeSteps": len(runtimeSteps), - "postgresqlSteps": len(postgresqlSteps), - "runtimeInputRefs": len(runtimeInputRefs), - "postgresqlInputRefs": len(postgresqlInputRefsKeys), - "runtimeInputRefDependencies": len(runtimeDependenciesFromInputRefs), - "postgresqlInputRefDependencies": len(postgresqlDependenciesFromInputRefs), - "runtimeParams": len(runtimeParams), - "postgresqlParams": len(postgresqlParams), - "issues": issuesCount, - }, + "summary": summary, "issues": issues, } \ No newline at end of file From 99e6ba01d9ab6ec563b6d8ba7d99d6051fc7ab05 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 09:51:06 +0200 Subject: [PATCH 196/278] Extract consistency comparisons builder --- .../services/project_consistency_service.py | 94 +++++++++++-------- 1 file changed, 55 insertions(+), 39 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 3c0e4a61..6a0199f6 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1362,39 +1362,12 @@ def buildSummary( "issues": issuesCount, } - def validateProjectPostgresqlConsistency( + def buildComparisons( self, - mapper: PostgresqlFlatMapper, - projectId: int, - currentUser: dict, - refresh: bool = True, - checkPid: bool = True, - ) -> Dict[str, Any]: - dbProj = self.getProjectDbRow(mapper, projectId, currentUser) - if not dbProj: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Project not found", - ) - - self.loadProjectForThumbnails(dbProj) - - runtimeSnapshot = self.collectRuntimeSnapshot( - projectId=projectId, - refresh=refresh, - checkPid=checkPid, - ) - - postgresqlSnapshot = self.collectPostgresqlSnapshot( - mapper=mapper, - projectId=projectId, - ) - - derivedSets = self.buildDerivedSets( - runtimeSnapshot=runtimeSnapshot, - postgresqlSnapshot=postgresqlSnapshot, - ) - + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Dict[str, Any]]: protocolComparison = self.compareProtocols( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, @@ -1436,16 +1409,59 @@ def validateProjectPostgresqlConsistency( derivedSets=derivedSets, ) + return { + "protocolComparison": protocolComparison, + "dependencyComparison": dependencyComparison, + "outputComparison": outputComparison, + "stepComparison": stepComparison, + "paramComparison": paramComparison, + "inputRefComparison": inputRefComparison, + "inputRefDependencyComparison": inputRefDependencyComparison, + } + + def validateProjectPostgresqlConsistency( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser: dict, + refresh: bool = True, + checkPid: bool = True, + ) -> Dict[str, Any]: + dbProj = self.getProjectDbRow(mapper, projectId, currentUser) + if not dbProj: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Project not found", + ) + + self.loadProjectForThumbnails(dbProj) + + runtimeSnapshot = self.collectRuntimeSnapshot( + projectId=projectId, + refresh=refresh, + checkPid=checkPid, + ) + + postgresqlSnapshot = self.collectPostgresqlSnapshot( + mapper=mapper, + projectId=projectId, + ) + + derivedSets = self.buildDerivedSets( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + ) + + comparisons = self.buildComparisons( + runtimeSnapshot=runtimeSnapshot, + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + issues = self.buildIssues( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, - protocolComparison=protocolComparison, - dependencyComparison=dependencyComparison, - outputComparison=outputComparison, - stepComparison=stepComparison, - paramComparison=paramComparison, - inputRefComparison=inputRefComparison, - inputRefDependencyComparison=inputRefDependencyComparison, + **comparisons, ) issuesCount = sum(len(items) for items in issues.values()) From 92685943c9f0443f1cdc88c166e6cd959a46e94f Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 09:57:24 +0200 Subject: [PATCH 197/278] Clean consistency service formatting --- .../services/project_consistency_service.py | 97 ++++++++++--------- 1 file changed, 49 insertions(+), 48 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 6a0199f6..9c4c93eb 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -979,58 +979,59 @@ def compareOutputs( } def compareSteps( - self, - runtimeSnapshot: Dict[str, Any], - postgresqlSnapshot: Dict[str, Any], - derivedSets: Dict[str, Any], + self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], ) -> Dict[str, Any]: - runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] - normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] - runtimeSteps = derivedSets["runtimeSteps"] - postgresqlSteps = derivedSets["postgresqlSteps"] + runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] + normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] - missingSteps = sorted( - runtimeSteps - postgresqlSteps, - key=self.stepSortKey, - ) - extraSteps = sorted( - postgresqlSteps - runtimeSteps, - key=self.stepSortKey, - ) + runtimeSteps = derivedSets["runtimeSteps"] + postgresqlSteps = derivedSets["postgresqlSteps"] - stepMismatches = [] - for protocolId, stepIndex in sorted(runtimeSteps.intersection(postgresqlSteps), key=self.stepSortKey): - runtimeStep = runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) - postgresqlStep = normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) - - runtimeName = str(runtimeStep.get("name") or "") - postgresqlName = str(postgresqlStep.get("name") or "") - runtimeStatus = self.normalizeStatus(runtimeStep.get("status")) - postgresqlStatus = self.normalizeStatus(postgresqlStep.get("status")) - - changedFields = [] - if runtimeName != postgresqlName: - changedFields.append("name") - if runtimeStatus != postgresqlStatus: - changedFields.append("status") - - if changedFields: - stepMismatches.append({ - "protocolId": protocolId, - "index": int(stepIndex), - "fields": changedFields, - "runtimeName": runtimeName, - "postgresqlName": postgresqlName, - "runtimeStatus": runtimeStatus, - "postgresqlStatus": postgresqlStatus, - }) - - return { - "missingSteps": missingSteps, - "extraSteps": extraSteps, - "stepMismatches": stepMismatches, - } + missingSteps = sorted( + runtimeSteps - postgresqlSteps, + key=self.stepSortKey, + ) + extraSteps = sorted( + postgresqlSteps - runtimeSteps, + key=self.stepSortKey, + ) + + stepMismatches = [] + for protocolId, stepIndex in sorted(runtimeSteps.intersection(postgresqlSteps), key=self.stepSortKey): + runtimeStep = runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) + postgresqlStep = normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) + + runtimeName = str(runtimeStep.get("name") or "") + postgresqlName = str(postgresqlStep.get("name") or "") + runtimeStatus = self.normalizeStatus(runtimeStep.get("status")) + postgresqlStatus = self.normalizeStatus(postgresqlStep.get("status")) + + changedFields = [] + if runtimeName != postgresqlName: + changedFields.append("name") + if runtimeStatus != postgresqlStatus: + changedFields.append("status") + + if changedFields: + stepMismatches.append({ + "protocolId": protocolId, + "index": int(stepIndex), + "fields": changedFields, + "runtimeName": runtimeName, + "postgresqlName": postgresqlName, + "runtimeStatus": runtimeStatus, + "postgresqlStatus": postgresqlStatus, + }) + + return { + "missingSteps": missingSteps, + "extraSteps": extraSteps, + "stepMismatches": stepMismatches, + } def compareParams( self, From fb6408089a3e006b228e234704440e6dbcb8b4c2 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 11:02:20 +0200 Subject: [PATCH 198/278] Align consistency tests with input reference target validation --- .../services/project_consistency_service.py | 56 ++++++ .../test_project_service_consistency.py | 183 +++++++++++++++++- 2 files changed, 237 insertions(+), 2 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 9c4c93eb..9a8db187 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1186,6 +1186,47 @@ def compareInputRefDependencies( "postgresqlDependenciesWithoutInputRefs": postgresqlDependenciesWithoutInputRefs, } + def comparePostgresqlInputRefTargets( + self, + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + postgresqlInputRefsByKey = postgresqlSnapshot["inputRefsByKey"] + persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] + + postgresqlProtocolIds = derivedSets["postgresqlProtocolIds"] + + postgresqlInputRefsWithMissingParentProtocols = [] + postgresqlInputRefsWithMissingParentOutputs = [] + + for key in sorted(postgresqlInputRefsByKey.keys(), key=self.inputRefSortKey): + inputRef = postgresqlInputRefsByKey.get(key, {}) + + parentProtocolId = self.normalizeOptionalText(inputRef.get("parentProtocolId")) + parentOutputName = self.normalizeOptionalText(inputRef.get("parentOutputName")) + + if not parentProtocolId or parentProtocolId == "PROJECT": + continue + + if parentProtocolId not in postgresqlProtocolIds: + issue = self.buildInputRef(key, inputRef) + issue["missingParentProtocolId"] = parentProtocolId + postgresqlInputRefsWithMissingParentProtocols.append(issue) + continue + + if not parentOutputName: + continue + + if parentOutputName not in persistedOutputsByProtocolId.get(parentProtocolId, {}): + issue = self.buildInputRef(key, inputRef) + issue["missingParentOutputName"] = parentOutputName + postgresqlInputRefsWithMissingParentOutputs.append(issue) + + return { + "postgresqlInputRefsWithMissingParentProtocols": postgresqlInputRefsWithMissingParentProtocols, + "postgresqlInputRefsWithMissingParentOutputs": postgresqlInputRefsWithMissingParentOutputs, + } + def buildIssues( self, runtimeSnapshot: Dict[str, Any], @@ -1197,6 +1238,7 @@ def buildIssues( paramComparison: Dict[str, Any], inputRefComparison: Dict[str, Any], inputRefDependencyComparison: Dict[str, Any], + postgresqlInputRefTargetComparison: Dict[str, Any], ) -> Dict[str, List[Dict[str, Any]]]: runtimeStatuses = runtimeSnapshot["statuses"] runtimeOutputsByProtocolId = runtimeSnapshot["outputsByProtocolId"] @@ -1240,6 +1282,12 @@ def buildIssues( runtimeDependenciesWithoutInputRefs = inputRefDependencyComparison["runtimeDependenciesWithoutInputRefs"] postgresqlInputRefDependenciesMissing = inputRefDependencyComparison["postgresqlInputRefDependenciesMissing"] postgresqlDependenciesWithoutInputRefs = inputRefDependencyComparison["postgresqlDependenciesWithoutInputRefs"] + postgresqlInputRefsWithMissingParentProtocols = ( + postgresqlInputRefTargetComparison["postgresqlInputRefsWithMissingParentProtocols"] + ) + postgresqlInputRefsWithMissingParentOutputs = ( + postgresqlInputRefTargetComparison["postgresqlInputRefsWithMissingParentOutputs"] + ) return { "missingProtocols": [ @@ -1311,6 +1359,8 @@ def buildIssues( for key in extraInputRefs ], "inputRefMismatches": inputRefMismatches, + "postgresqlInputRefsWithMissingParentProtocols": postgresqlInputRefsWithMissingParentProtocols, + "postgresqlInputRefsWithMissingParentOutputs": postgresqlInputRefsWithMissingParentOutputs, "runtimeInputRefDependenciesMissing": [ self.buildDependency(parentId, childId) for parentId, childId in runtimeInputRefDependenciesMissing @@ -1410,6 +1460,11 @@ def buildComparisons( derivedSets=derivedSets, ) + postgresqlInputRefTargetComparison = self.comparePostgresqlInputRefTargets( + postgresqlSnapshot=postgresqlSnapshot, + derivedSets=derivedSets, + ) + return { "protocolComparison": protocolComparison, "dependencyComparison": dependencyComparison, @@ -1418,6 +1473,7 @@ def buildComparisons( "paramComparison": paramComparison, "inputRefComparison": inputRefComparison, "inputRefDependencyComparison": inputRefDependencyComparison, + "postgresqlInputRefTargetComparison": postgresqlInputRefTargetComparison, } def validateProjectPostgresqlConsistency( diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 3c34a475..5b85dc02 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -231,6 +231,25 @@ def fetchAll(self, query, params): return [] + +def makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=1, +): + return { + "protocolId": str(protocolId), + "id": 100, + "objectId": 200, + "outputName": outputName, + "setClassName": setClassName, + "itemClassName": "Particle", + "properties": {"itemsCount": itemsCount}, + "createdAt": None, + "updatedAt": None, + } + @pytest.fixture def projectServiceModule(authTestEnv): return importlib.import_module("app.backend.api.services.project_service") @@ -264,6 +283,9 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "11": FakeRunNode("11", status="running", parents=["10"]), } ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=1)), + ] currentProject.nodes["11"].run.inputs = [ ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), ] @@ -295,6 +317,14 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "objectId": "100", }, ], + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=1, + ), + ], ) result = service.validateProjectPostgresqlConsistency( @@ -311,8 +341,8 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "postgresqlProtocols": 2, "runtimeDependencies": 1, "postgresqlDependencies": 1, - "runtimeOutputs": 0, - "postgresqlOutputs": 0, + "runtimeOutputs": 1, + "postgresqlOutputs": 1, "runtimeSteps": 0, "postgresqlSteps": 0, "runtimeInputRefs": 1, @@ -348,6 +378,8 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "outputMapperKindMismatches": [], "outputItemsCountMismatches": [], "protocolClassMismatches": [], + "postgresqlInputRefsWithMissingParentProtocols": [], + "postgresqlInputRefsWithMissingParentOutputs": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -470,6 +502,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "outputMapperKindMismatches": [], "outputItemsCountMismatches": [], "protocolClassMismatches": [], + "postgresqlInputRefsWithMissingParentProtocols": [], + "postgresqlInputRefsWithMissingParentOutputs": [], + } @@ -845,6 +880,9 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( ), } ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=1)), + ] currentProject.nodes["11"].run.inputs = [ ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), ("inputVolume", FakePointer(parentProtocolId=10, outputName="outputVolume", className="Volume")), @@ -885,6 +923,14 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( "objectId": "101", }, ], + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=1, + ), + ], ) result = service.validateProjectPostgresqlConsistency( @@ -1475,4 +1521,137 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolClassMismatches( "runtimeClassName": "RuntimeProtocolClass", "postgresqlClassName": "PostgresqlProtocolClass", } + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsInputRefMissingParentProtocol( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "11": FakeRunNode("11", status="running", parents=["PROJECT"]), + } + ) + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "11", "status": "running", "protocolClassName": "FakeProtocol"}, + ], + adjacencyMap={ + "11": {"parents": [], "children": []}, + }, + inputRefs=[ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "99", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "objectId": "100", + }, + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=False, + checkPid=False, + ) + + assert result["ok"] is False + assert result["issues"]["postgresqlInputRefsWithMissingParentProtocols"] == [ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "99", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "missingParentProtocolId": "99", + } + ] + assert result["issues"]["postgresqlInputRefsWithMissingParentOutputs"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsInputRefMissingParentOutput( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode("10", status="finished", parents=["PROJECT"]), + "11": FakeRunNode("11", status="running", parents=["10"]), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles")), + ] + currentProject.nodes["11"].run.inputs = [ + ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + {"protocolId": "11", "status": "running", "protocolClassName": "FakeProtocol"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": ["11"]}, + "11": {"parents": ["10"], "children": []}, + }, + inputRefs=[ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "objectId": "100", + }, + ], + setRows=[], + treeRows=[], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=False, + checkPid=False, + ) + + assert result["ok"] is False + assert result["issues"]["postgresqlInputRefsWithMissingParentProtocols"] == [] + assert result["issues"]["postgresqlInputRefsWithMissingParentOutputs"] == [ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputParticles", + "objectClassName": "SetOfParticles", + "missingParentOutputName": "outputParticles", + } ] \ No newline at end of file From fa4106be4e2d3526cc3a19e1ea206e05ef183cc3 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 11:20:03 +0200 Subject: [PATCH 199/278] Align input ref consistency tests with target validation --- .../test_project_service_consistency.py | 72 +++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 5b85dc02..6f9af497 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -777,6 +777,9 @@ def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols ), } ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=1)), + ] currentProject.nodes["11"].run.inputs = [ ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), ] @@ -829,6 +832,14 @@ def test_ValidateProjectPostgresqlConsistencyCollectsStepsForAllRuntimeProtocols "objectId": "100", }, ], + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=1, + ), + ], ) result = service.validateProjectPostgresqlConsistency( @@ -947,8 +958,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( "postgresqlProtocols": 2, "runtimeDependencies": 1, "postgresqlDependencies": 1, - "runtimeOutputs": 0, - "postgresqlOutputs": 0, + "runtimeOutputs": 1, + "postgresqlOutputs": 1, "runtimeSteps": 0, "postgresqlSteps": 0, "runtimeInputRefs": 2, @@ -957,7 +968,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( "postgresqlInputRefDependencies": 1, "runtimeParams": 0, "postgresqlParams": 0, - "issues": 3, + "issues": 5, } assert result["issues"]["missingInputRefs"] == [ @@ -996,6 +1007,27 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( "postgresqlObjectClassName": "SetOfParticles", } ] + assert result["issues"]["postgresqlInputRefsWithMissingParentProtocols"] == [] + assert result["issues"]["postgresqlInputRefsWithMissingParentOutputs"] == [ + { + "protocolId": "11", + "inputName": "inputParticles", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "wrongOutput", + "objectClassName": "SetOfParticles", + "missingParentOutputName": "wrongOutput", + }, + { + "protocolId": "11", + "inputName": "inputMask", + "itemIndex": 0, + "parentProtocolId": "10", + "parentOutputName": "outputMask", + "objectClassName": "VolumeMask", + "missingParentOutputName": "outputMask", + }, + ] assert result["issues"]["missingProtocols"] == [] assert result["issues"]["extraProtocols"] == [] @@ -1042,6 +1074,10 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatche ), } ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=1)), + ("outputVolume", FakeOutput("Volume")), + ] currentProject.nodes["11"].run.inputs = [ ("inputParticles", FakePointer(parentProtocolId=10, outputName="outputParticles")), ] @@ -1086,6 +1122,30 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatche "objectId": "101", }, ], + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=1, + ), + ], + treeRows=[ + { + "protocolId": "10", + "id": 300, + "scipionObjId": 400, + "name": "outputVolume", + "path": "outputVolume", + "className": "Volume", + "value": None, + "label": None, + "comment": None, + "metadata": {}, + "createdAt": None, + "updatedAt": None, + }, + ], ) result = service.validateProjectPostgresqlConsistency( @@ -1102,8 +1162,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatche "postgresqlProtocols": 3, "runtimeDependencies": 1, "postgresqlDependencies": 1, - "runtimeOutputs": 0, - "postgresqlOutputs": 0, + "runtimeOutputs": 2, + "postgresqlOutputs": 2, "runtimeSteps": 0, "postgresqlSteps": 0, "runtimeInputRefs": 2, @@ -1147,6 +1207,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefsDependencyMismatche assert result["issues"]["outputMapperKindMismatches"] == [] assert result["issues"]["outputItemsCountMismatches"] == [] assert result["issues"]["protocolClassMismatches"] == [] + assert result["issues"]["postgresqlInputRefsWithMissingParentProtocols"] == [] + assert result["issues"]["postgresqlInputRefsWithMissingParentOutputs"] == [] def test_ValidateProjectPostgresqlConsistencyReportsParamMismatches( From 295c4bef96d40e600d296ac74ef27e10ef7e4185 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 11:38:04 +0200 Subject: [PATCH 200/278] Fix input ref target validation test order --- .../services/test_project_service_consistency.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 6f9af497..861e8a92 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -1011,21 +1011,21 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( assert result["issues"]["postgresqlInputRefsWithMissingParentOutputs"] == [ { "protocolId": "11", - "inputName": "inputParticles", + "inputName": "inputMask", "itemIndex": 0, "parentProtocolId": "10", - "parentOutputName": "wrongOutput", - "objectClassName": "SetOfParticles", - "missingParentOutputName": "wrongOutput", + "parentOutputName": "outputMask", + "objectClassName": "VolumeMask", + "missingParentOutputName": "outputMask", }, { "protocolId": "11", - "inputName": "inputMask", + "inputName": "inputParticles", "itemIndex": 0, "parentProtocolId": "10", - "parentOutputName": "outputMask", - "objectClassName": "VolumeMask", - "missingParentOutputName": "outputMask", + "parentOutputName": "wrongOutput", + "objectClassName": "SetOfParticles", + "missingParentOutputName": "wrongOutput", }, ] From 3a3c8e409d90dbaf7aeeed2354c76cdf896f86bc Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 20:52:53 +0200 Subject: [PATCH 201/278] Validate PostgreSQL output payload completeness --- .../services/project_consistency_service.py | 103 +++++++++++++++++ .../test_project_service_consistency.py | 108 ++++++++++++++++++ 2 files changed, 211 insertions(+) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 9a8db187..993f3063 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -185,6 +185,33 @@ def buildExtraOutputIssue( .get("className"), } + def buildPostgresqlOutputPayloadIssue( + self, + protocolId: Any, + outputName: Any, + payload: Dict[str, Any], + missingFields: List[str], + ) -> Dict[str, Any]: + issue = { + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": payload.get("mapperKind"), + "className": payload.get("className"), + "missingFields": list(missingFields), + } + + for fieldName in ( + "setId", + "rootObjectId", + "scipionObjId", + "itemsCount", + "itemClassName", + ): + if fieldName in payload: + issue[fieldName] = payload.get(fieldName) + + return issue + def expectedOutputMapperKind(self, className: Any) -> Optional[str]: classNameText = self.normalizeOptionalText(className) if classNameText is None: @@ -978,6 +1005,67 @@ def compareOutputs( "outputItemsCountMismatches": outputItemsCountMismatches, } + def comparePostgresqlOutputPayloads( + self, + postgresqlSnapshot: Dict[str, Any], + ) -> Dict[str, Any]: + persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] + + flatSetOutputsWithIncompletePayload = [] + treeOutputsWithIncompletePayload = [] + + for protocolId in sorted(persistedOutputsByProtocolId.keys(), key=self.protocolSortKey): + outputsByName = persistedOutputsByProtocolId.get(protocolId, {}) + + for outputName in sorted(outputsByName.keys()): + payload = outputsByName.get(outputName, {}) + mapperKind = payload.get("mapperKind") + + if mapperKind == "flat_set": + missingFields = [] + + if payload.get("setId") in (None, ""): + missingFields.append("setId") + if payload.get("rootObjectId") in (None, ""): + missingFields.append("rootObjectId") + if self.normalizeOptionalText(payload.get("className")) is None: + missingFields.append("className") + if payload.get("itemsCount") is None: + missingFields.append("itemsCount") + + if missingFields: + flatSetOutputsWithIncompletePayload.append( + self.buildPostgresqlOutputPayloadIssue( + protocolId=protocolId, + outputName=outputName, + payload=payload, + missingFields=missingFields, + ) + ) + + elif mapperKind == "tree": + missingFields = [] + + if payload.get("rootObjectId") in (None, ""): + missingFields.append("rootObjectId") + if self.normalizeOptionalText(payload.get("className")) is None: + missingFields.append("className") + + if missingFields: + treeOutputsWithIncompletePayload.append( + self.buildPostgresqlOutputPayloadIssue( + protocolId=protocolId, + outputName=outputName, + payload=payload, + missingFields=missingFields, + ) + ) + + return { + "postgresqlFlatSetOutputsWithIncompletePayload": flatSetOutputsWithIncompletePayload, + "postgresqlTreeOutputsWithIncompletePayload": treeOutputsWithIncompletePayload, + } + def compareSteps( self, runtimeSnapshot: Dict[str, Any], @@ -1234,6 +1322,7 @@ def buildIssues( protocolComparison: Dict[str, Any], dependencyComparison: Dict[str, Any], outputComparison: Dict[str, Any], + outputPayloadComparison: Dict[str, Any], stepComparison: Dict[str, Any], paramComparison: Dict[str, Any], inputRefComparison: Dict[str, Any], @@ -1266,6 +1355,13 @@ def buildIssues( outputMapperKindMismatches = outputComparison["outputMapperKindMismatches"] outputItemsCountMismatches = outputComparison["outputItemsCountMismatches"] + postgresqlFlatSetOutputsWithIncompletePayload = ( + outputPayloadComparison["postgresqlFlatSetOutputsWithIncompletePayload"] + ) + postgresqlTreeOutputsWithIncompletePayload = ( + outputPayloadComparison["postgresqlTreeOutputsWithIncompletePayload"] + ) + missingSteps = stepComparison["missingSteps"] extraSteps = stepComparison["extraSteps"] stepMismatches = stepComparison["stepMismatches"] @@ -1333,6 +1429,8 @@ def buildIssues( "outputClassMismatches": outputClassMismatches, "outputMapperKindMismatches": outputMapperKindMismatches, "outputItemsCountMismatches": outputItemsCountMismatches, + "postgresqlFlatSetOutputsWithIncompletePayload": postgresqlFlatSetOutputsWithIncompletePayload, + "postgresqlTreeOutputsWithIncompletePayload": postgresqlTreeOutputsWithIncompletePayload, "missingSteps": [ self.buildStep( protocolId, @@ -1436,6 +1534,10 @@ def buildComparisons( derivedSets=derivedSets, ) + outputPayloadComparison = self.comparePostgresqlOutputPayloads( + postgresqlSnapshot=postgresqlSnapshot, + ) + stepComparison = self.compareSteps( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, @@ -1474,6 +1576,7 @@ def buildComparisons( "inputRefComparison": inputRefComparison, "inputRefDependencyComparison": inputRefDependencyComparison, "postgresqlInputRefTargetComparison": postgresqlInputRefTargetComparison, + "outputPayloadComparison": outputPayloadComparison, } def validateProjectPostgresqlConsistency( diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 861e8a92..6cb8f074 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -380,6 +380,8 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "protocolClassMismatches": [], "postgresqlInputRefsWithMissingParentProtocols": [], "postgresqlInputRefsWithMissingParentOutputs": [], + "postgresqlFlatSetOutputsWithIncompletePayload": [], + "postgresqlTreeOutputsWithIncompletePayload": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -504,6 +506,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "protocolClassMismatches": [], "postgresqlInputRefsWithMissingParentProtocols": [], "postgresqlInputRefsWithMissingParentOutputs": [], + "postgresqlFlatSetOutputsWithIncompletePayload": [], + "postgresqlTreeOutputsWithIncompletePayload": [], } @@ -1716,4 +1720,108 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMissingParentOutput( "objectClassName": "SetOfParticles", "missingParentOutputName": "outputParticles", } + ] + +def test_ValidateProjectPostgresqlConsistencyReportsIncompletePostgresqlOutputPayloads( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=None)), + ("outputVolume", FakeOutput("Volume")), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + {"protocolId": "10", "status": "finished", "protocolClassName": "FakeProtocol"}, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=None, + ), + ], + treeRows=[ + { + "protocolId": "10", + "id": None, + "scipionObjId": 400, + "name": "outputVolume", + "path": "outputVolume", + "className": "Volume", + "value": None, + "label": None, + "comment": None, + "metadata": {}, + "createdAt": None, + "updatedAt": None, + }, + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeOutputs"] == 2 + assert result["summary"]["postgresqlOutputs"] == 2 + assert result["summary"]["issues"] == 2 + + assert result["issues"]["missingOutputs"] == [] + assert result["issues"]["extraOutputs"] == [] + assert result["issues"]["outputClassMismatches"] == [] + assert result["issues"]["outputMapperKindMismatches"] == [] + assert result["issues"]["outputItemsCountMismatches"] == [] + + assert result["issues"]["postgresqlFlatSetOutputsWithIncompletePayload"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "missingFields": ["itemsCount"], + "setId": 100, + "rootObjectId": 200, + "itemsCount": None, + "itemClassName": "Particle", + } + ] + + assert result["issues"]["postgresqlTreeOutputsWithIncompletePayload"] == [ + { + "protocolId": "10", + "outputName": "outputVolume", + "mapperKind": "tree", + "className": "Volume", + "missingFields": ["rootObjectId"], + "rootObjectId": None, + "scipionObjId": 400, + } ] \ No newline at end of file From cf308684429cda255c34b6e91671d16cb0478493 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 21:21:49 +0200 Subject: [PATCH 202/278] Clean step consistency comparison formatting --- .../services/project_consistency_service.py | 96 +++++++++---------- 1 file changed, 47 insertions(+), 49 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 993f3063..384de9b4 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1066,60 +1066,58 @@ def comparePostgresqlOutputPayloads( "postgresqlTreeOutputsWithIncompletePayload": treeOutputsWithIncompletePayload, } - def compareSteps( - self, - runtimeSnapshot: Dict[str, Any], - postgresqlSnapshot: Dict[str, Any], - derivedSets: Dict[str, Any], - ) -> Dict[str, Any]: + def compareSteps(self, + runtimeSnapshot: Dict[str, Any], + postgresqlSnapshot: Dict[str, Any], + derivedSets: Dict[str, Any], + ) -> Dict[str, Any]: + runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] + normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] - runtimeStepsByProtocolId = runtimeSnapshot["stepsByProtocolId"] - normalizedPostgresqlStepsByProtocolId = postgresqlSnapshot["stepsByProtocolId"] + runtimeSteps = derivedSets["runtimeSteps"] + postgresqlSteps = derivedSets["postgresqlSteps"] - runtimeSteps = derivedSets["runtimeSteps"] - postgresqlSteps = derivedSets["postgresqlSteps"] + missingSteps = sorted( + runtimeSteps - postgresqlSteps, + key=self.stepSortKey, + ) + extraSteps = sorted( + postgresqlSteps - runtimeSteps, + key=self.stepSortKey, + ) - missingSteps = sorted( - runtimeSteps - postgresqlSteps, - key=self.stepSortKey, - ) - extraSteps = sorted( - postgresqlSteps - runtimeSteps, - key=self.stepSortKey, - ) + stepMismatches = [] + for protocolId, stepIndex in sorted(runtimeSteps.intersection(postgresqlSteps), key=self.stepSortKey): + runtimeStep = runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) + postgresqlStep = normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) - stepMismatches = [] - for protocolId, stepIndex in sorted(runtimeSteps.intersection(postgresqlSteps), key=self.stepSortKey): - runtimeStep = runtimeStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) - postgresqlStep = normalizedPostgresqlStepsByProtocolId.get(protocolId, {}).get(stepIndex, {}) - - runtimeName = str(runtimeStep.get("name") or "") - postgresqlName = str(postgresqlStep.get("name") or "") - runtimeStatus = self.normalizeStatus(runtimeStep.get("status")) - postgresqlStatus = self.normalizeStatus(postgresqlStep.get("status")) - - changedFields = [] - if runtimeName != postgresqlName: - changedFields.append("name") - if runtimeStatus != postgresqlStatus: - changedFields.append("status") - - if changedFields: - stepMismatches.append({ - "protocolId": protocolId, - "index": int(stepIndex), - "fields": changedFields, - "runtimeName": runtimeName, - "postgresqlName": postgresqlName, - "runtimeStatus": runtimeStatus, - "postgresqlStatus": postgresqlStatus, - }) + runtimeName = str(runtimeStep.get("name") or "") + postgresqlName = str(postgresqlStep.get("name") or "") + runtimeStatus = self.normalizeStatus(runtimeStep.get("status")) + postgresqlStatus = self.normalizeStatus(postgresqlStep.get("status")) - return { - "missingSteps": missingSteps, - "extraSteps": extraSteps, - "stepMismatches": stepMismatches, - } + changedFields = [] + if runtimeName != postgresqlName: + changedFields.append("name") + if runtimeStatus != postgresqlStatus: + changedFields.append("status") + + if changedFields: + stepMismatches.append({ + "protocolId": protocolId, + "index": int(stepIndex), + "fields": changedFields, + "runtimeName": runtimeName, + "postgresqlName": postgresqlName, + "runtimeStatus": runtimeStatus, + "postgresqlStatus": postgresqlStatus, + }) + + return { + "missingSteps": missingSteps, + "extraSteps": extraSteps, + "stepMismatches": stepMismatches, + } def compareParams( self, From d2b9096327a4c7901b8a2acd154b882cf6e93284 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 22:45:52 +0200 Subject: [PATCH 203/278] feat(consistency): compare flat set itemsCount with persisted rows --- .../services/project_consistency_service.py | 27 +++++++ app/backend/api/services/project_service.py | 10 +++ .../test_project_service_consistency.py | 80 +++++++++++++++++++ 3 files changed, 117 insertions(+) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 384de9b4..eab9e943 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1013,6 +1013,7 @@ def comparePostgresqlOutputPayloads( flatSetOutputsWithIncompletePayload = [] treeOutputsWithIncompletePayload = [] + flatSetItemsCountMismatches = [] for protocolId in sorted(persistedOutputsByProtocolId.keys(), key=self.protocolSortKey): outputsByName = persistedOutputsByProtocolId.get(protocolId, {}) @@ -1043,6 +1044,26 @@ def comparePostgresqlOutputPayloads( ) ) + propertiesItemsCount = self.toOptionalInt(payload.get("itemsCount")) + itemsTableCount = self.toOptionalInt(payload.get("itemsTableCount")) + + if ( + propertiesItemsCount is not None + and itemsTableCount is not None + and propertiesItemsCount != itemsTableCount + ): + flatSetItemsCountMismatches.append({ + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": mapperKind, + "className": payload.get("className"), + "setId": payload.get("setId"), + "rootObjectId": payload.get("rootObjectId"), + "itemsCount": propertiesItemsCount, + "itemsTableCount": itemsTableCount, + "itemClassName": payload.get("itemClassName"), + }) + elif mapperKind == "tree": missingFields = [] @@ -1064,6 +1085,7 @@ def comparePostgresqlOutputPayloads( return { "postgresqlFlatSetOutputsWithIncompletePayload": flatSetOutputsWithIncompletePayload, "postgresqlTreeOutputsWithIncompletePayload": treeOutputsWithIncompletePayload, + "postgresqlFlatSetItemsCountMismatches": flatSetItemsCountMismatches, } def compareSteps(self, @@ -1383,6 +1405,10 @@ def buildIssues( postgresqlInputRefTargetComparison["postgresqlInputRefsWithMissingParentOutputs"] ) + postgresqlFlatSetItemsCountMismatches = ( + outputPayloadComparison["postgresqlFlatSetItemsCountMismatches"] + ) + return { "missingProtocols": [ { @@ -1429,6 +1455,7 @@ def buildIssues( "outputItemsCountMismatches": outputItemsCountMismatches, "postgresqlFlatSetOutputsWithIncompletePayload": postgresqlFlatSetOutputsWithIncompletePayload, "postgresqlTreeOutputsWithIncompletePayload": postgresqlTreeOutputsWithIncompletePayload, + "postgresqlFlatSetItemsCountMismatches": postgresqlFlatSetItemsCountMismatches, "missingSteps": [ self.buildStep( protocolId, diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index cf533822..a67b89e9 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2023,11 +2023,20 @@ def toOptionalInt(value: Any) -> Optional[int]: s."setClassName", s."itemClassName", s.properties, + COALESCE(items_stats."itemsTableCount", 0) AS "itemsTableCount", s."createdAt", s."updatedAt" FROM scipion_sets s JOIN protocols p ON p.id = s."protocolDbId" + LEFT JOIN ( + SELECT + "setId", + COUNT(*)::int AS "itemsTableCount" + FROM scipion_set_items + GROUP BY "setId" + ) items_stats + ON items_stats."setId" = s.id WHERE s."projectId" = %s ORDER BY p."protocolId", s."outputName" """, @@ -2049,6 +2058,7 @@ def toOptionalInt(value: Any) -> Optional[int]: "className": row.get("setClassName"), "itemClassName": row.get("itemClassName"), "itemsCount": toOptionalInt(properties.get("itemsCount")) if isinstance(properties, dict) else None, + "itemsTableCount": toOptionalInt(row.get("itemsTableCount")), "maxItemId": toOptionalInt(properties.get("maxItemId")) if isinstance(properties, dict) else None, "lastSyncAt": properties.get("lastSyncAt") if isinstance(properties, dict) else None, "lastCheckedAt": properties.get("lastCheckedAt") if isinstance(properties, dict) else None, diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 6cb8f074..248970c7 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -237,6 +237,7 @@ def makeSetRow( outputName="outputParticles", setClassName="SetOfParticles", itemsCount=1, + itemsTableCount=None, ): return { "protocolId": str(protocolId), @@ -246,6 +247,7 @@ def makeSetRow( "setClassName": setClassName, "itemClassName": "Particle", "properties": {"itemsCount": itemsCount}, + "itemsTableCount": itemsCount if itemsTableCount is None else itemsTableCount, "createdAt": None, "updatedAt": None, } @@ -382,6 +384,7 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "postgresqlInputRefsWithMissingParentOutputs": [], "postgresqlFlatSetOutputsWithIncompletePayload": [], "postgresqlTreeOutputsWithIncompletePayload": [], + "postgresqlFlatSetItemsCountMismatches": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -508,6 +511,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "postgresqlInputRefsWithMissingParentOutputs": [], "postgresqlFlatSetOutputsWithIncompletePayload": [], "postgresqlTreeOutputsWithIncompletePayload": [], + "postgresqlFlatSetItemsCountMismatches": [], } @@ -1533,6 +1537,82 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputItemsCountMismatches( ] assert result["issues"]["protocolClassMismatches"] == [] + +def test_ValidateProjectPostgresqlConsistencyReportsFlatSetItemsTableCountMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=12)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=12, + itemsTableCount=10, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeOutputs"] == 1 + assert result["summary"]["postgresqlOutputs"] == 1 + assert result["summary"]["issues"] == 1 + + assert result["issues"]["outputItemsCountMismatches"] == [] + assert result["issues"]["postgresqlFlatSetItemsCountMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "itemsCount": 12, + "itemsTableCount": 10, + "itemClassName": "Particle", + } + ] + + def test_ValidateProjectPostgresqlConsistencyReportsProtocolClassMismatches( service, monkeypatch, From 208d48ca6221923071c3b726386badaabe4b9406 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 23:02:05 +0200 Subject: [PATCH 204/278] feat(consistency): validate flat set max item id --- .../services/project_consistency_service.py | 27 ++++++ app/backend/api/services/project_service.py | 16 ++-- .../test_project_service_consistency.py | 84 ++++++++++++++++++- 3 files changed, 119 insertions(+), 8 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index eab9e943..0125dd99 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1014,6 +1014,7 @@ def comparePostgresqlOutputPayloads( flatSetOutputsWithIncompletePayload = [] treeOutputsWithIncompletePayload = [] flatSetItemsCountMismatches = [] + flatSetMaxItemIdMismatches = [] for protocolId in sorted(persistedOutputsByProtocolId.keys(), key=self.protocolSortKey): outputsByName = persistedOutputsByProtocolId.get(protocolId, {}) @@ -1064,6 +1065,26 @@ def comparePostgresqlOutputPayloads( "itemClassName": payload.get("itemClassName"), }) + propertiesMaxItemId = self.toOptionalInt(payload.get("maxItemId")) + maxItemIdFromItems = self.toOptionalInt(payload.get("maxItemIdFromItems")) + + if ( + propertiesMaxItemId is not None + and maxItemIdFromItems is not None + and propertiesMaxItemId != maxItemIdFromItems + ): + flatSetMaxItemIdMismatches.append({ + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": mapperKind, + "className": payload.get("className"), + "setId": payload.get("setId"), + "rootObjectId": payload.get("rootObjectId"), + "maxItemId": propertiesMaxItemId, + "maxItemIdFromItems": maxItemIdFromItems, + "itemClassName": payload.get("itemClassName"), + }) + elif mapperKind == "tree": missingFields = [] @@ -1086,6 +1107,7 @@ def comparePostgresqlOutputPayloads( "postgresqlFlatSetOutputsWithIncompletePayload": flatSetOutputsWithIncompletePayload, "postgresqlTreeOutputsWithIncompletePayload": treeOutputsWithIncompletePayload, "postgresqlFlatSetItemsCountMismatches": flatSetItemsCountMismatches, + "postgresqlFlatSetMaxItemIdMismatches": flatSetMaxItemIdMismatches, } def compareSteps(self, @@ -1409,6 +1431,10 @@ def buildIssues( outputPayloadComparison["postgresqlFlatSetItemsCountMismatches"] ) + postgresqlFlatSetMaxItemIdMismatches = ( + outputPayloadComparison["postgresqlFlatSetMaxItemIdMismatches"] + ) + return { "missingProtocols": [ { @@ -1509,6 +1535,7 @@ def buildIssues( for key in extraParams ], "paramValueMismatches": paramValueMismatches, + "postgresqlFlatSetMaxItemIdMismatches": postgresqlFlatSetMaxItemIdMismatches, } def buildSummary( diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index a67b89e9..27252827 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2030,13 +2030,14 @@ def toOptionalInt(value: Any) -> Optional[int]: JOIN protocols p ON p.id = s."protocolDbId" LEFT JOIN ( - SELECT - "setId", - COUNT(*)::int AS "itemsTableCount" - FROM scipion_set_items - GROUP BY "setId" - ) items_stats - ON items_stats."setId" = s.id + SELECT + "setId", + COUNT(*)::int AS "itemsTableCount", + MAX("scipionItemId")::int AS "maxItemIdFromItems" + FROM scipion_set_items + GROUP BY "setId" + ) items_stats + ON items_stats."setId" = s.id WHERE s."projectId" = %s ORDER BY p."protocolId", s."outputName" """, @@ -2059,6 +2060,7 @@ def toOptionalInt(value: Any) -> Optional[int]: "itemClassName": row.get("itemClassName"), "itemsCount": toOptionalInt(properties.get("itemsCount")) if isinstance(properties, dict) else None, "itemsTableCount": toOptionalInt(row.get("itemsTableCount")), + "maxItemIdFromItems": toOptionalInt(row.get("maxItemIdFromItems")), "maxItemId": toOptionalInt(properties.get("maxItemId")) if isinstance(properties, dict) else None, "lastSyncAt": properties.get("lastSyncAt") if isinstance(properties, dict) else None, "lastCheckedAt": properties.get("lastCheckedAt") if isinstance(properties, dict) else None, diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 248970c7..02bababb 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -238,6 +238,8 @@ def makeSetRow( setClassName="SetOfParticles", itemsCount=1, itemsTableCount=None, + maxItemId=1, + maxItemIdFromItems=None, ): return { "protocolId": str(protocolId), @@ -246,8 +248,12 @@ def makeSetRow( "outputName": outputName, "setClassName": setClassName, "itemClassName": "Particle", - "properties": {"itemsCount": itemsCount}, + "properties": { + "itemsCount": itemsCount, + "maxItemId": maxItemId, + }, "itemsTableCount": itemsCount if itemsTableCount is None else itemsTableCount, + "maxItemIdFromItems": maxItemId if maxItemIdFromItems is None else maxItemIdFromItems, "createdAt": None, "updatedAt": None, } @@ -379,6 +385,7 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "outputClassMismatches": [], "outputMapperKindMismatches": [], "outputItemsCountMismatches": [], + "postgresqlFlatSetMaxItemIdMismatches": [], "protocolClassMismatches": [], "postgresqlInputRefsWithMissingParentProtocols": [], "postgresqlInputRefsWithMissingParentOutputs": [], @@ -506,6 +513,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "outputClassMismatches": [], "outputMapperKindMismatches": [], "outputItemsCountMismatches": [], + "postgresqlFlatSetMaxItemIdMismatches": [], "protocolClassMismatches": [], "postgresqlInputRefsWithMissingParentProtocols": [], "postgresqlInputRefsWithMissingParentOutputs": [], @@ -1904,4 +1912,78 @@ def test_ValidateProjectPostgresqlConsistencyReportsIncompletePostgresqlOutputPa "rootObjectId": None, "scipionObjId": 400, } + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsFlatSetMaxItemIdMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=20, + maxItemIdFromItems=30, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetItemsCountMismatches"] == [] + assert result["issues"]["postgresqlFlatSetMaxItemIdMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "maxItemId": 20, + "maxItemIdFromItems": 30, + "itemClassName": "Particle", + } ] \ No newline at end of file From 57eb7d2c241ac8c2bc7fba0c167eb8adfc5cbcdd Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 23:26:35 +0200 Subject: [PATCH 205/278] fix(consistency): avoid extra root table mismatch issues --- .../services/project_consistency_service.py | 52 ++++++++ app/backend/api/services/project_service.py | 39 ++++-- .../test_project_service_consistency.py | 121 +++++++++++++++++- 3 files changed, 202 insertions(+), 10 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 0125dd99..08fe818b 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1015,6 +1015,7 @@ def comparePostgresqlOutputPayloads( treeOutputsWithIncompletePayload = [] flatSetItemsCountMismatches = [] flatSetMaxItemIdMismatches = [] + flatSetRootTableMismatches = [] for protocolId in sorted(persistedOutputsByProtocolId.keys(), key=self.protocolSortKey): outputsByName = persistedOutputsByProtocolId.get(protocolId, {}) @@ -1085,6 +1086,51 @@ def comparePostgresqlOutputPayloads( "itemClassName": payload.get("itemClassName"), }) + rootTablesCount = self.toOptionalInt(payload.get("rootTablesCount")) + rootTableId = self.toOptionalInt(payload.get("rootTableId")) + rootTableItemsCount = self.toOptionalInt(payload.get("rootTableItemsCount")) + rootTableMaxItemId = self.toOptionalInt(payload.get("rootTableMaxItemId")) + + changedFields = [] + + if rootTablesCount is not None: + if rootTablesCount == 0: + changedFields.append("rootTableMissing") + elif rootTablesCount != 1: + changedFields.append("rootTablesCount") + else: + if ( + itemsTableCount is not None + and rootTableItemsCount is not None + and itemsTableCount != rootTableItemsCount + ): + changedFields.append("rootTableItemsCount") + + if ( + maxItemIdFromItems is not None + and rootTableMaxItemId is not None + and maxItemIdFromItems != rootTableMaxItemId + ): + changedFields.append("rootTableMaxItemId") + + if changedFields: + flatSetRootTableMismatches.append({ + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": mapperKind, + "className": payload.get("className"), + "setId": payload.get("setId"), + "rootObjectId": payload.get("rootObjectId"), + "rootTableId": rootTableId, + "fields": changedFields, + "rootTablesCount": rootTablesCount, + "itemsTableCount": itemsTableCount, + "rootTableItemsCount": rootTableItemsCount, + "maxItemIdFromItems": maxItemIdFromItems, + "rootTableMaxItemId": rootTableMaxItemId, + "itemClassName": payload.get("itemClassName"), + }) + elif mapperKind == "tree": missingFields = [] @@ -1108,6 +1154,7 @@ def comparePostgresqlOutputPayloads( "postgresqlTreeOutputsWithIncompletePayload": treeOutputsWithIncompletePayload, "postgresqlFlatSetItemsCountMismatches": flatSetItemsCountMismatches, "postgresqlFlatSetMaxItemIdMismatches": flatSetMaxItemIdMismatches, + "postgresqlFlatSetRootTableMismatches": flatSetRootTableMismatches, } def compareSteps(self, @@ -1435,6 +1482,10 @@ def buildIssues( outputPayloadComparison["postgresqlFlatSetMaxItemIdMismatches"] ) + postgresqlFlatSetRootTableMismatches = ( + outputPayloadComparison["postgresqlFlatSetRootTableMismatches"] + ) + return { "missingProtocols": [ { @@ -1536,6 +1587,7 @@ def buildIssues( ], "paramValueMismatches": paramValueMismatches, "postgresqlFlatSetMaxItemIdMismatches": postgresqlFlatSetMaxItemIdMismatches, + "postgresqlFlatSetRootTableMismatches": postgresqlFlatSetRootTableMismatches, } def buildSummary( diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 27252827..6b1dbab0 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2024,20 +2024,39 @@ def toOptionalInt(value: Any) -> Optional[int]: s."itemClassName", s.properties, COALESCE(items_stats."itemsTableCount", 0) AS "itemsTableCount", + items_stats."maxItemIdFromItems" AS "maxItemIdFromItems", + COALESCE(root_table_stats."rootTablesCount", 0) AS "rootTablesCount", + root_table_stats."rootTableId" AS "rootTableId", + COALESCE(root_table_stats."rootTableItemsCount", 0) AS "rootTableItemsCount", + root_table_stats."rootTableMaxItemId" AS "rootTableMaxItemId", s."createdAt", s."updatedAt" FROM scipion_sets s JOIN protocols p ON p.id = s."protocolDbId" LEFT JOIN ( - SELECT - "setId", - COUNT(*)::int AS "itemsTableCount", - MAX("scipionItemId")::int AS "maxItemIdFromItems" - FROM scipion_set_items - GROUP BY "setId" - ) items_stats - ON items_stats."setId" = s.id + SELECT + "setId", + COUNT(*)::int AS "itemsTableCount", + MAX("scipionItemId")::int AS "maxItemIdFromItems" + FROM scipion_set_items + GROUP BY "setId" + ) items_stats + ON items_stats."setId" = s.id + LEFT JOIN ( + SELECT + t."setId", + COUNT(DISTINCT t.id)::int AS "rootTablesCount", + MIN(t.id)::int AS "rootTableId", + COUNT(ti.id)::int AS "rootTableItemsCount", + MAX(ti."scipionItemId")::int AS "rootTableMaxItemId" + FROM scipion_set_tables t + LEFT JOIN scipion_set_table_items ti + ON ti."tableId" = t.id + WHERE t."tableKind" = 'root' + GROUP BY t."setId" + ) root_table_stats + ON root_table_stats."setId" = s.id WHERE s."projectId" = %s ORDER BY p."protocolId", s."outputName" """, @@ -2062,6 +2081,10 @@ def toOptionalInt(value: Any) -> Optional[int]: "itemsTableCount": toOptionalInt(row.get("itemsTableCount")), "maxItemIdFromItems": toOptionalInt(row.get("maxItemIdFromItems")), "maxItemId": toOptionalInt(properties.get("maxItemId")) if isinstance(properties, dict) else None, + "rootTablesCount": toOptionalInt(row.get("rootTablesCount")), + "rootTableId": toOptionalInt(row.get("rootTableId")), + "rootTableItemsCount": toOptionalInt(row.get("rootTableItemsCount")), + "rootTableMaxItemId": toOptionalInt(row.get("rootTableMaxItemId")), "lastSyncAt": properties.get("lastSyncAt") if isinstance(properties, dict) else None, "lastCheckedAt": properties.get("lastCheckedAt") if isinstance(properties, dict) else None, "skippedLastSync": properties.get("skippedLastSync") if isinstance(properties, dict) else None, diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 02bababb..c695c07b 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -240,7 +240,35 @@ def makeSetRow( itemsTableCount=None, maxItemId=1, maxItemIdFromItems=None, + rootTablesCount=1, + rootTableId=500, + rootTableItemsCount=None, + rootTableMaxItemId=None, ): + resolvedItemsTableCount = ( + itemsCount if itemsTableCount is None else itemsTableCount + ) + resolvedMaxItemIdFromItems = ( + maxItemId if maxItemIdFromItems is None else maxItemIdFromItems + ) + + if rootTablesCount == 0: + resolvedRootTableId = None + resolvedRootTableItemsCount = 0 + resolvedRootTableMaxItemId = None + else: + resolvedRootTableId = rootTableId + resolvedRootTableItemsCount = ( + resolvedItemsTableCount + if rootTableItemsCount is None + else rootTableItemsCount + ) + resolvedRootTableMaxItemId = ( + resolvedMaxItemIdFromItems + if rootTableMaxItemId is None + else rootTableMaxItemId + ) + return { "protocolId": str(protocolId), "id": 100, @@ -252,12 +280,17 @@ def makeSetRow( "itemsCount": itemsCount, "maxItemId": maxItemId, }, - "itemsTableCount": itemsCount if itemsTableCount is None else itemsTableCount, - "maxItemIdFromItems": maxItemId if maxItemIdFromItems is None else maxItemIdFromItems, + "itemsTableCount": resolvedItemsTableCount, + "maxItemIdFromItems": resolvedMaxItemIdFromItems, + "rootTablesCount": rootTablesCount, + "rootTableId": resolvedRootTableId, + "rootTableItemsCount": resolvedRootTableItemsCount, + "rootTableMaxItemId": resolvedRootTableMaxItemId, "createdAt": None, "updatedAt": None, } + @pytest.fixture def projectServiceModule(authTestEnv): return importlib.import_module("app.backend.api.services.project_service") @@ -392,6 +425,7 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "postgresqlFlatSetOutputsWithIncompletePayload": [], "postgresqlTreeOutputsWithIncompletePayload": [], "postgresqlFlatSetItemsCountMismatches": [], + "postgresqlFlatSetRootTableMismatches": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -520,6 +554,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "postgresqlFlatSetOutputsWithIncompletePayload": [], "postgresqlTreeOutputsWithIncompletePayload": [], "postgresqlFlatSetItemsCountMismatches": [], + "postgresqlFlatSetRootTableMismatches": [], } @@ -1986,4 +2021,86 @@ def test_ValidateProjectPostgresqlConsistencyReportsFlatSetMaxItemIdMismatches( "maxItemIdFromItems": 30, "itemClassName": "Particle", } + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsMissingFlatSetRootTable( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + rootTablesCount=0, + rootTableId=None, + rootTableItemsCount=0, + rootTableMaxItemId=None, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "rootTableId": None, + "fields": ["rootTableMissing"], + "rootTablesCount": 0, + "itemsTableCount": 3, + "rootTableItemsCount": 0, + "maxItemIdFromItems": 30, + "rootTableMaxItemId": None, + "itemClassName": "Particle", + } ] \ No newline at end of file From aa5edacbf19faf0581722f44a672064111b6e6ff Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Fri, 26 Jun 2026 23:45:36 +0200 Subject: [PATCH 206/278] fix(consistency): include flat set column stats in payload --- .../services/project_consistency_service.py | 27 ++++++ app/backend/api/services/project_service.py | 11 +++ .../test_project_service_consistency.py | 92 ++++++++++++++++++- 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 08fe818b..2a26bd79 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1016,6 +1016,7 @@ def comparePostgresqlOutputPayloads( flatSetItemsCountMismatches = [] flatSetMaxItemIdMismatches = [] flatSetRootTableMismatches = [] + flatSetColumnsCountMismatches = [] for protocolId in sorted(persistedOutputsByProtocolId.keys(), key=self.protocolSortKey): outputsByName = persistedOutputsByProtocolId.get(protocolId, {}) @@ -1086,6 +1087,26 @@ def comparePostgresqlOutputPayloads( "itemClassName": payload.get("itemClassName"), }) + propertiesColumnsCount = self.toOptionalInt(payload.get("columnsCount")) + setColumnsCount = self.toOptionalInt(payload.get("setColumnsCount")) + + if ( + propertiesColumnsCount is not None + and setColumnsCount is not None + and propertiesColumnsCount != setColumnsCount + ): + flatSetColumnsCountMismatches.append({ + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": mapperKind, + "className": payload.get("className"), + "setId": payload.get("setId"), + "rootObjectId": payload.get("rootObjectId"), + "columnsCount": propertiesColumnsCount, + "setColumnsCount": setColumnsCount, + "itemClassName": payload.get("itemClassName"), + }) + rootTablesCount = self.toOptionalInt(payload.get("rootTablesCount")) rootTableId = self.toOptionalInt(payload.get("rootTableId")) rootTableItemsCount = self.toOptionalInt(payload.get("rootTableItemsCount")) @@ -1154,6 +1175,7 @@ def comparePostgresqlOutputPayloads( "postgresqlTreeOutputsWithIncompletePayload": treeOutputsWithIncompletePayload, "postgresqlFlatSetItemsCountMismatches": flatSetItemsCountMismatches, "postgresqlFlatSetMaxItemIdMismatches": flatSetMaxItemIdMismatches, + "postgresqlFlatSetColumnsCountMismatches": flatSetColumnsCountMismatches, "postgresqlFlatSetRootTableMismatches": flatSetRootTableMismatches, } @@ -1482,6 +1504,10 @@ def buildIssues( outputPayloadComparison["postgresqlFlatSetMaxItemIdMismatches"] ) + postgresqlFlatSetColumnsCountMismatches = ( + outputPayloadComparison["postgresqlFlatSetColumnsCountMismatches"] + ) + postgresqlFlatSetRootTableMismatches = ( outputPayloadComparison["postgresqlFlatSetRootTableMismatches"] ) @@ -1588,6 +1614,7 @@ def buildIssues( "paramValueMismatches": paramValueMismatches, "postgresqlFlatSetMaxItemIdMismatches": postgresqlFlatSetMaxItemIdMismatches, "postgresqlFlatSetRootTableMismatches": postgresqlFlatSetRootTableMismatches, + "postgresqlFlatSetRootTableMismatches": postgresqlFlatSetRootTableMismatches, } def buildSummary( diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 6b1dbab0..8750feb9 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2025,6 +2025,7 @@ def toOptionalInt(value: Any) -> Optional[int]: s.properties, COALESCE(items_stats."itemsTableCount", 0) AS "itemsTableCount", items_stats."maxItemIdFromItems" AS "maxItemIdFromItems", + COALESCE(columns_stats."setColumnsCount", 0) AS "setColumnsCount", COALESCE(root_table_stats."rootTablesCount", 0) AS "rootTablesCount", root_table_stats."rootTableId" AS "rootTableId", COALESCE(root_table_stats."rootTableItemsCount", 0) AS "rootTableItemsCount", @@ -2043,6 +2044,14 @@ def toOptionalInt(value: Any) -> Optional[int]: GROUP BY "setId" ) items_stats ON items_stats."setId" = s.id + LEFT JOIN ( + SELECT + "setId", + COUNT(*)::int AS "setColumnsCount" + FROM scipion_set_columns + GROUP BY "setId" + ) columns_stats + ON columns_stats."setId" = s.id LEFT JOIN ( SELECT t."setId", @@ -2081,6 +2090,8 @@ def toOptionalInt(value: Any) -> Optional[int]: "itemsTableCount": toOptionalInt(row.get("itemsTableCount")), "maxItemIdFromItems": toOptionalInt(row.get("maxItemIdFromItems")), "maxItemId": toOptionalInt(properties.get("maxItemId")) if isinstance(properties, dict) else None, + "columnsCount": toOptionalInt(properties.get("columnsCount")) if isinstance(properties, dict) else None, + "setColumnsCount": toOptionalInt(row.get("setColumnsCount")), "rootTablesCount": toOptionalInt(row.get("rootTablesCount")), "rootTableId": toOptionalInt(row.get("rootTableId")), "rootTableItemsCount": toOptionalInt(row.get("rootTableItemsCount")), diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index c695c07b..f9725a8a 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -240,6 +240,8 @@ def makeSetRow( itemsTableCount=None, maxItemId=1, maxItemIdFromItems=None, + columnsCount=2, + setColumnsCount=None, rootTablesCount=1, rootTableId=500, rootTableItemsCount=None, @@ -252,6 +254,10 @@ def makeSetRow( maxItemId if maxItemIdFromItems is None else maxItemIdFromItems ) + resolvedSetColumnsCount = ( + columnsCount if setColumnsCount is None else setColumnsCount + ) + if rootTablesCount == 0: resolvedRootTableId = None resolvedRootTableItemsCount = 0 @@ -279,9 +285,11 @@ def makeSetRow( "properties": { "itemsCount": itemsCount, "maxItemId": maxItemId, + "columnsCount": columnsCount, }, "itemsTableCount": resolvedItemsTableCount, "maxItemIdFromItems": resolvedMaxItemIdFromItems, + "setColumnsCount": resolvedSetColumnsCount, "rootTablesCount": rootTablesCount, "rootTableId": resolvedRootTableId, "rootTableItemsCount": resolvedRootTableItemsCount, @@ -426,6 +434,7 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "postgresqlTreeOutputsWithIncompletePayload": [], "postgresqlFlatSetItemsCountMismatches": [], "postgresqlFlatSetRootTableMismatches": [], + "postgresqlFlatSetColumnsCountMismatches": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -2103,4 +2112,85 @@ def test_ValidateProjectPostgresqlConsistencyReportsMissingFlatSetRootTable( "rootTableMaxItemId": None, "itemClassName": "Particle", } - ] \ No newline at end of file + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsFlatSetColumnsCountMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + columnsCount=5, + setColumnsCount=4, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["runtimeOutputs"] == 1 + assert result["summary"]["postgresqlOutputs"] == 1 + assert result["summary"]["issues"] == 1 + + assert result["issues"]["postgresqlFlatSetColumnsCountMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "columnsCount": 5, + "setColumnsCount": 4, + "itemClassName": "Particle", + } + ] + assert result["issues"]["postgresqlFlatSetItemsCountMismatches"] == [] + assert result["issues"]["postgresqlFlatSetMaxItemIdMismatches"] == [] + assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [] \ No newline at end of file From c1b2313731f83dfcad98b869b1e4abd4ef7a2a6d Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 27 Jun 2026 10:38:43 +0200 Subject: [PATCH 207/278] feat(consistency): validate flat set columns count --- app/backend/api/services/project_consistency_service.py | 2 +- .../backend/api/services/test_project_service_consistency.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 2a26bd79..578480b9 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1613,7 +1613,7 @@ def buildIssues( ], "paramValueMismatches": paramValueMismatches, "postgresqlFlatSetMaxItemIdMismatches": postgresqlFlatSetMaxItemIdMismatches, - "postgresqlFlatSetRootTableMismatches": postgresqlFlatSetRootTableMismatches, + "postgresqlFlatSetColumnsCountMismatches": postgresqlFlatSetColumnsCountMismatches, "postgresqlFlatSetRootTableMismatches": postgresqlFlatSetRootTableMismatches, } diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index f9725a8a..adee948b 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -564,7 +564,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "postgresqlTreeOutputsWithIncompletePayload": [], "postgresqlFlatSetItemsCountMismatches": [], "postgresqlFlatSetRootTableMismatches": [], - + "postgresqlFlatSetColumnsCountMismatches": [], } From 23de5cf247c124d9e3e9506cbcec8d2e1af05454 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 27 Jun 2026 10:49:25 +0200 Subject: [PATCH 208/278] feat(consistency): validate root table columns count --- .../services/project_consistency_service.py | 10 ++ app/backend/api/services/project_service.py | 13 +++ .../test_project_service_consistency.py | 104 +++++++++++++++++- 3 files changed, 126 insertions(+), 1 deletion(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 578480b9..1b57853c 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1111,6 +1111,7 @@ def comparePostgresqlOutputPayloads( rootTableId = self.toOptionalInt(payload.get("rootTableId")) rootTableItemsCount = self.toOptionalInt(payload.get("rootTableItemsCount")) rootTableMaxItemId = self.toOptionalInt(payload.get("rootTableMaxItemId")) + rootTableColumnsCount = self.toOptionalInt(payload.get("rootTableColumnsCount")) changedFields = [] @@ -1134,6 +1135,13 @@ def comparePostgresqlOutputPayloads( ): changedFields.append("rootTableMaxItemId") + if ( + setColumnsCount is not None + and rootTableColumnsCount is not None + and setColumnsCount != rootTableColumnsCount + ): + changedFields.append("rootTableColumnsCount") + if changedFields: flatSetRootTableMismatches.append({ "protocolId": self.normalizeProtocolId(protocolId), @@ -1149,6 +1157,8 @@ def comparePostgresqlOutputPayloads( "rootTableItemsCount": rootTableItemsCount, "maxItemIdFromItems": maxItemIdFromItems, "rootTableMaxItemId": rootTableMaxItemId, + "setColumnsCount": setColumnsCount, + "rootTableColumnsCount": rootTableColumnsCount, "itemClassName": payload.get("itemClassName"), }) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 8750feb9..24d7ae78 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2030,6 +2030,7 @@ def toOptionalInt(value: Any) -> Optional[int]: root_table_stats."rootTableId" AS "rootTableId", COALESCE(root_table_stats."rootTableItemsCount", 0) AS "rootTableItemsCount", root_table_stats."rootTableMaxItemId" AS "rootTableMaxItemId", + COALESCE(root_table_columns_stats."rootTableColumnsCount", 0) AS "rootTableColumnsCount", s."createdAt", s."updatedAt" FROM scipion_sets s @@ -2066,6 +2067,17 @@ def toOptionalInt(value: Any) -> Optional[int]: GROUP BY t."setId" ) root_table_stats ON root_table_stats."setId" = s.id + LEFT JOIN ( + SELECT + t."setId", + COUNT(tc.id)::int AS "rootTableColumnsCount" + FROM scipion_set_tables t + LEFT JOIN scipion_set_table_columns tc + ON tc."tableId" = t.id + WHERE t."tableKind" = 'root' + GROUP BY t."setId" + ) root_table_columns_stats + ON root_table_columns_stats."setId" = s.id WHERE s."projectId" = %s ORDER BY p."protocolId", s."outputName" """, @@ -2096,6 +2108,7 @@ def toOptionalInt(value: Any) -> Optional[int]: "rootTableId": toOptionalInt(row.get("rootTableId")), "rootTableItemsCount": toOptionalInt(row.get("rootTableItemsCount")), "rootTableMaxItemId": toOptionalInt(row.get("rootTableMaxItemId")), + "rootTableColumnsCount": toOptionalInt(row.get("rootTableColumnsCount")), "lastSyncAt": properties.get("lastSyncAt") if isinstance(properties, dict) else None, "lastCheckedAt": properties.get("lastCheckedAt") if isinstance(properties, dict) else None, "skippedLastSync": properties.get("skippedLastSync") if isinstance(properties, dict) else None, diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index adee948b..b969779a 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -246,6 +246,7 @@ def makeSetRow( rootTableId=500, rootTableItemsCount=None, rootTableMaxItemId=None, + rootTableColumnsCount=None, ): resolvedItemsTableCount = ( itemsCount if itemsTableCount is None else itemsTableCount @@ -262,6 +263,7 @@ def makeSetRow( resolvedRootTableId = None resolvedRootTableItemsCount = 0 resolvedRootTableMaxItemId = None + resolvedRootTableColumnsCount = 0 else: resolvedRootTableId = rootTableId resolvedRootTableItemsCount = ( @@ -275,6 +277,12 @@ def makeSetRow( else rootTableMaxItemId ) + resolvedRootTableColumnsCount = ( + resolvedSetColumnsCount + if rootTableColumnsCount is None + else rootTableColumnsCount + ) + return { "protocolId": str(protocolId), "id": 100, @@ -294,6 +302,7 @@ def makeSetRow( "rootTableId": resolvedRootTableId, "rootTableItemsCount": resolvedRootTableItemsCount, "rootTableMaxItemId": resolvedRootTableMaxItemId, + "rootTableColumnsCount": resolvedRootTableColumnsCount, "createdAt": None, "updatedAt": None, } @@ -2081,6 +2090,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsMissingFlatSetRootTable( rootTableId=None, rootTableItemsCount=0, rootTableMaxItemId=None, + rootTableColumnsCount=0, ), ], ) @@ -2110,6 +2120,8 @@ def test_ValidateProjectPostgresqlConsistencyReportsMissingFlatSetRootTable( "rootTableItemsCount": 0, "maxItemIdFromItems": 30, "rootTableMaxItemId": None, + "setColumnsCount": 2, + "rootTableColumnsCount": 0, "itemClassName": "Particle", } ] @@ -2193,4 +2205,94 @@ def test_ValidateProjectPostgresqlConsistencyReportsFlatSetColumnsCountMismatche ] assert result["issues"]["postgresqlFlatSetItemsCountMismatches"] == [] assert result["issues"]["postgresqlFlatSetMaxItemIdMismatches"] == [] - assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [] \ No newline at end of file + assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [] + + +def test_ValidateProjectPostgresqlConsistencyReportsRootTableColumnsCountMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + columnsCount=5, + setColumnsCount=5, + rootTablesCount=1, + rootTableId=500, + rootTableItemsCount=3, + rootTableMaxItemId=30, + rootTableColumnsCount=4, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "rootTableId": 500, + "fields": ["rootTableColumnsCount"], + "rootTablesCount": 1, + "itemsTableCount": 3, + "rootTableItemsCount": 3, + "maxItemIdFromItems": 30, + "rootTableMaxItemId": 30, + "setColumnsCount": 5, + "rootTableColumnsCount": 4, + "itemClassName": "Particle", + } + ] + assert result["issues"]["postgresqlFlatSetColumnsCountMismatches"] == [] + assert result["issues"]["postgresqlFlatSetItemsCountMismatches"] == [] + assert result["issues"]["postgresqlFlatSetMaxItemIdMismatches"] == [] \ No newline at end of file From 5d702e05f92fbcbe1139e06a6c1e56d73aa8fbda Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 27 Jun 2026 11:49:55 +0200 Subject: [PATCH 209/278] feat(consistency): validate root table column schema --- .../services/project_consistency_service.py | 19 ++- app/backend/api/services/project_service.py | 42 ++++- .../test_project_service_consistency.py | 160 +++++++++++++++++- 3 files changed, 210 insertions(+), 11 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 1b57853c..4c2e39de 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1112,6 +1112,8 @@ def comparePostgresqlOutputPayloads( rootTableItemsCount = self.toOptionalInt(payload.get("rootTableItemsCount")) rootTableMaxItemId = self.toOptionalInt(payload.get("rootTableMaxItemId")) rootTableColumnsCount = self.toOptionalInt(payload.get("rootTableColumnsCount")) + setColumnsSignature = payload.get("setColumnsSignature") or [] + rootTableColumnsSignature = payload.get("rootTableColumnsSignature") or [] changedFields = [] @@ -1142,8 +1144,15 @@ def comparePostgresqlOutputPayloads( ): changedFields.append("rootTableColumnsCount") + if ( + setColumnsSignature + and rootTableColumnsSignature + and setColumnsSignature != rootTableColumnsSignature + ): + changedFields.append("rootTableColumnsSignature") + if changedFields: - flatSetRootTableMismatches.append({ + issue = { "protocolId": self.normalizeProtocolId(protocolId), "outputName": str(outputName), "mapperKind": mapperKind, @@ -1160,7 +1169,13 @@ def comparePostgresqlOutputPayloads( "setColumnsCount": setColumnsCount, "rootTableColumnsCount": rootTableColumnsCount, "itemClassName": payload.get("itemClassName"), - }) + } + + if "rootTableColumnsSignature" in changedFields: + issue["setColumnsSignature"] = setColumnsSignature + issue["rootTableColumnsSignature"] = rootTableColumnsSignature + + flatSetRootTableMismatches.append(issue) elif mapperKind == "tree": missingFields = [] diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 24d7ae78..256b88b0 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2026,11 +2026,13 @@ def toOptionalInt(value: Any) -> Optional[int]: COALESCE(items_stats."itemsTableCount", 0) AS "itemsTableCount", items_stats."maxItemIdFromItems" AS "maxItemIdFromItems", COALESCE(columns_stats."setColumnsCount", 0) AS "setColumnsCount", + columns_stats."setColumnsSignature" AS "setColumnsSignature", COALESCE(root_table_stats."rootTablesCount", 0) AS "rootTablesCount", root_table_stats."rootTableId" AS "rootTableId", COALESCE(root_table_stats."rootTableItemsCount", 0) AS "rootTableItemsCount", root_table_stats."rootTableMaxItemId" AS "rootTableMaxItemId", COALESCE(root_table_columns_stats."rootTableColumnsCount", 0) AS "rootTableColumnsCount", + root_table_columns_stats."rootTableColumnsSignature" AS "rootTableColumnsSignature", s."createdAt", s."updatedAt" FROM scipion_sets s @@ -2046,13 +2048,24 @@ def toOptionalInt(value: Any) -> Optional[int]: ) items_stats ON items_stats."setId" = s.id LEFT JOIN ( - SELECT - "setId", - COUNT(*)::int AS "setColumnsCount" - FROM scipion_set_columns - GROUP BY "setId" - ) columns_stats - ON columns_stats."setId" = s.id + SELECT + "setId", + COUNT(*)::int AS "setColumnsCount", + jsonb_agg( + jsonb_build_object( + 'labelProperty', "labelProperty", + 'columnName', "columnName", + 'className', "className", + 'valueType', "valueType", + 'position', position, + 'indexed', indexed + ) + ORDER BY position ASC, "labelProperty" ASC + ) AS "setColumnsSignature" + FROM scipion_set_columns + GROUP BY "setId" + ) columns_stats + ON columns_stats."setId" = s.id LEFT JOIN ( SELECT t."setId", @@ -2070,7 +2083,18 @@ def toOptionalInt(value: Any) -> Optional[int]: LEFT JOIN ( SELECT t."setId", - COUNT(tc.id)::int AS "rootTableColumnsCount" + COUNT(tc.id)::int AS "rootTableColumnsCount", + jsonb_agg( + jsonb_build_object( + 'labelProperty', tc."labelProperty", + 'columnName', tc."columnName", + 'className', tc."className", + 'valueType', tc."valueType", + 'position', tc.position, + 'indexed', tc.indexed + ) + ORDER BY tc.position ASC, tc."labelProperty" ASC + ) FILTER (WHERE tc.id IS NOT NULL) AS "rootTableColumnsSignature" FROM scipion_set_tables t LEFT JOIN scipion_set_table_columns tc ON tc."tableId" = t.id @@ -2104,11 +2128,13 @@ def toOptionalInt(value: Any) -> Optional[int]: "maxItemId": toOptionalInt(properties.get("maxItemId")) if isinstance(properties, dict) else None, "columnsCount": toOptionalInt(properties.get("columnsCount")) if isinstance(properties, dict) else None, "setColumnsCount": toOptionalInt(row.get("setColumnsCount")), + "setColumnsSignature": row.get("setColumnsSignature") or [], "rootTablesCount": toOptionalInt(row.get("rootTablesCount")), "rootTableId": toOptionalInt(row.get("rootTableId")), "rootTableItemsCount": toOptionalInt(row.get("rootTableItemsCount")), "rootTableMaxItemId": toOptionalInt(row.get("rootTableMaxItemId")), "rootTableColumnsCount": toOptionalInt(row.get("rootTableColumnsCount")), + "rootTableColumnsSignature": row.get("rootTableColumnsSignature") or [], "lastSyncAt": properties.get("lastSyncAt") if isinstance(properties, dict) else None, "lastCheckedAt": properties.get("lastCheckedAt") if isinstance(properties, dict) else None, "skippedLastSync": properties.get("skippedLastSync") if isinstance(properties, dict) else None, diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index b969779a..be9e5c00 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -247,6 +247,8 @@ def makeSetRow( rootTableItemsCount=None, rootTableMaxItemId=None, rootTableColumnsCount=None, + setColumnsSignature=None, + rootTableColumnsSignature=None, ): resolvedItemsTableCount = ( itemsCount if itemsTableCount is None else itemsTableCount @@ -264,6 +266,7 @@ def makeSetRow( resolvedRootTableItemsCount = 0 resolvedRootTableMaxItemId = None resolvedRootTableColumnsCount = 0 + resolvedRootTableColumnsSignature = [] else: resolvedRootTableId = rootTableId resolvedRootTableItemsCount = ( @@ -283,6 +286,30 @@ def makeSetRow( else rootTableColumnsCount ) + resolvedSetColumnsSignature = setColumnsSignature or [ + { + "labelProperty": "_id", + "columnName": "id", + "className": "Integer", + "valueType": "int", + "position": 0, + "indexed": True, + }, + { + "labelProperty": "_enabled", + "columnName": "enabled", + "className": "Boolean", + "valueType": "bool", + "position": 1, + "indexed": False, + }, + ] + resolvedRootTableColumnsSignature = ( + resolvedSetColumnsSignature + if rootTableColumnsSignature is None + else rootTableColumnsSignature + ) + return { "protocolId": str(protocolId), "id": 100, @@ -305,6 +332,8 @@ def makeSetRow( "rootTableColumnsCount": resolvedRootTableColumnsCount, "createdAt": None, "updatedAt": None, + "setColumnsSignature": resolvedSetColumnsSignature, + "rootTableColumnsSignature": resolvedRootTableColumnsSignature, } @@ -2295,4 +2324,133 @@ def test_ValidateProjectPostgresqlConsistencyReportsRootTableColumnsCountMismatc ] assert result["issues"]["postgresqlFlatSetColumnsCountMismatches"] == [] assert result["issues"]["postgresqlFlatSetItemsCountMismatches"] == [] - assert result["issues"]["postgresqlFlatSetMaxItemIdMismatches"] == [] \ No newline at end of file + assert result["issues"]["postgresqlFlatSetMaxItemIdMismatches"] == [] + +def test_ValidateProjectPostgresqlConsistencyReportsRootTableColumnSchemaMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + setColumnsSignature = [ + { + "labelProperty": "_id", + "columnName": "id", + "className": "Integer", + "valueType": "int", + "position": 0, + "indexed": True, + }, + { + "labelProperty": "_enabled", + "columnName": "enabled", + "className": "Boolean", + "valueType": "bool", + "position": 1, + "indexed": False, + }, + ] + + rootTableColumnsSignature = [ + { + "labelProperty": "_id", + "columnName": "id", + "className": "Integer", + "valueType": "int", + "position": 0, + "indexed": True, + }, + { + "labelProperty": "_enabled", + "columnName": "wrong_enabled", + "className": "Boolean", + "valueType": "bool", + "position": 1, + "indexed": False, + }, + ] + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + columnsCount=2, + setColumnsCount=2, + rootTablesCount=1, + rootTableId=500, + rootTableItemsCount=3, + rootTableMaxItemId=30, + rootTableColumnsCount=2, + setColumnsSignature=setColumnsSignature, + rootTableColumnsSignature=rootTableColumnsSignature, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "rootTableId": 500, + "fields": ["rootTableColumnsSignature"], + "rootTablesCount": 1, + "itemsTableCount": 3, + "rootTableItemsCount": 3, + "maxItemIdFromItems": 30, + "rootTableMaxItemId": 30, + "setColumnsCount": 2, + "rootTableColumnsCount": 2, + "setColumnsSignature": setColumnsSignature, + "rootTableColumnsSignature": rootTableColumnsSignature, + "itemClassName": "Particle", + } + ] + From e6a2a1dfc44aabe1ae765fc1020261a5fcde3c07 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 27 Jun 2026 11:59:58 +0200 Subject: [PATCH 210/278] feat(consistency): validate root table item id signature --- .../services/project_consistency_service.py | 13 +++ app/backend/api/services/project_service.py | 19 ++-- .../test_project_service_consistency.py | 99 ++++++++++++++++++- 3 files changed, 124 insertions(+), 7 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 4c2e39de..60f4306a 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1069,6 +1069,7 @@ def comparePostgresqlOutputPayloads( propertiesMaxItemId = self.toOptionalInt(payload.get("maxItemId")) maxItemIdFromItems = self.toOptionalInt(payload.get("maxItemIdFromItems")) + itemsIdSignature = self.normalizeOptionalText(payload.get("itemsIdSignature")) if ( propertiesMaxItemId is not None @@ -1114,6 +1115,7 @@ def comparePostgresqlOutputPayloads( rootTableColumnsCount = self.toOptionalInt(payload.get("rootTableColumnsCount")) setColumnsSignature = payload.get("setColumnsSignature") or [] rootTableColumnsSignature = payload.get("rootTableColumnsSignature") or [] + rootTableItemsIdSignature = self.normalizeOptionalText(payload.get("rootTableItemsIdSignature")) changedFields = [] @@ -1137,6 +1139,13 @@ def comparePostgresqlOutputPayloads( ): changedFields.append("rootTableMaxItemId") + if ( + itemsIdSignature is not None + and rootTableItemsIdSignature is not None + and itemsIdSignature != rootTableItemsIdSignature + ): + changedFields.append("rootTableItemsIdSignature") + if ( setColumnsCount is not None and rootTableColumnsCount is not None @@ -1171,6 +1180,10 @@ def comparePostgresqlOutputPayloads( "itemClassName": payload.get("itemClassName"), } + if "rootTableItemsIdSignature" in changedFields: + issue["itemsIdSignature"] = itemsIdSignature + issue["rootTableItemsIdSignature"] = rootTableItemsIdSignature + if "rootTableColumnsSignature" in changedFields: issue["setColumnsSignature"] = setColumnsSignature issue["rootTableColumnsSignature"] = rootTableColumnsSignature diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 256b88b0..f621f0d5 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2025,12 +2025,14 @@ def toOptionalInt(value: Any) -> Optional[int]: s.properties, COALESCE(items_stats."itemsTableCount", 0) AS "itemsTableCount", items_stats."maxItemIdFromItems" AS "maxItemIdFromItems", + items_stats."itemsIdSignature" AS "itemsIdSignature", COALESCE(columns_stats."setColumnsCount", 0) AS "setColumnsCount", columns_stats."setColumnsSignature" AS "setColumnsSignature", COALESCE(root_table_stats."rootTablesCount", 0) AS "rootTablesCount", root_table_stats."rootTableId" AS "rootTableId", COALESCE(root_table_stats."rootTableItemsCount", 0) AS "rootTableItemsCount", root_table_stats."rootTableMaxItemId" AS "rootTableMaxItemId", + root_table_stats."rootTableItemsIdSignature" AS "rootTableItemsIdSignature", COALESCE(root_table_columns_stats."rootTableColumnsCount", 0) AS "rootTableColumnsCount", root_table_columns_stats."rootTableColumnsSignature" AS "rootTableColumnsSignature", s."createdAt", @@ -2040,11 +2042,12 @@ def toOptionalInt(value: Any) -> Optional[int]: ON p.id = s."protocolDbId" LEFT JOIN ( SELECT - "setId", - COUNT(*)::int AS "itemsTableCount", - MAX("scipionItemId")::int AS "maxItemIdFromItems" - FROM scipion_set_items - GROUP BY "setId" + "setId", + COUNT(*)::int AS "itemsTableCount", + MAX("scipionItemId")::int AS "maxItemIdFromItems", + md5(string_agg("scipionItemId"::text, ',' ORDER BY "scipionItemId")) AS "itemsIdSignature" + FROM scipion_set_items + GROUP BY "setId" ) items_stats ON items_stats."setId" = s.id LEFT JOIN ( @@ -2072,7 +2075,9 @@ def toOptionalInt(value: Any) -> Optional[int]: COUNT(DISTINCT t.id)::int AS "rootTablesCount", MIN(t.id)::int AS "rootTableId", COUNT(ti.id)::int AS "rootTableItemsCount", - MAX(ti."scipionItemId")::int AS "rootTableMaxItemId" + MAX(ti."scipionItemId")::int AS "rootTableMaxItemId", + md5(string_agg(ti."scipionItemId"::text, ',' ORDER BY ti."scipionItemId")) + FILTER (WHERE ti.id IS NOT NULL) AS "rootTableItemsIdSignature" FROM scipion_set_tables t LEFT JOIN scipion_set_table_items ti ON ti."tableId" = t.id @@ -2125,6 +2130,7 @@ def toOptionalInt(value: Any) -> Optional[int]: "itemsCount": toOptionalInt(properties.get("itemsCount")) if isinstance(properties, dict) else None, "itemsTableCount": toOptionalInt(row.get("itemsTableCount")), "maxItemIdFromItems": toOptionalInt(row.get("maxItemIdFromItems")), + "itemsIdSignature": row.get("itemsIdSignature"), "maxItemId": toOptionalInt(properties.get("maxItemId")) if isinstance(properties, dict) else None, "columnsCount": toOptionalInt(properties.get("columnsCount")) if isinstance(properties, dict) else None, "setColumnsCount": toOptionalInt(row.get("setColumnsCount")), @@ -2133,6 +2139,7 @@ def toOptionalInt(value: Any) -> Optional[int]: "rootTableId": toOptionalInt(row.get("rootTableId")), "rootTableItemsCount": toOptionalInt(row.get("rootTableItemsCount")), "rootTableMaxItemId": toOptionalInt(row.get("rootTableMaxItemId")), + "rootTableItemsIdSignature": row.get("rootTableItemsIdSignature"), "rootTableColumnsCount": toOptionalInt(row.get("rootTableColumnsCount")), "rootTableColumnsSignature": row.get("rootTableColumnsSignature") or [], "lastSyncAt": properties.get("lastSyncAt") if isinstance(properties, dict) else None, diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index be9e5c00..761a3d51 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -249,6 +249,8 @@ def makeSetRow( rootTableColumnsCount=None, setColumnsSignature=None, rootTableColumnsSignature=None, + itemsIdSignature="items-1-2-3", + rootTableItemsIdSignature=None, ): resolvedItemsTableCount = ( itemsCount if itemsTableCount is None else itemsTableCount @@ -261,12 +263,18 @@ def makeSetRow( columnsCount if setColumnsCount is None else setColumnsCount ) + resolvedRootTableItemsIdSignature = ( + itemsIdSignature + if rootTableItemsIdSignature is None + else rootTableItemsIdSignature + ) + if rootTablesCount == 0: resolvedRootTableId = None resolvedRootTableItemsCount = 0 resolvedRootTableMaxItemId = None resolvedRootTableColumnsCount = 0 - resolvedRootTableColumnsSignature = [] + resolvedRootTableColumnsSignature = None else: resolvedRootTableId = rootTableId resolvedRootTableItemsCount = ( @@ -334,6 +342,8 @@ def makeSetRow( "updatedAt": None, "setColumnsSignature": resolvedSetColumnsSignature, "rootTableColumnsSignature": resolvedRootTableColumnsSignature, + "itemsIdSignature": itemsIdSignature, + "rootTableItemsIdSignature": resolvedRootTableItemsIdSignature, } @@ -2454,3 +2464,90 @@ def test_ValidateProjectPostgresqlConsistencyReportsRootTableColumnSchemaMismatc } ] + +def test_ValidateProjectPostgresqlConsistencyReportsRootTableItemsIdSignatureMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + rootTablesCount=1, + rootTableId=500, + rootTableItemsCount=3, + rootTableMaxItemId=30, + itemsIdSignature="items-10-20-30", + rootTableItemsIdSignature="items-10-25-30", + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "rootTableId": 500, + "fields": ["rootTableItemsIdSignature"], + "rootTablesCount": 1, + "itemsTableCount": 3, + "rootTableItemsCount": 3, + "maxItemIdFromItems": 30, + "rootTableMaxItemId": 30, + "setColumnsCount": 2, + "rootTableColumnsCount": 2, + "itemsIdSignature": "items-10-20-30", + "rootTableItemsIdSignature": "items-10-25-30", + "itemClassName": "Particle", + } + ] From 6cd044b9127cf61fd678e7cc8c262a4a00c3f9a9 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 27 Jun 2026 12:12:17 +0200 Subject: [PATCH 211/278] feat(consistency): validate root table item id signature --- .../test_project_service_consistency.py | 75 ++++++++++--------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 761a3d51..7bef0109 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -252,21 +252,38 @@ def makeSetRow( itemsIdSignature="items-1-2-3", rootTableItemsIdSignature=None, ): + defaultColumnsSignature = [ + { + "labelProperty": "_id", + "columnName": "id", + "className": "Integer", + "valueType": "int", + "position": 0, + "indexed": True, + }, + { + "labelProperty": "_enabled", + "columnName": "enabled", + "className": "Boolean", + "valueType": "bool", + "position": 1, + "indexed": False, + }, + ] + resolvedItemsTableCount = ( itemsCount if itemsTableCount is None else itemsTableCount ) resolvedMaxItemIdFromItems = ( maxItemId if maxItemIdFromItems is None else maxItemIdFromItems ) - resolvedSetColumnsCount = ( columnsCount if setColumnsCount is None else setColumnsCount ) - - resolvedRootTableItemsIdSignature = ( - itemsIdSignature - if rootTableItemsIdSignature is None - else rootTableItemsIdSignature + resolvedSetColumnsSignature = ( + defaultColumnsSignature + if setColumnsSignature is None + else setColumnsSignature ) if rootTablesCount == 0: @@ -274,7 +291,8 @@ def makeSetRow( resolvedRootTableItemsCount = 0 resolvedRootTableMaxItemId = None resolvedRootTableColumnsCount = 0 - resolvedRootTableColumnsSignature = None + resolvedRootTableColumnsSignature = [] + resolvedRootTableItemsIdSignature = None else: resolvedRootTableId = rootTableId resolvedRootTableItemsCount = ( @@ -287,36 +305,21 @@ def makeSetRow( if rootTableMaxItemId is None else rootTableMaxItemId ) - resolvedRootTableColumnsCount = ( resolvedSetColumnsCount if rootTableColumnsCount is None else rootTableColumnsCount ) - - resolvedSetColumnsSignature = setColumnsSignature or [ - { - "labelProperty": "_id", - "columnName": "id", - "className": "Integer", - "valueType": "int", - "position": 0, - "indexed": True, - }, - { - "labelProperty": "_enabled", - "columnName": "enabled", - "className": "Boolean", - "valueType": "bool", - "position": 1, - "indexed": False, - }, - ] - resolvedRootTableColumnsSignature = ( - resolvedSetColumnsSignature - if rootTableColumnsSignature is None - else rootTableColumnsSignature - ) + resolvedRootTableColumnsSignature = ( + resolvedSetColumnsSignature + if rootTableColumnsSignature is None + else rootTableColumnsSignature + ) + resolvedRootTableItemsIdSignature = ( + itemsIdSignature + if rootTableItemsIdSignature is None + else rootTableItemsIdSignature + ) return { "protocolId": str(protocolId), @@ -332,18 +335,18 @@ def makeSetRow( }, "itemsTableCount": resolvedItemsTableCount, "maxItemIdFromItems": resolvedMaxItemIdFromItems, + "itemsIdSignature": itemsIdSignature, "setColumnsCount": resolvedSetColumnsCount, + "setColumnsSignature": resolvedSetColumnsSignature, "rootTablesCount": rootTablesCount, "rootTableId": resolvedRootTableId, "rootTableItemsCount": resolvedRootTableItemsCount, "rootTableMaxItemId": resolvedRootTableMaxItemId, + "rootTableItemsIdSignature": resolvedRootTableItemsIdSignature, "rootTableColumnsCount": resolvedRootTableColumnsCount, + "rootTableColumnsSignature": resolvedRootTableColumnsSignature, "createdAt": None, "updatedAt": None, - "setColumnsSignature": resolvedSetColumnsSignature, - "rootTableColumnsSignature": resolvedRootTableColumnsSignature, - "itemsIdSignature": itemsIdSignature, - "rootTableItemsIdSignature": resolvedRootTableItemsIdSignature, } From 61859e34354b040305565e085a033c055bd09267 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 27 Jun 2026 22:49:04 +0200 Subject: [PATCH 212/278] feat(consistency): validate root table item value signature --- .../services/project_consistency_service.py | 15 +++ app/backend/api/services/project_service.py | 80 ++++++++++----- .../test_project_service_consistency.py | 99 +++++++++++++++++++ 3 files changed, 170 insertions(+), 24 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 60f4306a..9b51b020 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1070,6 +1070,7 @@ def comparePostgresqlOutputPayloads( propertiesMaxItemId = self.toOptionalInt(payload.get("maxItemId")) maxItemIdFromItems = self.toOptionalInt(payload.get("maxItemIdFromItems")) itemsIdSignature = self.normalizeOptionalText(payload.get("itemsIdSignature")) + itemsValueSignature = self.normalizeOptionalText(payload.get("itemsValueSignature")) if ( propertiesMaxItemId is not None @@ -1116,6 +1117,9 @@ def comparePostgresqlOutputPayloads( setColumnsSignature = payload.get("setColumnsSignature") or [] rootTableColumnsSignature = payload.get("rootTableColumnsSignature") or [] rootTableItemsIdSignature = self.normalizeOptionalText(payload.get("rootTableItemsIdSignature")) + rootTableItemsValueSignature = self.normalizeOptionalText( + payload.get("rootTableItemsValueSignature") + ) changedFields = [] @@ -1146,6 +1150,13 @@ def comparePostgresqlOutputPayloads( ): changedFields.append("rootTableItemsIdSignature") + if ( + itemsValueSignature is not None + and rootTableItemsValueSignature is not None + and itemsValueSignature != rootTableItemsValueSignature + ): + changedFields.append("rootTableItemsValueSignature") + if ( setColumnsCount is not None and rootTableColumnsCount is not None @@ -1180,6 +1191,10 @@ def comparePostgresqlOutputPayloads( "itemClassName": payload.get("itemClassName"), } + if "rootTableItemsValueSignature" in changedFields: + issue["itemsValueSignature"] = itemsValueSignature + issue["rootTableItemsValueSignature"] = rootTableItemsValueSignature + if "rootTableItemsIdSignature" in changedFields: issue["itemsIdSignature"] = itemsIdSignature issue["rootTableItemsIdSignature"] = rootTableItemsIdSignature diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index f621f0d5..a637e4cd 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2026,6 +2026,7 @@ def toOptionalInt(value: Any) -> Optional[int]: COALESCE(items_stats."itemsTableCount", 0) AS "itemsTableCount", items_stats."maxItemIdFromItems" AS "maxItemIdFromItems", items_stats."itemsIdSignature" AS "itemsIdSignature", + items_stats."itemsValueSignature" AS "itemsValueSignature", COALESCE(columns_stats."setColumnsCount", 0) AS "setColumnsCount", columns_stats."setColumnsSignature" AS "setColumnsSignature", COALESCE(root_table_stats."rootTablesCount", 0) AS "rootTablesCount", @@ -2033,6 +2034,7 @@ def toOptionalInt(value: Any) -> Optional[int]: COALESCE(root_table_stats."rootTableItemsCount", 0) AS "rootTableItemsCount", root_table_stats."rootTableMaxItemId" AS "rootTableMaxItemId", root_table_stats."rootTableItemsIdSignature" AS "rootTableItemsIdSignature", + root_table_stats."rootTableItemsValueSignature" AS "rootTableItemsValueSignature", COALESCE(root_table_columns_stats."rootTableColumnsCount", 0) AS "rootTableColumnsCount", root_table_columns_stats."rootTableColumnsSignature" AS "rootTableColumnsSignature", s."createdAt", @@ -2041,15 +2043,29 @@ def toOptionalInt(value: Any) -> Optional[int]: JOIN protocols p ON p.id = s."protocolDbId" LEFT JOIN ( - SELECT - "setId", - COUNT(*)::int AS "itemsTableCount", - MAX("scipionItemId")::int AS "maxItemIdFromItems", - md5(string_agg("scipionItemId"::text, ',' ORDER BY "scipionItemId")) AS "itemsIdSignature" - FROM scipion_set_items - GROUP BY "setId" - ) items_stats - ON items_stats."setId" = s.id + SELECT + "setId", + COUNT(*)::int AS "itemsTableCount", + MAX("scipionItemId")::int AS "maxItemIdFromItems", + md5(string_agg("scipionItemId"::text, ',' ORDER BY "scipionItemId")) AS "itemsIdSignature", + md5( + string_agg( + jsonb_build_object( + 'scipionItemId', "scipionItemId", + 'enabled', enabled, + 'label', label, + 'comment', comment, + 'creation', creation, + 'values', "values" + )::text, + ',' + ORDER BY "scipionItemId" + ) + ) AS "itemsValueSignature" + FROM scipion_set_items + GROUP BY "setId" + ) items_stats + ON items_stats."setId" = s.id LEFT JOIN ( SELECT "setId", @@ -2070,21 +2086,35 @@ def toOptionalInt(value: Any) -> Optional[int]: ) columns_stats ON columns_stats."setId" = s.id LEFT JOIN ( - SELECT - t."setId", - COUNT(DISTINCT t.id)::int AS "rootTablesCount", - MIN(t.id)::int AS "rootTableId", - COUNT(ti.id)::int AS "rootTableItemsCount", - MAX(ti."scipionItemId")::int AS "rootTableMaxItemId", - md5(string_agg(ti."scipionItemId"::text, ',' ORDER BY ti."scipionItemId")) - FILTER (WHERE ti.id IS NOT NULL) AS "rootTableItemsIdSignature" - FROM scipion_set_tables t - LEFT JOIN scipion_set_table_items ti - ON ti."tableId" = t.id - WHERE t."tableKind" = 'root' - GROUP BY t."setId" - ) root_table_stats - ON root_table_stats."setId" = s.id + SELECT + t."setId", + COUNT(DISTINCT t.id)::int AS "rootTablesCount", + MIN(t.id)::int AS "rootTableId", + COUNT(ti.id)::int AS "rootTableItemsCount", + MAX(ti."scipionItemId")::int AS "rootTableMaxItemId", + md5(string_agg(ti."scipionItemId"::text, ',' ORDER BY ti."scipionItemId")) + FILTER (WHERE ti.id IS NOT NULL) AS "rootTableItemsIdSignature", + md5( + string_agg( + jsonb_build_object( + 'scipionItemId', ti."scipionItemId", + 'enabled', ti.enabled, + 'label', ti.label, + 'comment', ti.comment, + 'creation', ti.creation, + 'values', ti."values" + )::text, + ',' + ORDER BY ti."scipionItemId" + ) + ) FILTER (WHERE ti.id IS NOT NULL) AS "rootTableItemsValueSignature" + FROM scipion_set_tables t + LEFT JOIN scipion_set_table_items ti + ON ti."tableId" = t.id + WHERE t."tableKind" = 'root' + GROUP BY t."setId" + ) root_table_stats + ON root_table_stats."setId" = s.id LEFT JOIN ( SELECT t."setId", @@ -2131,6 +2161,7 @@ def toOptionalInt(value: Any) -> Optional[int]: "itemsTableCount": toOptionalInt(row.get("itemsTableCount")), "maxItemIdFromItems": toOptionalInt(row.get("maxItemIdFromItems")), "itemsIdSignature": row.get("itemsIdSignature"), + "itemsValueSignature": row.get("itemsValueSignature"), "maxItemId": toOptionalInt(properties.get("maxItemId")) if isinstance(properties, dict) else None, "columnsCount": toOptionalInt(properties.get("columnsCount")) if isinstance(properties, dict) else None, "setColumnsCount": toOptionalInt(row.get("setColumnsCount")), @@ -2140,6 +2171,7 @@ def toOptionalInt(value: Any) -> Optional[int]: "rootTableItemsCount": toOptionalInt(row.get("rootTableItemsCount")), "rootTableMaxItemId": toOptionalInt(row.get("rootTableMaxItemId")), "rootTableItemsIdSignature": row.get("rootTableItemsIdSignature"), + "rootTableItemsValueSignature": row.get("rootTableItemsValueSignature"), "rootTableColumnsCount": toOptionalInt(row.get("rootTableColumnsCount")), "rootTableColumnsSignature": row.get("rootTableColumnsSignature") or [], "lastSyncAt": properties.get("lastSyncAt") if isinstance(properties, dict) else None, diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 7bef0109..b76398ee 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -251,6 +251,8 @@ def makeSetRow( rootTableColumnsSignature=None, itemsIdSignature="items-1-2-3", rootTableItemsIdSignature=None, + itemsValueSignature="values-1-2-3", + rootTableItemsValueSignature=None, ): defaultColumnsSignature = [ { @@ -293,6 +295,7 @@ def makeSetRow( resolvedRootTableColumnsCount = 0 resolvedRootTableColumnsSignature = [] resolvedRootTableItemsIdSignature = None + resolvedRootTableItemsValueSignature = None else: resolvedRootTableId = rootTableId resolvedRootTableItemsCount = ( @@ -320,6 +323,11 @@ def makeSetRow( if rootTableItemsIdSignature is None else rootTableItemsIdSignature ) + resolvedRootTableItemsValueSignature = ( + itemsValueSignature + if rootTableItemsValueSignature is None + else rootTableItemsValueSignature + ) return { "protocolId": str(protocolId), @@ -347,6 +355,8 @@ def makeSetRow( "rootTableColumnsSignature": resolvedRootTableColumnsSignature, "createdAt": None, "updatedAt": None, + "itemsValueSignature": itemsValueSignature, + "rootTableItemsValueSignature": resolvedRootTableItemsValueSignature, } @@ -2554,3 +2564,92 @@ def test_ValidateProjectPostgresqlConsistencyReportsRootTableItemsIdSignatureMis "itemClassName": "Particle", } ] + +def test_ValidateProjectPostgresqlConsistencyReportsRootTableItemsValueSignatureMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + rootTablesCount=1, + rootTableId=500, + rootTableItemsCount=3, + rootTableMaxItemId=30, + itemsIdSignature="items-10-20-30", + rootTableItemsIdSignature="items-10-20-30", + itemsValueSignature="values-original", + rootTableItemsValueSignature="values-corrupted", + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootTableMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "rootTableId": 500, + "fields": ["rootTableItemsValueSignature"], + "rootTablesCount": 1, + "itemsTableCount": 3, + "rootTableItemsCount": 3, + "maxItemIdFromItems": 30, + "rootTableMaxItemId": 30, + "setColumnsCount": 2, + "rootTableColumnsCount": 2, + "itemsValueSignature": "values-original", + "rootTableItemsValueSignature": "values-corrupted", + "itemClassName": "Particle", + } + ] From c341faf6619b14eed62aafb22fa1a72b257897dc Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 27 Jun 2026 23:16:16 +0200 Subject: [PATCH 213/278] feat(consistency): validate flat set properties signature --- .../services/project_consistency_service.py | 44 +++++ app/backend/api/services/project_service.py | 50 ++++++ .../test_project_service_consistency.py | 152 ++++++++++++++++++ 3 files changed, 246 insertions(+) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 9b51b020..debeee8a 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1017,6 +1017,7 @@ def comparePostgresqlOutputPayloads( flatSetMaxItemIdMismatches = [] flatSetRootTableMismatches = [] flatSetColumnsCountMismatches = [] + flatSetPropertiesMismatches = [] for protocolId in sorted(persistedOutputsByProtocolId.keys(), key=self.protocolSortKey): outputsByName = persistedOutputsByProtocolId.get(protocolId, {}) @@ -1109,6 +1110,43 @@ def comparePostgresqlOutputPayloads( "itemClassName": payload.get("itemClassName"), }) + propertiesPayloadCount = self.toOptionalInt(payload.get("propertiesPayloadCount")) + setPropertiesCount = self.toOptionalInt(payload.get("setPropertiesCount")) + propertiesPayloadSignature = payload.get("propertiesPayloadSignature") or [] + setPropertiesSignature = payload.get("setPropertiesSignature") or [] + + propertiesChangedFields = [] + + if ( + propertiesPayloadCount is not None + and setPropertiesCount is not None + and propertiesPayloadCount != setPropertiesCount + ): + propertiesChangedFields.append("setPropertiesCount") + + if ( + propertiesPayloadSignature + and setPropertiesSignature + and propertiesPayloadSignature != setPropertiesSignature + ): + propertiesChangedFields.append("setPropertiesSignature") + + if propertiesChangedFields: + flatSetPropertiesMismatches.append({ + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": mapperKind, + "className": payload.get("className"), + "setId": payload.get("setId"), + "rootObjectId": payload.get("rootObjectId"), + "fields": propertiesChangedFields, + "propertiesPayloadCount": propertiesPayloadCount, + "setPropertiesCount": setPropertiesCount, + "propertiesPayloadSignature": propertiesPayloadSignature, + "setPropertiesSignature": setPropertiesSignature, + "itemClassName": payload.get("itemClassName"), + }) + rootTablesCount = self.toOptionalInt(payload.get("rootTablesCount")) rootTableId = self.toOptionalInt(payload.get("rootTableId")) rootTableItemsCount = self.toOptionalInt(payload.get("rootTableItemsCount")) @@ -1230,6 +1268,7 @@ def comparePostgresqlOutputPayloads( "postgresqlFlatSetMaxItemIdMismatches": flatSetMaxItemIdMismatches, "postgresqlFlatSetColumnsCountMismatches": flatSetColumnsCountMismatches, "postgresqlFlatSetRootTableMismatches": flatSetRootTableMismatches, + "postgresqlFlatSetPropertiesMismatches": flatSetPropertiesMismatches, } def compareSteps(self, @@ -1526,6 +1565,10 @@ def buildIssues( outputPayloadComparison["postgresqlTreeOutputsWithIncompletePayload"] ) + postgresqlFlatSetPropertiesMismatches = ( + outputPayloadComparison["postgresqlFlatSetPropertiesMismatches"] + ) + missingSteps = stepComparison["missingSteps"] extraSteps = stepComparison["extraSteps"] stepMismatches = stepComparison["stepMismatches"] @@ -1668,6 +1711,7 @@ def buildIssues( "postgresqlFlatSetMaxItemIdMismatches": postgresqlFlatSetMaxItemIdMismatches, "postgresqlFlatSetColumnsCountMismatches": postgresqlFlatSetColumnsCountMismatches, "postgresqlFlatSetRootTableMismatches": postgresqlFlatSetRootTableMismatches, + "postgresqlFlatSetPropertiesMismatches": postgresqlFlatSetPropertiesMismatches, } def buildSummary( diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index a637e4cd..4da6d8bb 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2037,6 +2037,10 @@ def toOptionalInt(value: Any) -> Optional[int]: root_table_stats."rootTableItemsValueSignature" AS "rootTableItemsValueSignature", COALESCE(root_table_columns_stats."rootTableColumnsCount", 0) AS "rootTableColumnsCount", root_table_columns_stats."rootTableColumnsSignature" AS "rootTableColumnsSignature", + COALESCE(properties_payload_stats."propertiesPayloadCount", 0) AS "propertiesPayloadCount", + properties_payload_stats."propertiesPayloadSignature" AS "propertiesPayloadSignature", + COALESCE(set_properties_stats."setPropertiesCount", 0) AS "setPropertiesCount", + set_properties_stats."setPropertiesSignature" AS "setPropertiesSignature", s."createdAt", s."updatedAt" FROM scipion_sets s @@ -2137,6 +2141,48 @@ def toOptionalInt(value: Any) -> Optional[int]: GROUP BY t."setId" ) root_table_columns_stats ON root_table_columns_stats."setId" = s.id + LEFT JOIN ( + SELECT + s2.id AS "setId", + COUNT(*)::int AS "propertiesPayloadCount", + jsonb_agg( + jsonb_build_object( + 'key', stable_keys.key, + 'value', s2.properties ->> stable_keys.key + ) + ORDER BY stable_keys.key ASC + ) AS "propertiesPayloadSignature" + FROM scipion_sets s2 + CROSS JOIN ( + VALUES + ('columnsCount'), + ('itemsCount'), + ('nestedTablesVersion') + ) AS stable_keys(key) + WHERE s2.properties ? stable_keys.key + GROUP BY s2.id + ) properties_payload_stats + ON properties_payload_stats."setId" = s.id + LEFT JOIN ( + SELECT + "setId", + COUNT(*)::int AS "setPropertiesCount", + jsonb_agg( + jsonb_build_object( + 'key', key, + 'value', value + ) + ORDER BY key ASC + ) AS "setPropertiesSignature" + FROM scipion_set_properties + WHERE key IN ( + 'columnsCount', + 'itemsCount', + 'nestedTablesVersion' + ) + GROUP BY "setId" + ) set_properties_stats + ON set_properties_stats."setId" = s.id WHERE s."projectId" = %s ORDER BY p."protocolId", s."outputName" """, @@ -2174,6 +2220,10 @@ def toOptionalInt(value: Any) -> Optional[int]: "rootTableItemsValueSignature": row.get("rootTableItemsValueSignature"), "rootTableColumnsCount": toOptionalInt(row.get("rootTableColumnsCount")), "rootTableColumnsSignature": row.get("rootTableColumnsSignature") or [], + "propertiesPayloadCount": toOptionalInt(row.get("propertiesPayloadCount")), + "propertiesPayloadSignature": row.get("propertiesPayloadSignature") or [], + "setPropertiesCount": toOptionalInt(row.get("setPropertiesCount")), + "setPropertiesSignature": row.get("setPropertiesSignature") or [], "lastSyncAt": properties.get("lastSyncAt") if isinstance(properties, dict) else None, "lastCheckedAt": properties.get("lastCheckedAt") if isinstance(properties, dict) else None, "skippedLastSync": properties.get("skippedLastSync") if isinstance(properties, dict) else None, diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index b76398ee..37c016b2 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -253,6 +253,10 @@ def makeSetRow( rootTableItemsIdSignature=None, itemsValueSignature="values-1-2-3", rootTableItemsValueSignature=None, + propertiesPayloadSignature=None, + setPropertiesSignature=None, + propertiesPayloadCount=None, + setPropertiesCount=None, ): defaultColumnsSignature = [ { @@ -329,6 +333,42 @@ def makeSetRow( else rootTableItemsValueSignature ) + defaultPropertiesSignature = [ + { + "key": "columnsCount", + "value": str(columnsCount), + }, + { + "key": "itemsCount", + "value": str(itemsCount), + }, + { + "key": "nestedTablesVersion", + "value": "14", + }, + ] + + resolvedPropertiesPayloadSignature = ( + defaultPropertiesSignature + if propertiesPayloadSignature is None + else propertiesPayloadSignature + ) + resolvedSetPropertiesSignature = ( + resolvedPropertiesPayloadSignature + if setPropertiesSignature is None + else setPropertiesSignature + ) + resolvedPropertiesPayloadCount = ( + len(resolvedPropertiesPayloadSignature) + if propertiesPayloadCount is None + else propertiesPayloadCount + ) + resolvedSetPropertiesCount = ( + len(resolvedSetPropertiesSignature) + if setPropertiesCount is None + else setPropertiesCount + ) + return { "protocolId": str(protocolId), "id": 100, @@ -357,6 +397,10 @@ def makeSetRow( "updatedAt": None, "itemsValueSignature": itemsValueSignature, "rootTableItemsValueSignature": resolvedRootTableItemsValueSignature, + "propertiesPayloadCount": resolvedPropertiesPayloadCount, + "propertiesPayloadSignature": resolvedPropertiesPayloadSignature, + "setPropertiesCount": resolvedSetPropertiesCount, + "setPropertiesSignature": resolvedSetPropertiesSignature, } @@ -496,6 +540,7 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "postgresqlFlatSetItemsCountMismatches": [], "postgresqlFlatSetRootTableMismatches": [], "postgresqlFlatSetColumnsCountMismatches": [], + "postgresqlFlatSetPropertiesMismatches": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -626,6 +671,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "postgresqlFlatSetItemsCountMismatches": [], "postgresqlFlatSetRootTableMismatches": [], "postgresqlFlatSetColumnsCountMismatches": [], + "postgresqlFlatSetPropertiesMismatches": [], } @@ -2653,3 +2699,109 @@ def test_ValidateProjectPostgresqlConsistencyReportsRootTableItemsValueSignature "itemClassName": "Particle", } ] + +def test_ValidateProjectPostgresqlConsistencyReportsFlatSetPropertiesMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + propertiesPayloadSignature = [ + { + "key": "columnsCount", + "value": "2", + }, + { + "key": "itemsCount", + "value": "3", + }, + { + "key": "nestedTablesVersion", + "value": "14", + }, + ] + setPropertiesSignature = [ + { + "key": "columnsCount", + "value": "2", + }, + { + "key": "itemsCount", + "value": "4", + }, + { + "key": "nestedTablesVersion", + "value": "14", + }, + ] + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + propertiesPayloadSignature=propertiesPayloadSignature, + setPropertiesSignature=setPropertiesSignature, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetPropertiesMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "fields": ["setPropertiesSignature"], + "propertiesPayloadCount": 3, + "setPropertiesCount": 3, + "propertiesPayloadSignature": propertiesPayloadSignature, + "setPropertiesSignature": setPropertiesSignature, + "itemClassName": "Particle", + } + ] \ No newline at end of file From 6e533542224c8943db806be279ba64545fc1bb98 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 27 Jun 2026 23:30:02 +0200 Subject: [PATCH 214/278] feat(consistency): validate flat set root object --- .../services/project_consistency_service.py | 81 +++++++ app/backend/api/services/project_service.py | 18 ++ .../test_project_service_consistency.py | 211 ++++++++++++++++++ 3 files changed, 310 insertions(+) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index debeee8a..7107c0f7 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1008,6 +1008,7 @@ def compareOutputs( def comparePostgresqlOutputPayloads( self, postgresqlSnapshot: Dict[str, Any], + projectId: Optional[int] = None, ) -> Dict[str, Any]: persistedOutputsByProtocolId = postgresqlSnapshot["outputsByProtocolId"] @@ -1018,6 +1019,7 @@ def comparePostgresqlOutputPayloads( flatSetRootTableMismatches = [] flatSetColumnsCountMismatches = [] flatSetPropertiesMismatches = [] + flatSetRootObjectMismatches = [] for protocolId in sorted(persistedOutputsByProtocolId.keys(), key=self.protocolSortKey): outputsByName = persistedOutputsByProtocolId.get(protocolId, {}) @@ -1147,6 +1149,79 @@ def comparePostgresqlOutputPayloads( "itemClassName": payload.get("itemClassName"), }) + protocolDbId = self.toOptionalInt(payload.get("protocolDbId")) + setId = self.toOptionalInt(payload.get("setId")) + rootObjectId = self.toOptionalInt(payload.get("rootObjectId")) + rootObjectDbId = self.toOptionalInt(payload.get("rootObjectDbId")) + rootObjectProjectId = self.toOptionalInt(payload.get("rootObjectProjectId")) + rootObjectProtocolDbId = self.toOptionalInt(payload.get("rootObjectProtocolDbId")) + rootObjectParentObjectId = self.toOptionalInt(payload.get("rootObjectParentObjectId")) + rootObjectName = self.normalizeOptionalText(payload.get("rootObjectName")) + rootObjectPath = self.normalizeOptionalText(payload.get("rootObjectPath")) + rootObjectClassName = self.normalizeOptionalText(payload.get("rootObjectClassName")) + setClassName = self.normalizeOptionalText(payload.get("className")) + + rootObjectChangedFields = [] + + if rootObjectId is not None and rootObjectDbId is None: + rootObjectChangedFields.append("rootObjectMissing") + else: + if ( + rootObjectProjectId is not None + and projectId is not None + and rootObjectProjectId != projectId + ): + rootObjectChangedFields.append("rootObjectProjectId") + + if ( + rootObjectProtocolDbId is not None + and protocolDbId is not None + and rootObjectProtocolDbId != protocolDbId + ): + rootObjectChangedFields.append("rootObjectProtocolDbId") + + if rootObjectParentObjectId is not None: + rootObjectChangedFields.append("rootObjectParentObjectId") + + if ( + rootObjectName is not None + and str(outputName) != rootObjectName + ): + rootObjectChangedFields.append("rootObjectName") + + if ( + rootObjectPath is not None + and str(outputName) != rootObjectPath + ): + rootObjectChangedFields.append("rootObjectPath") + + if ( + setClassName is not None + and rootObjectClassName is not None + and setClassName != rootObjectClassName + ): + rootObjectChangedFields.append("rootObjectClassName") + + if rootObjectChangedFields: + flatSetRootObjectMismatches.append({ + "protocolId": self.normalizeProtocolId(protocolId), + "outputName": str(outputName), + "mapperKind": mapperKind, + "className": payload.get("className"), + "setId": setId, + "rootObjectId": rootObjectId, + "fields": rootObjectChangedFields, + "protocolDbId": protocolDbId, + "rootObjectDbId": rootObjectDbId, + "rootObjectProjectId": rootObjectProjectId, + "rootObjectProtocolDbId": rootObjectProtocolDbId, + "rootObjectParentObjectId": rootObjectParentObjectId, + "rootObjectName": rootObjectName, + "rootObjectPath": rootObjectPath, + "rootObjectClassName": rootObjectClassName, + "itemClassName": payload.get("itemClassName"), + }) + rootTablesCount = self.toOptionalInt(payload.get("rootTablesCount")) rootTableId = self.toOptionalInt(payload.get("rootTableId")) rootTableItemsCount = self.toOptionalInt(payload.get("rootTableItemsCount")) @@ -1269,6 +1344,7 @@ def comparePostgresqlOutputPayloads( "postgresqlFlatSetColumnsCountMismatches": flatSetColumnsCountMismatches, "postgresqlFlatSetRootTableMismatches": flatSetRootTableMismatches, "postgresqlFlatSetPropertiesMismatches": flatSetPropertiesMismatches, + "postgresqlFlatSetRootObjectMismatches": flatSetRootObjectMismatches, } def compareSteps(self, @@ -1608,6 +1684,10 @@ def buildIssues( outputPayloadComparison["postgresqlFlatSetRootTableMismatches"] ) + postgresqlFlatSetRootObjectMismatches = ( + outputPayloadComparison["postgresqlFlatSetRootObjectMismatches"] + ) + return { "missingProtocols": [ { @@ -1712,6 +1792,7 @@ def buildIssues( "postgresqlFlatSetColumnsCountMismatches": postgresqlFlatSetColumnsCountMismatches, "postgresqlFlatSetRootTableMismatches": postgresqlFlatSetRootTableMismatches, "postgresqlFlatSetPropertiesMismatches": postgresqlFlatSetPropertiesMismatches, + "postgresqlFlatSetRootObjectMismatches": postgresqlFlatSetRootObjectMismatches, } def buildSummary( diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 4da6d8bb..a0dc25d2 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2016,6 +2016,7 @@ def toOptionalInt(value: Any) -> Optional[int]: setRows = mapper.db.fetchAll( """ SELECT + p.id AS "protocolDbId", p."protocolId", s.id, s."objectId", @@ -2023,6 +2024,13 @@ def toOptionalInt(value: Any) -> Optional[int]: s."setClassName", s."itemClassName", s.properties, + root_object.id AS "rootObjectDbId", + root_object."projectId" AS "rootObjectProjectId", + root_object."protocolDbId" AS "rootObjectProtocolDbId", + root_object."parentObjectId" AS "rootObjectParentObjectId", + root_object.name AS "rootObjectName", + root_object.path AS "rootObjectPath", + root_object."className" AS "rootObjectClassName", COALESCE(items_stats."itemsTableCount", 0) AS "itemsTableCount", items_stats."maxItemIdFromItems" AS "maxItemIdFromItems", items_stats."itemsIdSignature" AS "itemsIdSignature", @@ -2046,6 +2054,8 @@ def toOptionalInt(value: Any) -> Optional[int]: FROM scipion_sets s JOIN protocols p ON p.id = s."protocolDbId" + LEFT JOIN scipion_objects root_object + ON root_object.id = s."objectId" LEFT JOIN ( SELECT "setId", @@ -2200,7 +2210,15 @@ def toOptionalInt(value: Any) -> Optional[int]: result.setdefault(protocolId, {})[outputName] = { "mapperKind": "flat_set", "setId": row.get("id"), + "protocolDbId": toOptionalInt(row.get("protocolDbId")), "rootObjectId": row.get("objectId"), + "rootObjectDbId": toOptionalInt(row.get("rootObjectDbId")), + "rootObjectProjectId": toOptionalInt(row.get("rootObjectProjectId")), + "rootObjectProtocolDbId": toOptionalInt(row.get("rootObjectProtocolDbId")), + "rootObjectParentObjectId": toOptionalInt(row.get("rootObjectParentObjectId")), + "rootObjectName": row.get("rootObjectName"), + "rootObjectPath": row.get("rootObjectPath"), + "rootObjectClassName": row.get("rootObjectClassName"), "className": row.get("setClassName"), "itemClassName": row.get("itemClassName"), "itemsCount": toOptionalInt(properties.get("itemsCount")) if isinstance(properties, dict) else None, diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 37c016b2..5afb0207 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -257,6 +257,14 @@ def makeSetRow( setPropertiesSignature=None, propertiesPayloadCount=None, setPropertiesCount=None, + protocolDbId=1000, + rootObjectDbId=None, + rootObjectProjectId=1, + rootObjectProtocolDbId=None, + rootObjectParentObjectId=None, + rootObjectName=None, + rootObjectPath=None, + rootObjectClassName=None, ): defaultColumnsSignature = [ { @@ -369,6 +377,22 @@ def makeSetRow( else setPropertiesCount ) + resolvedRootObjectDbId = ( + 200 if rootObjectDbId is None else rootObjectDbId + ) + resolvedRootObjectProtocolDbId = ( + protocolDbId if rootObjectProtocolDbId is None else rootObjectProtocolDbId + ) + resolvedRootObjectName = ( + outputName if rootObjectName is None else rootObjectName + ) + resolvedRootObjectPath = ( + outputName if rootObjectPath is None else rootObjectPath + ) + resolvedRootObjectClassName = ( + setClassName if rootObjectClassName is None else rootObjectClassName + ) + return { "protocolId": str(protocolId), "id": 100, @@ -401,6 +425,14 @@ def makeSetRow( "propertiesPayloadSignature": resolvedPropertiesPayloadSignature, "setPropertiesCount": resolvedSetPropertiesCount, "setPropertiesSignature": resolvedSetPropertiesSignature, + "protocolDbId": protocolDbId, + "rootObjectDbId": resolvedRootObjectDbId, + "rootObjectProjectId": rootObjectProjectId, + "rootObjectProtocolDbId": resolvedRootObjectProtocolDbId, + "rootObjectParentObjectId": rootObjectParentObjectId, + "rootObjectName": resolvedRootObjectName, + "rootObjectPath": resolvedRootObjectPath, + "rootObjectClassName": resolvedRootObjectClassName, } @@ -541,6 +573,7 @@ def test_ValidateProjectPostgresqlConsistencyReturnsOkWhenRuntimeAndDbMatch( "postgresqlFlatSetRootTableMismatches": [], "postgresqlFlatSetColumnsCountMismatches": [], "postgresqlFlatSetPropertiesMismatches": [], + "postgresqlFlatSetRootObjectMismatches": [], } assert currentProject.lastRefresh is False assert currentProject.lastCheckPids is False @@ -672,6 +705,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsProtocolAndDependencyMismatc "postgresqlFlatSetRootTableMismatches": [], "postgresqlFlatSetColumnsCountMismatches": [], "postgresqlFlatSetPropertiesMismatches": [], + "postgresqlFlatSetRootObjectMismatches": [], } @@ -2804,4 +2838,181 @@ def test_ValidateProjectPostgresqlConsistencyReportsFlatSetPropertiesMismatches( "setPropertiesSignature": setPropertiesSignature, "itemClassName": "Particle", } + ] + + +def test_ValidateProjectPostgresqlConsistencyReportsFlatSetRootObjectMismatches( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "id": 1000, + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + protocolDbId=1000, + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + rootObjectProjectId=2, + rootObjectProtocolDbId=9999, + rootObjectParentObjectId=123, + rootObjectName="wrongName", + rootObjectPath="wrongPath", + rootObjectClassName="WrongSetClass", + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootObjectMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "fields": [ + "rootObjectProjectId", + "rootObjectProtocolDbId", + "rootObjectParentObjectId", + "rootObjectName", + "rootObjectPath", + "rootObjectClassName", + ], + "protocolDbId": 1000, + "rootObjectDbId": 200, + "rootObjectProjectId": 2, + "rootObjectProtocolDbId": 9999, + "rootObjectParentObjectId": 123, + "rootObjectName": "wrongName", + "rootObjectPath": "wrongPath", + "rootObjectClassName": "WrongSetClass", + "itemClassName": "Particle", + } + ] + +def test_ValidateProjectPostgresqlConsistencyReportsMissingFlatSetRootObject( + service, + monkeypatch, + tmp_path, +): + currentProject = FakeCurrentProject( + nodes={ + "PROJECT": FakeRunNode("PROJECT"), + "10": FakeRunNode( + "10", + status="finished", + parents=["PROJECT"], + ), + } + ) + currentProject.nodes["10"].run.outputs = [ + ("outputParticles", FakeOutput("SetOfParticles", itemsCount=3)), + ] + patchRuntimeProject(service, monkeypatch, currentProject) + + mapper = FakeMapper( + projectRow={ + "id": 1, + "ownerId": 7, + "name": str(tmp_path), + }, + protocolRows=[ + { + "id": 1000, + "protocolId": "10", + "status": "finished", + }, + ], + adjacencyMap={ + "10": {"parents": [], "children": []}, + }, + setRows=[ + makeSetRow( + protocolId="10", + protocolDbId=1000, + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=3, + itemsTableCount=3, + maxItemId=30, + maxItemIdFromItems=30, + rootObjectDbId=None, + ), + ], + ) + + result = service.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=1, + currentUser={"id": 7}, + refresh=True, + checkPid=True, + ) + + assert result["ok"] is False + assert result["summary"]["issues"] == 1 + assert result["issues"]["postgresqlFlatSetRootObjectMismatches"] == [ + { + "protocolId": "10", + "outputName": "outputParticles", + "mapperKind": "flat_set", + "className": "SetOfParticles", + "setId": 100, + "rootObjectId": 200, + "fields": ["rootObjectMissing"], + "protocolDbId": 1000, + "rootObjectDbId": None, + "rootObjectProjectId": 1, + "rootObjectProtocolDbId": 1000, + "rootObjectParentObjectId": None, + "rootObjectName": "outputParticles", + "rootObjectPath": "outputParticles", + "rootObjectClassName": "SetOfParticles", + "itemClassName": "Particle", + } ] \ No newline at end of file From 340ab4b4b61e2bb93f9f298a01d1f047cd970f39 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 27 Jun 2026 23:45:03 +0200 Subject: [PATCH 215/278] feat(consistency): validate flat set root object --- .../services/project_consistency_service.py | 3 + .../test_project_service_consistency.py | 55 +++++++------------ 2 files changed, 22 insertions(+), 36 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 7107c0f7..a398e431 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1825,6 +1825,7 @@ def buildComparisons( runtimeSnapshot: Dict[str, Any], postgresqlSnapshot: Dict[str, Any], derivedSets: Dict[str, Any], + projectId: Optional[int] = None, ) -> Dict[str, Dict[str, Any]]: protocolComparison = self.compareProtocols( runtimeSnapshot=runtimeSnapshot, @@ -1845,6 +1846,7 @@ def buildComparisons( outputPayloadComparison = self.comparePostgresqlOutputPayloads( postgresqlSnapshot=postgresqlSnapshot, + projectId=projectId, ) stepComparison = self.compareSteps( @@ -1925,6 +1927,7 @@ def validateProjectPostgresqlConsistency( runtimeSnapshot=runtimeSnapshot, postgresqlSnapshot=postgresqlSnapshot, derivedSets=derivedSets, + projectId=projectId, ) issues = self.buildIssues( diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 5afb0207..031453d2 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -743,17 +743,12 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputMismatches( "10": {"parents": [], "children": []}, }, setRows=[ - { - "protocolId": "10", - "id": 100, - "objectId": 200, - "outputName": "outputParticles", - "setClassName": "SetOfParticles", - "itemClassName": "Particle", - "properties": {"itemsCount": 12}, - "createdAt": None, - "updatedAt": None, - } + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=12, + ), ], treeRows=[ { @@ -1525,17 +1520,12 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputClassMismatches( "10": {"parents": [], "children": []}, }, setRows=[ - { - "protocolId": "10", - "id": 100, - "objectId": 200, - "outputName": "outputParticles", - "setClassName": "Volume", - "itemClassName": "Particle", - "properties": {"itemsCount": 12}, - "createdAt": None, - "updatedAt": None, - } + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="Volume", + itemsCount=12, + ), ], ) @@ -1685,19 +1675,12 @@ def test_ValidateProjectPostgresqlConsistencyReportsOutputItemsCountMismatches( "10": {"parents": [], "children": []}, }, setRows=[ - { - "protocolId": "10", - "id": 100, - "objectId": 200, - "outputName": "outputParticles", - "setClassName": "SetOfParticles", - "itemClassName": "Particle", - "properties": { - "itemsCount": 10, - }, - "createdAt": None, - "updatedAt": None, - } + makeSetRow( + protocolId="10", + outputName="outputParticles", + setClassName="SetOfParticles", + itemsCount=10, + ), ], ) @@ -2981,7 +2964,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsMissingFlatSetRootObject( itemsTableCount=3, maxItemId=30, maxItemIdFromItems=30, - rootObjectDbId=None, + rootObjectDbId=200, ), ], ) From 495304668bbcba7b43fed8c167e5d3e669bf7966 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 27 Jun 2026 23:52:50 +0200 Subject: [PATCH 216/278] Fixing a tests --- .../test_project_service_consistency.py | 61 +++++++++++-------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 031453d2..ad49627b 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -258,13 +258,14 @@ def makeSetRow( propertiesPayloadCount=None, setPropertiesCount=None, protocolDbId=1000, - rootObjectDbId=None, + rootObjectDbId=True, rootObjectProjectId=1, rootObjectProtocolDbId=None, rootObjectParentObjectId=None, rootObjectName=None, rootObjectPath=None, rootObjectClassName=None, + rootObjectMissing=False, ): defaultColumnsSignature = [ { @@ -377,21 +378,32 @@ def makeSetRow( else setPropertiesCount ) - resolvedRootObjectDbId = ( - 200 if rootObjectDbId is None else rootObjectDbId - ) - resolvedRootObjectProtocolDbId = ( - protocolDbId if rootObjectProtocolDbId is None else rootObjectProtocolDbId - ) - resolvedRootObjectName = ( - outputName if rootObjectName is None else rootObjectName - ) - resolvedRootObjectPath = ( - outputName if rootObjectPath is None else rootObjectPath - ) - resolvedRootObjectClassName = ( - setClassName if rootObjectClassName is None else rootObjectClassName - ) + if rootObjectMissing: + resolvedRootObjectDbId = None + resolvedRootObjectProjectId = None + resolvedRootObjectProtocolDbId = None + resolvedRootObjectParentObjectId = None + resolvedRootObjectName = None + resolvedRootObjectPath = None + resolvedRootObjectClassName = None + else: + resolvedRootObjectDbId = ( + 200 if rootObjectDbId is None else rootObjectDbId + ) + resolvedRootObjectProjectId = rootObjectProjectId + resolvedRootObjectProtocolDbId = ( + protocolDbId if rootObjectProtocolDbId is None else rootObjectProtocolDbId + ) + resolvedRootObjectParentObjectId = rootObjectParentObjectId + resolvedRootObjectName = ( + outputName if rootObjectName is None else rootObjectName + ) + resolvedRootObjectPath = ( + outputName if rootObjectPath is None else rootObjectPath + ) + resolvedRootObjectClassName = ( + setClassName if rootObjectClassName is None else rootObjectClassName + ) return { "protocolId": str(protocolId), @@ -427,9 +439,9 @@ def makeSetRow( "setPropertiesSignature": resolvedSetPropertiesSignature, "protocolDbId": protocolDbId, "rootObjectDbId": resolvedRootObjectDbId, - "rootObjectProjectId": rootObjectProjectId, + "rootObjectProjectId": resolvedRootObjectProjectId, "rootObjectProtocolDbId": resolvedRootObjectProtocolDbId, - "rootObjectParentObjectId": rootObjectParentObjectId, + "rootObjectParentObjectId": resolvedRootObjectParentObjectId, "rootObjectName": resolvedRootObjectName, "rootObjectPath": resolvedRootObjectPath, "rootObjectClassName": resolvedRootObjectClassName, @@ -2918,6 +2930,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsFlatSetRootObjectMismatches( } ] + def test_ValidateProjectPostgresqlConsistencyReportsMissingFlatSetRootObject( service, monkeypatch, @@ -2964,7 +2977,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsMissingFlatSetRootObject( itemsTableCount=3, maxItemId=30, maxItemIdFromItems=30, - rootObjectDbId=200, + rootObjectDbId=True, ), ], ) @@ -2990,12 +3003,12 @@ def test_ValidateProjectPostgresqlConsistencyReportsMissingFlatSetRootObject( "fields": ["rootObjectMissing"], "protocolDbId": 1000, "rootObjectDbId": None, - "rootObjectProjectId": 1, - "rootObjectProtocolDbId": 1000, + "rootObjectProjectId": None, + "rootObjectProtocolDbId": None, "rootObjectParentObjectId": None, - "rootObjectName": "outputParticles", - "rootObjectPath": "outputParticles", - "rootObjectClassName": "SetOfParticles", + "rootObjectName": None, + "rootObjectPath": None, + "rootObjectClassName": None, "itemClassName": "Particle", } ] \ No newline at end of file From 06c06ccd2a4f469fee0cead44d3dc9d279c5775e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sat, 27 Jun 2026 23:56:28 +0200 Subject: [PATCH 217/278] feat(consistency): validate flat set root object --- .../backend/api/services/test_project_service_consistency.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index ad49627b..76fd5da0 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -258,7 +258,7 @@ def makeSetRow( propertiesPayloadCount=None, setPropertiesCount=None, protocolDbId=1000, - rootObjectDbId=True, + rootObjectDbId=None, rootObjectProjectId=1, rootObjectProtocolDbId=None, rootObjectParentObjectId=None, @@ -2977,7 +2977,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsMissingFlatSetRootObject( itemsTableCount=3, maxItemId=30, maxItemIdFromItems=30, - rootObjectDbId=True, + rootObjectMissing=True, ), ], ) From 226842ce375d2210b563b8bdec13b688a6935cb9 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sun, 28 Jun 2026 00:45:13 +0200 Subject: [PATCH 218/278] perf(projects): use lightweight persisted output summaries --- app/backend/api/routers/project_router.py | 12 +- app/backend/api/services/project_service.py | 326 +++++++++++++++----- 2 files changed, 259 insertions(+), 79 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 963e9101..f8fb90eb 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -156,12 +156,20 @@ def importProject( @router.get("/{projectId}", response_model=Any) def getProject( - projectId: int, # id in the DB + projectId: int, + validateConsistency: bool = Query(False), currentUser=Depends(getCurrentUser), mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=True) + project = service.getProjectById( + mapper, + projectId, + currentUser, + refresh=True, + checkPid=True, + validateConsistency=validateConsistency, + ) if not project: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") return project diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index a0dc25d2..71633c40 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -1868,7 +1868,16 @@ def revokeProjectShareForUser( return {"success": True} - def getProjectById(self, mapper: PostgresqlFlatMapper, projectId: int, currentUser, refresh=True, checkPid=True) -> Optional[dict]: + def getProjectById( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser, + refresh=True, + checkPid=True, + validateConsistency: bool = False, + failOnConsistencyError: bool = False, + ) -> Optional[dict]: # Retrieve project from PostgreSQL using the mapper userId = currentUser["id"] dbProj = mapper.getProject(projectId=projectId, userId=userId) @@ -1878,7 +1887,48 @@ def getProjectById(self, mapper: PostgresqlFlatMapper, projectId: int, currentUs if not os.path.exists(projectPath): return None - return self.loadProject(dbProj, mapper, refresh=refresh, checkPid=checkPid) + project = self.loadProject(dbProj, mapper, refresh=refresh, checkPid=checkPid) + + if validateConsistency: + try: + from app.backend.api.services.project_consistency_service import ( + ProjectConsistencyService, + ) + + consistencyService = ProjectConsistencyService(self) + project["postgresqlConsistency"] = ( + consistencyService.validateProjectPostgresqlConsistency( + mapper=mapper, + projectId=projectId, + currentUser=currentUser, + refresh=refresh, + checkPid=checkPid, + ) + ) + except HTTPException: + if failOnConsistencyError: + raise + logger.exception( + "PostgreSQL consistency validation failed. projectId=%s", + projectId, + ) + project["postgresqlConsistency"] = { + "ok": False, + "error": "PostgreSQL consistency validation failed", + } + except Exception as e: + if failOnConsistencyError: + raise + logger.exception( + "PostgreSQL consistency validation failed. projectId=%s", + projectId, + ) + project["postgresqlConsistency"] = { + "ok": False, + "error": str(e), + } + + return project def getProjectEffectiveSettings( self, @@ -1998,6 +2048,117 @@ def _buildProjectThumbnailVersion( return f"{projectId}:{updatedText}:{protocolsCount}:{runsMtime}" + def _loadPersistedOutputSummariesByProtocolId( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + ) -> Dict[str, Dict[str, Dict[str, Any]]]: + def toOptionalInt(value: Any) -> Optional[int]: + if value is None or value == "": + return None + try: + return int(value) + except Exception: + return None + + result: Dict[str, Dict[str, Dict[str, Any]]] = {} + + setRows = mapper.db.fetchAll( + """ + SELECT + p."protocolId", + s.id, + s."objectId", + s."outputName", + s."setClassName", + s."itemClassName", + s.properties, + s."createdAt", + s."updatedAt" + FROM scipion_sets s + JOIN protocols p + ON p.id = s."protocolDbId" + WHERE s."projectId" = %s + ORDER BY p."protocolId", s."outputName" + """, + (projectId,), + ) + + for row in setRows: + protocolId = str(row.get("protocolId")) + outputName = str(row.get("outputName") or "") + if not protocolId or not outputName: + continue + + properties = row.get("properties") or {} + + result.setdefault(protocolId, {})[outputName] = { + "mapperKind": "flat_set", + "setId": row.get("id"), + "rootObjectId": row.get("objectId"), + "className": row.get("setClassName"), + "itemClassName": row.get("itemClassName"), + "itemsCount": toOptionalInt(properties.get("itemsCount")) if isinstance(properties, dict) else None, + "maxItemId": toOptionalInt(properties.get("maxItemId")) if isinstance(properties, dict) else None, + "columnsCount": toOptionalInt(properties.get("columnsCount")) if isinstance(properties, dict) else None, + "lastSyncAt": properties.get("lastSyncAt") if isinstance(properties, dict) else None, + "lastCheckedAt": properties.get("lastCheckedAt") if isinstance(properties, dict) else None, + "skippedLastSync": properties.get("skippedLastSync") if isinstance(properties, dict) else None, + "createdAt": row.get("createdAt"), + "updatedAt": row.get("updatedAt"), + } + + treeRows = mapper.db.fetchAll( + """ + SELECT + p."protocolId", + o.id, + o."scipionObjId", + o.name, + o.path, + o."className", + o.value, + o.label, + o.comment, + o.metadata, + o."createdAt", + o."updatedAt" + FROM scipion_objects o + JOIN protocols p + ON p.id = o."protocolDbId" + WHERE o."projectId" = %s + AND o."parentObjectId" IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM scipion_sets s + WHERE s."objectId" = o.id + ) + ORDER BY p."protocolId", o.path + """, + (projectId,), + ) + + for row in treeRows: + protocolId = str(row.get("protocolId")) + outputName = str(row.get("path") or row.get("name") or "") + if not protocolId or not outputName: + continue + + result.setdefault(protocolId, {})[outputName] = { + "mapperKind": "tree", + "rootObjectId": row.get("id"), + "scipionObjId": row.get("scipionObjId"), + "className": row.get("className"), + "value": row.get("value"), + "label": row.get("label"), + "comment": row.get("comment"), + "metadata": row.get("metadata") or {}, + "createdAt": row.get("createdAt"), + "updatedAt": row.get("updatedAt"), + } + + return result + def _loadPersistedOutputsByProtocolId( self, mapper: PostgresqlFlatMapper, @@ -2057,29 +2218,35 @@ def toOptionalInt(value: Any) -> Optional[int]: LEFT JOIN scipion_objects root_object ON root_object.id = s."objectId" LEFT JOIN ( - SELECT - "setId", - COUNT(*)::int AS "itemsTableCount", - MAX("scipionItemId")::int AS "maxItemIdFromItems", - md5(string_agg("scipionItemId"::text, ',' ORDER BY "scipionItemId")) AS "itemsIdSignature", - md5( - string_agg( - jsonb_build_object( - 'scipionItemId', "scipionItemId", - 'enabled', enabled, - 'label', label, - 'comment', comment, - 'creation', creation, - 'values', "values" - )::text, - ',' - ORDER BY "scipionItemId" - ) - ) AS "itemsValueSignature" - FROM scipion_set_items - GROUP BY "setId" - ) items_stats - ON items_stats."setId" = s.id + SELECT + "setId", + COUNT(*)::int AS "itemsTableCount", + MAX("scipionItemId")::int AS "maxItemIdFromItems", + md5( + string_agg( + "scipionItemId"::text, + ',' + ORDER BY "scipionItemId" + ) + ) AS "itemsIdSignature", + md5( + string_agg( + jsonb_build_object( + 'scipionItemId', "scipionItemId", + 'enabled', enabled, + 'label', label, + 'comment', comment, + 'creation', creation, + 'values', "values" + )::text, + ',' + ORDER BY "scipionItemId" + ) + ) AS "itemsValueSignature" + FROM scipion_set_items + GROUP BY "setId" + ) items_stats + ON items_stats."setId" = s.id LEFT JOIN ( SELECT "setId", @@ -2097,38 +2264,43 @@ def toOptionalInt(value: Any) -> Optional[int]: ) AS "setColumnsSignature" FROM scipion_set_columns GROUP BY "setId" - ) columns_stats - ON columns_stats."setId" = s.id + ) columns_stats + ON columns_stats."setId" = s.id LEFT JOIN ( - SELECT - t."setId", - COUNT(DISTINCT t.id)::int AS "rootTablesCount", - MIN(t.id)::int AS "rootTableId", - COUNT(ti.id)::int AS "rootTableItemsCount", - MAX(ti."scipionItemId")::int AS "rootTableMaxItemId", - md5(string_agg(ti."scipionItemId"::text, ',' ORDER BY ti."scipionItemId")) - FILTER (WHERE ti.id IS NOT NULL) AS "rootTableItemsIdSignature", - md5( - string_agg( - jsonb_build_object( - 'scipionItemId', ti."scipionItemId", - 'enabled', ti.enabled, - 'label', ti.label, - 'comment', ti.comment, - 'creation', ti.creation, - 'values', ti."values" - )::text, - ',' - ORDER BY ti."scipionItemId" - ) - ) FILTER (WHERE ti.id IS NOT NULL) AS "rootTableItemsValueSignature" - FROM scipion_set_tables t - LEFT JOIN scipion_set_table_items ti - ON ti."tableId" = t.id - WHERE t."tableKind" = 'root' - GROUP BY t."setId" - ) root_table_stats - ON root_table_stats."setId" = s.id + SELECT + t."setId", + COUNT(DISTINCT t.id)::int AS "rootTablesCount", + MIN(t.id)::int AS "rootTableId", + COUNT(ti.id)::int AS "rootTableItemsCount", + MAX(ti."scipionItemId")::int AS "rootTableMaxItemId", + md5( + string_agg( + ti."scipionItemId"::text, + ',' + ORDER BY ti."scipionItemId" + ) FILTER (WHERE ti.id IS NOT NULL) + ) AS "rootTableItemsIdSignature", + md5( + string_agg( + jsonb_build_object( + 'scipionItemId', ti."scipionItemId", + 'enabled', ti.enabled, + 'label', ti.label, + 'comment', ti.comment, + 'creation', ti.creation, + 'values', ti."values" + )::text, + ',' + ORDER BY ti."scipionItemId" + ) FILTER (WHERE ti.id IS NOT NULL) + ) AS "rootTableItemsValueSignature" + FROM scipion_set_tables t + LEFT JOIN scipion_set_table_items ti + ON ti."tableId" = t.id + WHERE t."tableKind" = 'root' + GROUP BY t."setId" + ) root_table_stats + ON root_table_stats."setId" = s.id LEFT JOIN ( SELECT t."setId", @@ -2173,26 +2345,26 @@ def toOptionalInt(value: Any) -> Optional[int]: GROUP BY s2.id ) properties_payload_stats ON properties_payload_stats."setId" = s.id - LEFT JOIN ( - SELECT - "setId", - COUNT(*)::int AS "setPropertiesCount", - jsonb_agg( - jsonb_build_object( - 'key', key, - 'value', value - ) - ORDER BY key ASC - ) AS "setPropertiesSignature" - FROM scipion_set_properties - WHERE key IN ( - 'columnsCount', - 'itemsCount', - 'nestedTablesVersion' - ) - GROUP BY "setId" - ) set_properties_stats - ON set_properties_stats."setId" = s.id + LEFT JOIN ( + SELECT + "setId", + COUNT(*)::int AS "setPropertiesCount", + jsonb_agg( + jsonb_build_object( + 'key', key, + 'value', value + ) + ORDER BY key ASC + ) AS "setPropertiesSignature" + FROM scipion_set_properties + WHERE key IN ( + 'columnsCount', + 'itemsCount', + 'nestedTablesVersion' + ) + GROUP BY "setId" + ) set_properties_stats + ON set_properties_stats."setId" = s.id WHERE s."projectId" = %s ORDER BY p."protocolId", s."outputName" """, @@ -2715,7 +2887,7 @@ def normalizeStatus(value: Any) -> str: persistedOutputsByProtocolId = {} if mapper is not None: try: - persistedOutputsByProtocolId = self._loadPersistedOutputsByProtocolId( + persistedOutputsByProtocolId = self._loadPersistedOutputSummariesByProtocolId( mapper, dbProj['id'], ) From c95d778fa8ee02c5eaaefd147ac61a3b523571e4 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sun, 28 Jun 2026 12:56:59 +0200 Subject: [PATCH 219/278] fix(consistency): skip param comparison when runtime params are unavailable --- app/backend/api/routers/project_router.py | 2 +- .../api/services/project_consistency_service.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index f8fb90eb..37836045 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -157,7 +157,7 @@ def importProject( @router.get("/{projectId}", response_model=Any) def getProject( projectId: int, - validateConsistency: bool = Query(False), + validateConsistency: bool = Query(True), currentUser=Depends(getCurrentUser), mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index a398e431..7a2891ca 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -1406,6 +1406,16 @@ def compareParams( postgresqlSnapshot: Dict[str, Any], derivedSets: Dict[str, Any], ) -> Dict[str, Any]: + + if not derivedSets["runtimeParams"]: + return { + "missingParams": [], + "extraParams": [], + "paramValueMismatches": [], + "skipped": True, + "reason": "Runtime protocol params could not be extracted", + } + runtimeParamsByProtocolId = runtimeSnapshot["paramsByProtocolId"] postgresqlParamsByProtocolId = postgresqlSnapshot["paramsByProtocolId"] From 23b8b6cf9d50c15e24d70a28f0857af4ef6250ee Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sun, 28 Jun 2026 14:44:12 +0200 Subject: [PATCH 220/278] fix(consistency): ignore input refs to unavailable runtime outputs --- app/backend/api/services/project_consistency_service.py | 8 +++++++- app/backend/mapper/scipion_object_mapper.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index 7a2891ca..c04bf5c5 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -217,7 +217,7 @@ def expectedOutputMapperKind(self, className: Any) -> Optional[str]: if classNameText is None: return None - if classNameText.startswith("SetOf"): + if classNameText.startswith("SetOf") or "SetOf" in classNameText: return "flat_set" return "tree" @@ -1575,6 +1575,7 @@ def comparePostgresqlInputRefTargets( postgresqlInputRefsWithMissingParentProtocols = [] postgresqlInputRefsWithMissingParentOutputs = [] + runtimeOutputs = derivedSets.get("runtimeOutputs", set()) for key in sorted(postgresqlInputRefsByKey.keys(), key=self.inputRefSortKey): inputRef = postgresqlInputRefsByKey.get(key, {}) @@ -1595,6 +1596,11 @@ def comparePostgresqlInputRefTargets( continue if parentOutputName not in persistedOutputsByProtocolId.get(parentProtocolId, {}): + runtimeOutputKey = (parentProtocolId, parentOutputName) + + if runtimeOutputKey not in runtimeOutputs: + continue + issue = self.buildInputRef(key, inputRef) issue["missingParentOutputName"] = parentOutputName postgresqlInputRefsWithMissingParentOutputs.append(issue) diff --git a/app/backend/mapper/scipion_object_mapper.py b/app/backend/mapper/scipion_object_mapper.py index 984b76f1..e392c1c3 100644 --- a/app/backend/mapper/scipion_object_mapper.py +++ b/app/backend/mapper/scipion_object_mapper.py @@ -386,7 +386,7 @@ def _guessMapperKind(self, scipionObj: Any) -> str: className = self._getClassName(scipionObj) or "" if self._isPointer(scipionObj): return "pointer" - if className.startswith("SetOf"): + if className.startswith("SetOf") or "SetOf" in className: return "flat_set" if self._getAttributesToStore(scipionObj): return "tree" From 65192ad6dc8f1a3c868ee21930028fb43157b48c Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Sun, 28 Jun 2026 17:33:02 +0200 Subject: [PATCH 221/278] feat(projects): sync protocol steps from project snapshot --- .../services/project_consistency_service.py | 17 +- app/backend/api/services/project_service.py | 195 ++++++++++++++++++ 2 files changed, 208 insertions(+), 4 deletions(-) diff --git a/app/backend/api/services/project_consistency_service.py b/app/backend/api/services/project_consistency_service.py index c04bf5c5..920d6676 100644 --- a/app/backend/api/services/project_consistency_service.py +++ b/app/backend/api/services/project_consistency_service.py @@ -521,15 +521,15 @@ def collectRuntimeSnapshot( runtimeStepsByProtocolId.setdefault(protocolId, {}) try: - for step in protocol.loadSteps() or []: - stepIndex = self.toOptionalInt(self._safeCall(step, "getIndex", None)) + for stepPayload in self._buildProtocolStepsForPostgresql(protocol): + stepIndex = self.toOptionalInt(stepPayload.get("index")) if stepIndex is None: continue runtimeStepsByProtocolId[protocolId][stepIndex] = { "index": stepIndex, - "name": self.getStepName(step), - "status": self.normalizeStatus(self._safeCall(step, "getStatus", None)), + "name": str(stepPayload.get("name") or ""), + "status": self.normalizeStatus(stepPayload.get("status")), } except Exception: logger.debug( @@ -1358,6 +1358,15 @@ def compareSteps(self, runtimeSteps = derivedSets["runtimeSteps"] postgresqlSteps = derivedSets["postgresqlSteps"] + if not runtimeSteps: + return { + "missingSteps": [], + "extraSteps": [], + "stepMismatches": [], + "skipped": True, + "reason": "Runtime protocol steps could not be extracted", + } + missingSteps = sorted( runtimeSteps - postgresqlSteps, key=self.stepSortKey, diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 71633c40..3cb70112 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -1207,6 +1207,169 @@ def _buildProtocolInputRefsForPostgresql( return refs + def _safeProtocolStepValue(self, value: Any, default=None): + try: + if value is None: + return default + + if hasattr(value, "hasValue") and not value.hasValue(): + return default + + if hasattr(value, "get"): + try: + return value.get(default) + except TypeError: + return value.get() + + return value + except Exception: + return default + + def _safeProtocolStepCall(self, step: Any, methodName: str, default=None): + try: + method = getattr(step, methodName, None) + if not callable(method): + return default + value = method() + return value if value is not None else default + except Exception: + return default + + def _safeProtocolStepJsonValue(self, value: Any): + if value in (None, ""): + return None + + if isinstance(value, (dict, list, tuple)): + return value + + try: + return json.loads(str(value)) + except Exception: + return str(value) + + def _loadProtocolStepsForPostgresql(self, protocol: Any) -> List[Any]: + for methodName in ("loadSteps", "getSteps"): + try: + method = getattr(protocol, methodName, None) + if not callable(method): + continue + + steps = method() or [] + steps = list(steps) + if steps: + return steps + except Exception: + logger.debug( + "Could not load protocol steps using %s.", + methodName, + exc_info=True, + ) + + for attrName in ("_steps", "steps"): + try: + steps = getattr(protocol, attrName, None) + if not steps: + continue + + if isinstance(steps, dict): + steps = list(steps.values()) + else: + steps = list(steps) + + if steps: + return steps + except Exception: + logger.debug( + "Could not load protocol steps from attribute %s.", + attrName, + exc_info=True, + ) + + return [] + + def _buildProtocolStepsForPostgresql(self, protocol: Any) -> List[Dict[str, Any]]: + result: List[Dict[str, Any]] = [] + + steps = self._loadProtocolStepsForPostgresql(protocol) + if not steps: + return result + + for step in steps: + try: + stepIndex = self._safeProtocolStepCall(step, "getIndex", None) + if stepIndex is None: + continue + + elapsedSeconds = None + elapsed = self._safeProtocolStepCall(step, "getElapsedTime", None) + try: + if elapsed is not None: + elapsedSeconds = elapsed.total_seconds() + except Exception: + elapsedSeconds = None + + stepName = self._safeProtocolStepValue( + getattr(step, "funcName", None), + None, + ) + + if not stepName: + stepName = self._safeProtocolStepCall(step, "getClassName", "") + + prerequisites = [] + rawPrerequisites = self._safeProtocolStepCall( + step, + "getPrerequisites", + [], + ) + + try: + prerequisites = [ + int(prerequisite) + for prerequisite in (rawPrerequisites or []) + ] + except Exception: + prerequisites = [] + + rawArgs = self._safeProtocolStepValue( + getattr(step, "argsStr", None), + None, + ) + + needsGpu = self._safeProtocolStepCall(step, "needsGPU", None) + if needsGpu is None: + needsGpu = True + + result.append({ + "index": int(stepIndex), + "name": str(stepName or ""), + "status": self._safeProtocolStepCall(step, "getStatus", ""), + "prerequisites": prerequisites, + "args": self._safeProtocolStepJsonValue(rawArgs), + "initTime": self._safeProtocolStepValue( + getattr(step, "initTime", None), + None, + ), + "endTime": self._safeProtocolStepValue( + getattr(step, "endTime", None), + None, + ), + "elapsedSeconds": elapsedSeconds, + "error": self._safeProtocolStepCall(step, "getErrorMessage", None), + "interactive": bool( + self._safeProtocolStepCall(step, "isInteractive", False) + ), + "needsGpu": bool(needsGpu), + "event": "snapshot", + }) + except Exception: + logger.debug( + "Could not serialize protocol step for PostgreSQL.", + exc_info=True, + ) + + return result + def syncProjectProtocolsAndDependencies( self, mapper: PostgresqlFlatMapper, @@ -1232,6 +1395,10 @@ def syncProjectProtocolsAndDependencies( outputSyncDeclared: List[Dict[str, Any]] = [] outputSyncMissing: List[Dict[str, Any]] = [] + stepsSyncCount = 0 + stepsSyncProtocolsCount = 0 + stepsSyncErrors: List[Dict[str, Any]] = [] + # 1) Save all protocol nodes that are currently present in the real Scipion graph for nodeId, nodeObj in nodesDict.items(): nodeIdText = str(nodeId) @@ -1248,6 +1415,31 @@ def syncProjectProtocolsAndDependencies( protocolContext = self._buildProtocolContext(projectId, protocol) protocolDbId = mapper.saveProtocol(protocolContext) + try: + protocolSteps = self._buildProtocolStepsForPostgresql(protocol) + protocolScipionId = self._getScipionObjectId(protocol) + + if protocolSteps and protocolScipionId is not None: + mapper.replaceProtocolSteps( + projectId=projectId, + protocolDbId=int(protocolDbId), + protocolId=int(protocolScipionId), + steps=protocolSteps, + ) + + stepsSyncCount += len(protocolSteps) + stepsSyncProtocolsCount += 1 + except Exception as exc: + stepsSyncErrors.append({ + "protocolId": nodeIdText, + "error": str(exc), + }) + logger.exception( + "Failed to sync protocol steps. projectId=%s protocolId=%s", + projectId, + nodeIdText, + ) + if self._shouldRegisterProtocolOutputs(protocol): try: outputReport = self.registerOutput( @@ -1362,6 +1554,9 @@ def syncProjectProtocolsAndDependencies( "protocols": len(protocolDbIdByScipionId), "dependencies": int(savedEdges), "inputRefs": int(savedInputRefs), + "steps": int(stepsSyncCount), + "stepsProtocols": int(stepsSyncProtocolsCount), + "stepErrors": stepsSyncErrors, "outputsDeclared": len(outputSyncDeclared), "outputs": len(outputSyncResults), "outputsMissing": len(outputSyncMissing), From 834ffced20ef1eb628085a43e14ddb426f02cb14 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 09:33:00 +0200 Subject: [PATCH 222/278] feat(projects): validate postgres consistency across project mutations --- app/backend/api/routers/project_router.py | 2 +- tests/conftest.py | 13 ++++++++++- tests/integration/api/test_projects_router.py | 2 ++ .../test_project_service_consistency.py | 23 ++----------------- .../test_project_service_protocols.py | 3 +++ 5 files changed, 20 insertions(+), 23 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 37836045..f8fb90eb 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -157,7 +157,7 @@ def importProject( @router.get("/{projectId}", response_model=Any) def getProject( projectId: int, - validateConsistency: bool = Query(True), + validateConsistency: bool = Query(False), currentUser=Depends(getCurrentUser), mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), diff --git a/tests/conftest.py b/tests/conftest.py index 97452a06..e89de741 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -294,13 +294,24 @@ def listProjectWorkflows(self): raise self.listProjectWorkflowsError return self.listProjectWorkflowsResult - def getProjectById(self, mapper, projectId, currentUser, refresh=False, checkPid=False): + def getProjectById( + self, + mapper, + projectId, + currentUser, + refresh=False, + checkPid=False, + validateConsistency=False, + failOnConsistencyError=False, + ): self.lastGetProjectByIdCall = { "mapper": mapper, "projectId": projectId, "currentUser": currentUser, "refresh": refresh, "checkPid": checkPid, + "validateConsistency": validateConsistency, + "failOnConsistencyError": failOnConsistencyError, } return self.projectByIdResult diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 34856e82..44d45388 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -63,6 +63,8 @@ def test_GetProjectCallsServiceWithRefreshAndCheckPid(projectClient, fakeProject }, "refresh": True, "checkPid": True, + "validateConsistency": True, + "failOnConsistencyError": False, } diff --git a/tests/unit/backend/api/services/test_project_service_consistency.py b/tests/unit/backend/api/services/test_project_service_consistency.py index 76fd5da0..e622465f 100644 --- a/tests/unit/backend/api/services/test_project_service_consistency.py +++ b/tests/unit/backend/api/services/test_project_service_consistency.py @@ -1176,7 +1176,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( "postgresqlInputRefDependencies": 1, "runtimeParams": 0, "postgresqlParams": 0, - "issues": 5, + "issues": 3, } assert result["issues"]["missingInputRefs"] == [ @@ -1216,26 +1216,7 @@ def test_ValidateProjectPostgresqlConsistencyReportsInputRefMismatches( } ] assert result["issues"]["postgresqlInputRefsWithMissingParentProtocols"] == [] - assert result["issues"]["postgresqlInputRefsWithMissingParentOutputs"] == [ - { - "protocolId": "11", - "inputName": "inputMask", - "itemIndex": 0, - "parentProtocolId": "10", - "parentOutputName": "outputMask", - "objectClassName": "VolumeMask", - "missingParentOutputName": "outputMask", - }, - { - "protocolId": "11", - "inputName": "inputParticles", - "itemIndex": 0, - "parentProtocolId": "10", - "parentOutputName": "wrongOutput", - "objectClassName": "SetOfParticles", - "missingParentOutputName": "wrongOutput", - }, - ] + assert result["issues"]["postgresqlInputRefsWithMissingParentOutputs"] == [] assert result["issues"]["missingProtocols"] == [] assert result["issues"]["extraProtocols"] == [] diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index 2c594918..ee7aebc1 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -744,6 +744,9 @@ def replaceProjectProtocolInputRefs(self, projectId, inputRefs): "protocols": 1, "dependencies": 0, "inputRefs": 0, + "steps": 0, + "stepsProtocols": 0, + "stepErrors": [], "outputsDeclared": 5, "outputs": 2, "outputsMissing": 3, From 6f42a07b8159245fc3549f99f97eed49d01118fa Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 09:58:02 +0200 Subject: [PATCH 223/278] refactor(protocols): use request-scoped project service --- app/backend/api/routers/protocol_router.py | 15 ++++++++++----- tests/integration/api/test_projects_router.py | 2 +- .../backend/api/routers/test_protocols_router.py | 6 +++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/app/backend/api/routers/protocol_router.py b/app/backend/api/routers/protocol_router.py index b79d261d..ec8f7984 100644 --- a/app/backend/api/routers/protocol_router.py +++ b/app/backend/api/routers/protocol_router.py @@ -30,11 +30,13 @@ from app.backend.api.dependencies import getCurrentUser from app.backend.database import getMapper from app.backend.api.services.project_service import ProjectService -from app.backend.models.protocol_model import ProtocolRequest from app.backend.mapper.postgresql import PostgresqlFlatMapper router = APIRouter(prefix="/protocols", tags=["protocols"]) -service = ProjectService() + + +def getProjectService() -> ProjectService: + return ProjectService() @router.get("/{projectId}/{protocolId}", response_model=Any) @@ -42,7 +44,8 @@ async def loadProtocol( projectId: int, protocolId: int, currentUser=Depends(getCurrentUser), - mapper: PostgresqlFlatMapper = Depends(getMapper) + mapper: PostgresqlFlatMapper = Depends(getMapper), + service: ProjectService = Depends(getProjectService), ): project = service.getProjectById(mapper, projectId, currentUser) if not project: @@ -60,7 +63,8 @@ async def loadNewProtocol( projectId: int, protClassName: str, currentUser=Depends(getCurrentUser), - mapper: PostgresqlFlatMapper = Depends(getMapper) + mapper: PostgresqlFlatMapper = Depends(getMapper), + service: ProjectService = Depends(getProjectService), ): project = service.getProjectById(mapper, projectId, currentUser) if not project: @@ -99,7 +103,8 @@ async def getProtocolLogs( errOffset: int = 0, scheduleOffset: int = 0, currentUser=Depends(getCurrentUser), - mapper: PostgresqlFlatMapper = Depends(getMapper) + mapper: PostgresqlFlatMapper = Depends(getMapper), + service: ProjectService = Depends(getProjectService), ): project = service.getProjectById(mapper, projectId, currentUser) if not project: diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 44d45388..35d7c095 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -63,7 +63,7 @@ def test_GetProjectCallsServiceWithRefreshAndCheckPid(projectClient, fakeProject }, "refresh": True, "checkPid": True, - "validateConsistency": True, + "validateConsistency": False, "failOnConsistencyError": False, } diff --git a/tests/unit/backend/api/routers/test_protocols_router.py b/tests/unit/backend/api/routers/test_protocols_router.py index 693e3638..6d7292c7 100644 --- a/tests/unit/backend/api/routers/test_protocols_router.py +++ b/tests/unit/backend/api/routers/test_protocols_router.py @@ -133,9 +133,6 @@ def protocolClient( fakeProtocolRouterService, monkeypatch: pytest.MonkeyPatch, ) -> Iterator[TestClient]: - # protocolClient - monkeypatch.setattr(protocolRouterModule, "service", fakeProtocolRouterService) - app = FastAPI() app.include_router(protocolRouterModule.router) @@ -145,6 +142,9 @@ def protocolClient( "email": "user@example.com", "role": "user", } + app.dependency_overrides[protocolRouterModule.getProjectService] = ( + lambda: fakeProtocolRouterService + ) with TestClient(app) as client: yield client From 2d920246814e2b3f46de3732366fba036f813f70 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 12:23:03 +0200 Subject: [PATCH 224/278] fix(protocols): scope new protocol params cache by project --- app/backend/api/services/project_service.py | 2 +- .../test_project_service_protocols.py | 39 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 3cb70112..e8dd606d 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -5154,7 +5154,7 @@ def getNewProtocolParams(self, projectId, protocolClassName: str) -> dict: # getNewProtocolParams _invalidateNewProtocolCacheIfNeeded() - key = str(protocolClassName) + key = "%s:%s" % (str(projectId), str(protocolClassName)) with _newProtocolLock: cached = _newProtocolCache.get(key) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index ee7aebc1..9ae18e9d 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -2367,3 +2367,42 @@ def test_ApplyParamsToProtocolResolvesPostgresqlMultiPointerParentIds( assert mapper.db.fetchOneCalls[1]["params"] == (1, 501, "501") +def test_GetNewProtocolParamsCacheIsScopedByProject( + projectServiceModule, + service, + monkeypatch, +): + projectServiceModule._newProtocolCache.clear() + + class FakeProtocolClass: + pass + + service.currentProject.protocolFactories = { + "ProtClass": FakeProtocolClass, + } + + buildCalls = [] + + def fakeBuildProtocolContext(projectId, protocol): + buildCalls.append(projectId) + return { + "info": { + "projectId": projectId, + "protocolClassName": "ProtClass", + }, + "form": {}, + "values": {}, + } + + monkeypatch.setattr( + service, + "_buildProtocolContext", + fakeBuildProtocolContext, + ) + + first = service.getNewProtocolParams(1, "ProtClass") + second = service.getNewProtocolParams(2, "ProtClass") + + assert first["info"]["projectId"] == 1 + assert second["info"]["projectId"] == 2 + assert buildCalls == [1, 2] \ No newline at end of file From cf9c94a3174b2b453cfd0d78bc3abe211de7579b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 12:35:05 +0200 Subject: [PATCH 225/278] fix(protocols): handle duplicate protocol copy errors safely --- app/backend/api/services/project_service.py | 20 +++++++-------- .../test_project_service_protocols.py | 25 ++++++++++++++++++- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index e8dd606d..77e42fec 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6412,19 +6412,17 @@ def duplicateProtocol(self, mapper, projectId, protocols): protId = str(prot.getObjId()) duplicated.append({"sourceId": sourceIds[index], "newId": protId}) + except Exception as e: - errors.append("Failed to duplicate protocols. projectId=%s protocolIds=%s" %projectId, - [getattr(p, "getObjId", lambda: None)() for p in protocolList]) + protocolIds = [ + getattr(p, "getObjId", lambda: None)() + for p in protocolList + ] + logger.exception("Failed to duplicate protocols. projectId=%s protocolIds=%s", + projectId, protocolIds,) - logger.exception( - "Failed to duplicate protocols. projectId=%s protocolIds=%s", - projectId, - [getattr(p, "getObjId", lambda: None)() for p in protocolList], - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to duplicate protocols: {e}", - ) + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to duplicate protocols: {e}",) try: syncResult = self.syncProjectProtocolsAndDependencies( diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index 9ae18e9d..088ea0b4 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -2405,4 +2405,27 @@ def fakeBuildProtocolContext(projectId, protocol): assert first["info"]["projectId"] == 1 assert second["info"]["projectId"] == 2 - assert buildCalls == [1, 2] \ No newline at end of file + assert buildCalls == [1, 2] + +def test_DuplicateProtocolWrapsCopyErrorAsHttpException(service, mapper): + protocol = FakeProtocol(objId=10) + service.currentProject.protocols[10] = protocol + + def fakeCopyProtocol(protocols): + raise RuntimeError("copy failed") + + service.currentProject.copyProtocol = fakeCopyProtocol + + class DuplicateItem: + def __init__(self, itemId): + self.id = itemId + + with pytest.raises(HTTPException) as exc: + service.duplicateProtocol( + mapper=mapper, + projectId=1, + protocols=[DuplicateItem("10")], + ) + + assert exc.value.status_code == 500 + assert exc.value.detail == "Failed to duplicate protocols: copy failed" \ No newline at end of file From 445593758d033e2e6339f36d02fade5b3ba78afe Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 12:38:52 +0200 Subject: [PATCH 226/278] fix(protocols): normalize legacy log offsets --- app/backend/api/services/project_service.py | 3 ++ .../services/test_project_service_logs_fs.py | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 77e42fec..b88b6f0c 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6292,6 +6292,9 @@ def getProtocolLogs( logPath = protocol.getStdoutLog() errLogPath = protocol.getStderrLog() scheduleLogPath = protocol.getScheduleLog() + offset = max(0, int(offset or 0)) + errOffset = max(0, int(errOffset or 0)) + scheduleOffset = max(0, int(scheduleOffset or 0)) stdoutContent, stderrContent, scheduleContent = "", "", "" newOffsetOut, newOffsetErr, newOffsetSchedule = offset, errOffset, scheduleOffset diff --git a/tests/unit/backend/api/services/test_project_service_logs_fs.py b/tests/unit/backend/api/services/test_project_service_logs_fs.py index 77e28472..b8916b43 100644 --- a/tests/unit/backend/api/services/test_project_service_logs_fs.py +++ b/tests/unit/backend/api/services/test_project_service_logs_fs.py @@ -735,3 +735,35 @@ class FakePayload: } assert mapper.db.fetchCalls[0]["params"] == (1, 500, "500") + +def test_GetProtocolLogsNormalizesNegativeOffsets(service, tmp_path): + stdoutLog = tmp_path / "stdout.log" + stderrLog = tmp_path / "stderr.log" + scheduleLog = tmp_path / "schedule.log" + + stdoutLog.write_text("abc\n", encoding="utf-8") + stderrLog.write_text("ERR\n", encoding="utf-8") + scheduleLog.write_text("SCH\n", encoding="utf-8") + + service.currentProject.protocols[10] = FakeProtocol( + stdoutLog=str(stdoutLog), + stderrLog=str(stderrLog), + scheduleLog=str(scheduleLog), + ) + + result = service.getProtocolLogs( + projectId=1, + protocolId=10, + offset=-10, + errOffset=-5, + scheduleOffset=-1, + ) + + assert result == { + "stdoutLog": "abc\n", + "stderrLog": "ERR\n", + "stdoutOffset": 4, + "stderrOffset": 4, + "scheduleLog": "SCH\n", + "scheduleOffset": 4, + } \ No newline at end of file From dd1be6da4a465b2864111de39550a021a93859a2 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 12:43:14 +0200 Subject: [PATCH 227/278] fix(protocols): reset legacy log offsets after truncation --- app/backend/api/services/project_service.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index b88b6f0c..5b66d072 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6292,9 +6292,23 @@ def getProtocolLogs( logPath = protocol.getStdoutLog() errLogPath = protocol.getStderrLog() scheduleLogPath = protocol.getScheduleLog() - offset = max(0, int(offset or 0)) - errOffset = max(0, int(errOffset or 0)) - scheduleOffset = max(0, int(scheduleOffset or 0)) + + def normalizeOffset(value, filePath): + safeOffset = max(0, int(value or 0)) + + if filePath and os.path.exists(filePath): + try: + fileSize = os.path.getsize(filePath) + if safeOffset > fileSize: + return 0 + except Exception: + pass + + return safeOffset + + offset = normalizeOffset(offset, logPath) + errOffset = normalizeOffset(errOffset, errLogPath) + scheduleOffset = normalizeOffset(scheduleOffset, scheduleLogPath) stdoutContent, stderrContent, scheduleContent = "", "", "" newOffsetOut, newOffsetErr, newOffsetSchedule = offset, errOffset, scheduleOffset From 5278ecccc28f64a80086e40f7ff7ce65efd8ee96 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 12:54:23 +0200 Subject: [PATCH 228/278] fix(protocols): preserve delete protocol http errors --- app/backend/api/routers/project_router.py | 18 ++++++++++--- .../api/test_projects_router_protocol_ops.py | 27 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index f8fb90eb..6f9b6cfe 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -748,12 +748,24 @@ def deleteProtocol( "errors": [], "workflow": []} + except HTTPException as e: + return JSONResponse( + status_code=e.status_code, + content={ + "status": 1, + "errors": _normalizeErrors(e.detail), + "workflow": [], + }, + ) + except Exception as e: return JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - content={"status": 1, - "errors": _normalizeErrors(str(e)), - "workflow": []}, + content={ + "status": 1, + "errors": _normalizeErrors(str(e)), + "workflow": [], + }, ) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 946ad4c9..7dd3dcd6 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -615,4 +615,31 @@ def test_StopProtocolDelegatesToService(projectClient, fakeProjectService): "actionLabel": "stop protocol", "refresh": True, "checkPid": True, + } + +def test_DeleteProtocolReturnsErrorsWhenServiceRaisesHttpException( + projectClient, + fakeProjectService, +): + fakeProjectService.deleteProtocolError = HTTPException( + status_code=422, + detail=["cannot delete protocol"], + ) + + response = projectClient.post( + "/projects/1/protocols/delete", + json={"protocolIds": ["10"]}, + ) + + assert response.status_code == 422 + assert response.json() == { + "status": 1, + "errors": ["cannot delete protocol"], + "workflow": [], + } + + assert fakeProjectService.lastDeleteProtocolCall == { + "mapper": fakeProjectService.lastDeleteProtocolCall["mapper"], + "projectId": 1, + "protocolIds": ["10"], } \ No newline at end of file From d335104c492a6e8af17b2a2c5215848b1401def1 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 12:56:17 +0200 Subject: [PATCH 229/278] fix(protocols): return validation error for missing delete ids --- app/backend/api/routers/project_router.py | 2 +- tests/integration/api/test_projects_router_protocol_ops.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 6f9b6cfe..7d7c94f4 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -736,7 +736,7 @@ def deleteProtocol( protocolIds = getattr(payload, "protocolIds", None) if payload is not None else None if not protocolIds: return JSONResponse( - status_code=status.HTTP_404_NOT_FOUND, + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, content={"status": 1, "errors": ["Missing protocolIds"], "workflow": []}, diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 7dd3dcd6..544883c2 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -327,7 +327,7 @@ def test_DeleteProtocolRejectsMissingProtocolIds(projectClient): json={"protocolIds": []}, ) - assert response.status_code == 404 + assert response.status_code == 422 assert response.json() == { "status": 1, "errors": ["Missing protocolIds"], From 4b21b5dcce53ed81fd0c17d9528d503736edc1a4 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 13:04:58 +0200 Subject: [PATCH 230/278] fix(protocols): return delete protocol sync counts --- app/backend/api/routers/project_router.py | 18 +++++++-- .../api/test_projects_router_protocol_ops.py | 39 +++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 7d7c94f4..f4643c5a 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -742,11 +742,21 @@ def deleteProtocol( "workflow": []}, ) - service.deleteProtocol(mapper, projectId, protocolIds) + result = service.deleteProtocol(mapper, projectId, protocolIds) or {} - return {"status": 0, - "errors": [], - "workflow": []} + response = { + "status": 0, + "errors": [], + "workflow": [], + } + + if isinstance(result, dict): + if "protocolsCount" in result: + response["protocolsCount"] = result["protocolsCount"] + if "dependenciesCount" in result: + response["dependenciesCount"] = result["dependenciesCount"] + + return response except HTTPException as e: return JSONResponse( diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 544883c2..b71e7844 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -638,6 +638,45 @@ def test_DeleteProtocolReturnsErrorsWhenServiceRaisesHttpException( "workflow": [], } + assert fakeProjectService.lastDeleteProtocolCall == { + "mapper": fakeProjectService.lastDeleteProtocolCall["mapper"], + "projectId": 1, + "protocolIds": ["10"], + } + + +def test_DeleteProtocolReturnsSyncCounts( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeDeleteProtocol(mapper, projectId, protocolIds): + fakeProjectService.lastDeleteProtocolCall = { + "mapper": mapper, + "projectId": projectId, + "protocolIds": protocolIds, + } + return { + "protocolsCount": 3, + "dependenciesCount": 2, + } + + monkeypatch.setattr(fakeProjectService, "deleteProtocol", fakeDeleteProtocol) + + response = projectClient.post( + "/projects/1/protocols/delete", + json={"protocolIds": ["10"]}, + ) + + assert response.status_code == 200 + assert response.json() == { + "status": 0, + "errors": [], + "workflow": [], + "protocolsCount": 3, + "dependenciesCount": 2, + } + assert fakeProjectService.lastDeleteProtocolCall == { "mapper": fakeProjectService.lastDeleteProtocolCall["mapper"], "projectId": 1, From be151550a4d273aa16546db0b2d7849b13bc49b8 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 13:21:05 +0200 Subject: [PATCH 231/278] fix(protocols): return duplicate protocol sync counts --- app/backend/api/routers/project_router.py | 19 ++++++++--- .../api/test_projects_router_protocol_ops.py | 32 +++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index f4643c5a..beb53581 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -692,12 +692,21 @@ def duplicateProtocol( "workflow": []}, ) - result = service.duplicateProtocol(mapper, projectId, items) + result = service.duplicateProtocol(mapper, projectId, items) or {} # Keep 201 on success, but still return unified schema - return {"status": result['status'], - "errors": result['errors'], - "workflow": [], - "duplicated": result['duplicated']} + response = { + "status": result.get("status", 0), + "errors": result.get("errors", []), + "workflow": [], + "duplicated": result.get("duplicated", []), + } + + if "protocolsCount" in result: + response["protocolsCount"] = result["protocolsCount"] + if "dependenciesCount" in result: + response["dependenciesCount"] = result["dependenciesCount"] + + return response except HTTPException as e: return JSONResponse( diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index b71e7844..194b06ce 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -321,6 +321,38 @@ def test_DuplicateProtocolDelegatesToService(projectClient, fakeProjectService): assert items[1].name is None +def test_DuplicateProtocolReturnsSyncCounts( + projectClient, + fakeProjectService, +): + fakeProjectService.duplicateProtocolResult = { + "status": 0, + "errors": [], + "duplicated": [{"sourceId": "10", "newId": "20"}], + "protocolsCount": 4, + "dependenciesCount": 3, + } + + response = projectClient.post( + "/projects/1/protocols/duplicate", + json={ + "items": [ + {"id": "10", "name": "Copy 1"}, + ] + }, + ) + + assert response.status_code == 201 + assert response.json() == { + "status": 0, + "errors": [], + "workflow": [], + "duplicated": [{"sourceId": "10", "newId": "20"}], + "protocolsCount": 4, + "dependenciesCount": 3, + } + + def test_DeleteProtocolRejectsMissingProtocolIds(projectClient): response = projectClient.post( "/projects/1/protocols/delete", From 15a1d7fc6603afd8aea5bab46c52a4951122de33 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 13:37:00 +0200 Subject: [PATCH 232/278] fix(protocols): return stop protocol sync counts --- app/backend/api/routers/project_router.py | 20 ++++++++++++++----- .../api/test_projects_router_protocol_ops.py | 2 ++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index beb53581..defd2b8f 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -953,17 +953,27 @@ def stopProtocol( ) service.stopProtocol(mapper, projectId, protocolIds) - service.syncProjectGraphAfterMutation( + syncResult = service.syncProjectGraphAfterMutation( mapper, projectId, actionLabel="stop protocol", refresh=True, checkPid=True, - ) + ) or {} - return {"status": 0, - "errors": [], - "workflow": []} + response = { + "status": 0, + "errors": [], + "workflow": [], + } + + if isinstance(syncResult, dict): + if "protocols" in syncResult: + response["protocolsCount"] = syncResult["protocols"] + if "dependencies" in syncResult: + response["dependenciesCount"] = syncResult["dependencies"] + + return response except HTTPException as e: return JSONResponse( diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 194b06ce..b82d5112 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -633,6 +633,8 @@ def test_StopProtocolDelegatesToService(projectClient, fakeProjectService): "status": 0, "errors": [], "workflow": [], + "protocolsCount": 1, + "dependenciesCount": 0, } assert fakeProjectService.lastStopProtocolCall == { From bc809605019ddbdecca459f70f6a81ac297a6107 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 13:39:52 +0200 Subject: [PATCH 233/278] fix(protocols): return restart protocol sync counts --- app/backend/api/routers/project_router.py | 20 ++++++++++++++----- .../api/test_projects_router_protocol_ops.py | 2 ++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index defd2b8f..304ed969 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -811,17 +811,27 @@ def restartProtocolAll( try: service.restartProtocolAll(mapper, projectId, protocolId) - service.syncProjectGraphAfterMutation( + syncResult = service.syncProjectGraphAfterMutation( mapper, projectId, actionLabel="restart protocol subtree", refresh=True, checkPid=True, - ) + ) or {} - return {"status": 0, - "errors": [], - "workflow": []} + response = { + "status": 0, + "errors": [], + "workflow": [], + } + + if isinstance(syncResult, dict): + if "protocols" in syncResult: + response["protocolsCount"] = syncResult["protocols"] + if "dependencies" in syncResult: + response["dependenciesCount"] = syncResult["dependencies"] + + return response except HTTPException as e: return JSONResponse( diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index b82d5112..add992b7 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -413,6 +413,8 @@ def test_RestartProtocolAllReturnsSuccess(projectClient, fakeProjectService): "status": 0, "errors": [], "workflow": [], + "protocolsCount": 1, + "dependenciesCount": 0, } assert fakeProjectService.lastRestartProtocolAllCall == { From c30edf8e295a08395578c36983b6bf89c59dd873 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 13:52:07 +0200 Subject: [PATCH 234/278] fix(protocols): return continue protocol sync counts --- app/backend/api/routers/project_router.py | 18 +++++++++++++++--- .../api/test_projects_router_protocol_ops.py | 2 ++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 304ed969..17bb42cd 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -870,15 +870,27 @@ def continueProtocolAll( try: service.continueProtocolAll(mapper, projectId, protocolId, currentUser) - service.syncProjectGraphAfterMutation( + syncResult = service.syncProjectGraphAfterMutation( mapper, projectId, actionLabel="continue protocol subtree", refresh=True, checkPid=True, - ) + ) or {} - return {"status": 0, "errors": [], "workflow": []} + response = { + "status": 0, + "errors": [], + "workflow": [], + } + + if isinstance(syncResult, dict): + if "protocols" in syncResult: + response["protocolsCount"] = syncResult["protocols"] + if "dependencies" in syncResult: + response["dependenciesCount"] = syncResult["dependencies"] + + return response except HTTPException as e: return JSONResponse( diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index add992b7..06e0f064 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -439,6 +439,8 @@ def test_ContinueProtocolAllDelegatesToService(projectClient, fakeProjectService "status": 0, "errors": [], "workflow": [], + "protocolsCount": 1, + "dependenciesCount": 0, } assert fakeProjectService.lastSyncProjectGraphAfterMutationCall == { From 690f8d9983f96f9029d2cd0034715a86036a0b72 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 13:53:44 +0200 Subject: [PATCH 235/278] fix(protocols): return reset protocol sync counts --- app/backend/api/routers/project_router.py | 18 +++++++++++++++--- .../api/test_projects_router_protocol_ops.py | 2 ++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 17bb42cd..eca24064 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -925,15 +925,27 @@ def resetProtocolFrom( try: service.resetProtocolFrom(mapper, projectId, protocolId) - service.syncProjectGraphAfterMutation( + syncResult = service.syncProjectGraphAfterMutation( mapper, projectId, actionLabel="reset protocol from node", refresh=True, checkPid=True, - ) + ) or {} - return {"status": 0, "errors": [], "workflow": []} + response = { + "status": 0, + "errors": [], + "workflow": [], + } + + if isinstance(syncResult, dict): + if "protocols" in syncResult: + response["protocolsCount"] = syncResult["protocols"] + if "dependencies" in syncResult: + response["dependenciesCount"] = syncResult["dependencies"] + + return response except HTTPException as e: return JSONResponse( diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 06e0f064..d42aa4f6 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -471,6 +471,8 @@ def test_ResetProtocolFromDelegatesToService(projectClient, fakeProjectService): "status": 0, "errors": [], "workflow": [], + "protocolsCount": 1, + "dependenciesCount": 0, } assert fakeProjectService.lastResetProtocolFromCall == { From e916b1eb72ea0b977aedb3269bb731308fe7ee11 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 13:57:33 +0200 Subject: [PATCH 236/278] refactor(protocols): centralize sync count response mapping --- app/backend/api/routers/project_router.py | 61 ++++++++--------------- 1 file changed, 20 insertions(+), 41 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index eca24064..b417eab9 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -50,6 +50,20 @@ def getProjectService() -> ProjectService: """Return a fresh ProjectService per request to avoid shared state.""" return ProjectService() +def _appendProtocolSyncCounts(response: Dict[str, Any], result: Any) -> Dict[str, Any]: + if not isinstance(result, dict): + return response + + protocolsCount = result.get("protocolsCount", result.get("protocols")) + dependenciesCount = result.get("dependenciesCount", result.get("dependencies")) + + if protocolsCount is not None: + response["protocolsCount"] = protocolsCount + if dependenciesCount is not None: + response["dependenciesCount"] = dependenciesCount + + return response + # ====================================================================== # PROJECT WORKFLOWS # ====================================================================== @@ -701,12 +715,7 @@ def duplicateProtocol( "duplicated": result.get("duplicated", []), } - if "protocolsCount" in result: - response["protocolsCount"] = result["protocolsCount"] - if "dependenciesCount" in result: - response["dependenciesCount"] = result["dependenciesCount"] - - return response + return _appendProtocolSyncCounts(response, result) except HTTPException as e: return JSONResponse( @@ -759,13 +768,7 @@ def deleteProtocol( "workflow": [], } - if isinstance(result, dict): - if "protocolsCount" in result: - response["protocolsCount"] = result["protocolsCount"] - if "dependenciesCount" in result: - response["dependenciesCount"] = result["dependenciesCount"] - - return response + return _appendProtocolSyncCounts(response, result) except HTTPException as e: return JSONResponse( @@ -825,13 +828,7 @@ def restartProtocolAll( "workflow": [], } - if isinstance(syncResult, dict): - if "protocols" in syncResult: - response["protocolsCount"] = syncResult["protocols"] - if "dependencies" in syncResult: - response["dependenciesCount"] = syncResult["dependencies"] - - return response + return _appendProtocolSyncCounts(response, syncResult) except HTTPException as e: return JSONResponse( @@ -884,13 +881,7 @@ def continueProtocolAll( "workflow": [], } - if isinstance(syncResult, dict): - if "protocols" in syncResult: - response["protocolsCount"] = syncResult["protocols"] - if "dependencies" in syncResult: - response["dependenciesCount"] = syncResult["dependencies"] - - return response + return _appendProtocolSyncCounts(response, syncResult) except HTTPException as e: return JSONResponse( @@ -939,13 +930,7 @@ def resetProtocolFrom( "workflow": [], } - if isinstance(syncResult, dict): - if "protocols" in syncResult: - response["protocolsCount"] = syncResult["protocols"] - if "dependencies" in syncResult: - response["dependenciesCount"] = syncResult["dependencies"] - - return response + return _appendProtocolSyncCounts(response, syncResult) except HTTPException as e: return JSONResponse( @@ -1001,13 +986,7 @@ def stopProtocol( "workflow": [], } - if isinstance(syncResult, dict): - if "protocols" in syncResult: - response["protocolsCount"] = syncResult["protocols"] - if "dependencies" in syncResult: - response["dependenciesCount"] = syncResult["dependencies"] - - return response + return _appendProtocolSyncCounts(response, syncResult) except HTTPException as e: return JSONResponse( From c938d2e72262bfdc946c357467dac4b8588b33ff Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 14:08:52 +0200 Subject: [PATCH 237/278] fix(protocols): return rename protocol sync counts --- app/backend/api/routers/project_router.py | 14 +++++++++----- .../api/test_projects_router_protocol_ops.py | 2 ++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index b417eab9..cb07e14a 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -651,17 +651,21 @@ def renameProtocol( newNameText.strip(), str(newComment or "").strip(), ) - service.syncProjectGraphAfterMutation( + syncResult = service.syncProjectGraphAfterMutation( mapper, projectId, actionLabel="rename protocol", refresh=True, checkPid=True, - ) + ) or {} - return {"status": 0, - "errors": [], - "workflow": []} + response = { + "status": 0, + "errors": [], + "workflow": [], + } + + return _appendProtocolSyncCounts(response, syncResult) except HTTPException as e: return JSONResponse( diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index d42aa4f6..a1cf9304 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -260,6 +260,8 @@ def test_RenameProtocolDelegatesToService(projectClient, fakeProjectService): "status": 0, "errors": [], "workflow": [], + "protocolsCount": 1, + "dependenciesCount": 0, } assert fakeProjectService.lastRenameProtocolCall == { From 3063219a5c1495e50a03738344a09efea7f9e4f2 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 14:19:57 +0200 Subject: [PATCH 238/278] fix(protocols): return launch protocol sync counts --- app/backend/api/routers/project_router.py | 8 ++-- app/backend/api/services/project_service.py | 5 +- .../api/test_projects_router_protocol_ops.py | 48 +++++++++++++++++++ .../test_project_service_protocols.py | 25 ++++++++-- 4 files changed, 77 insertions(+), 9 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index cb07e14a..6ffdc10e 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -489,21 +489,23 @@ async def launchProtocol( }, ) - service.launchProtocol( + result = service.launchProtocol( mapper=mapper, projectId=projectId, protocolId=request.getProtocolId(), protocolClassName=request.getProtocolClassName(), params=request.getParams(), executeMode=request.getMode(), - ) + ) or {} - return { + response = { "status": 0, "errors": [], "workflow": [], } + return _appendProtocolSyncCounts(response, result) + except HTTPException as e: return JSONResponse( status_code=e.status_code, diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 5b66d072..af244177 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -5639,13 +5639,12 @@ def launchProtocol(self, mapper, projectId, protocolId, protocolClassName, param try: self.stopProtocol(mapper, projectId, [protocolId]) - self.syncProjectProtocolsAndDependencies( + return self.syncProjectProtocolsAndDependencies( mapper, projectId, refresh=True, checkPid=True, ) - return except HTTPException: raise except Exception as e: @@ -5731,7 +5730,7 @@ def launchProtocol(self, mapper, projectId, protocolId, protocolClassName, param self.currentProject.launchProtocol(protocol) - self.syncProjectProtocolsAndDependencies( + return self.syncProjectProtocolsAndDependencies( mapper, projectId, refresh=True, diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index a1cf9304..60f37a0c 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -132,6 +132,54 @@ def test_LaunchProtocolDelegatesToService(projectClient, fakeProjectService): } +def test_LaunchProtocolReturnsSyncCounts( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeLaunchProtocol( + mapper, + projectId, + protocolId, + protocolClassName, + params, + executeMode, + ): + fakeProjectService.lastLaunchProtocolCall = { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + "protocolClassName": protocolClassName, + "params": params, + "executeMode": executeMode, + } + return { + "protocols": 2, + "dependencies": 1, + } + + monkeypatch.setattr(fakeProjectService, "launchProtocol", fakeLaunchProtocol) + + response = projectClient.post( + "/projects/1/launch", + json={ + "protocolId": "10", + "protocolClassName": "ProtClass", + "params": {"a": 1}, + "mode": "resume", + }, + ) + + assert response.status_code == 200 + assert response.json() == { + "status": 0, + "errors": [], + "workflow": [], + "protocolsCount": 2, + "dependenciesCount": 1, + } + + def test_LaunchProtocolWrapsHttpException(projectClient, fakeProjectService): fakeProjectService.launchProtocolError = HTTPException(status_code=409, detail=["conflict", "busy"]) diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index 088ea0b4..29b54a0a 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -1171,13 +1171,28 @@ def test_LaunchProtocolRejectsUnknownExecuteMode(service, mapper): def test_LaunchProtocolStopDelegatesToStopProtocol(service, mapper, monkeypatch): calls = [] + def fakeStopProtocol(mapper, projectId, protocolIds): + calls.append(protocolIds) + + def fakeSyncProjectProtocolsAndDependencies( + mapper, + projectId, + refresh=False, + checkPid=False, + ): + return { + "protocols": 1, + "dependencies": 0, + } + + monkeypatch.setattr(service, "stopProtocol", fakeStopProtocol) monkeypatch.setattr( service, - "stopProtocol", - lambda mapper, projectId, protocolIds: calls.append(protocolIds), + "syncProjectProtocolsAndDependencies", + fakeSyncProjectProtocolsAndDependencies, ) - service.launchProtocol( + result = service.launchProtocol( mapper=mapper, projectId=1, protocolId="10", @@ -1187,6 +1202,10 @@ def test_LaunchProtocolStopDelegatesToStopProtocol(service, mapper, monkeypatch) ) assert calls == [["10"]] + assert result == { + "protocols": 1, + "dependencies": 0, + } def test_LaunchProtocolRaises422WhenValidationFails(service, mapper, monkeypatch): From 2e63504370abe428655eb5a7bfc410529633ef46 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 15:58:25 +0200 Subject: [PATCH 239/278] fix(protocols): accept resume launch mode alias --- app/backend/api/services/project_service.py | 5 +++++ .../backend/api/services/test_project_service_protocols.py | 1 + 2 files changed, 6 insertions(+) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index af244177..ba826a77 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -5628,6 +5628,11 @@ def launchProtocol(self, mapper, projectId, protocolId, protocolClassName, param Save, validate, and execute a protocol action. Supported execute modes: launch, restart, schedule, stop. """ + modeAliases = { + "resume": "launch", + } + + executeMode = modeAliases.get(executeMode, executeMode) allowedModes = {"launch", "restart", "schedule", "stop"} if executeMode not in allowedModes: raise HTTPException( diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index 29b54a0a..fb3b622d 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -1230,6 +1230,7 @@ def test_LaunchProtocolRaises422WhenValidationFails(service, mapper, monkeypatch "executeMode, expectedRunMode", [ ("launch", "resume-mode"), + ("resume", "resume-mode"), ("restart", "restart-mode"), ], ) From 0472263da1c4aef635ea7160c455479a39fb428c Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:01:12 +0200 Subject: [PATCH 240/278] fix(protocols): default missing launch mode to launch --- app/backend/api/services/project_service.py | 2 +- .../api/test_projects_router_protocol_ops.py | 20 +++++++++++++++++++ .../test_project_service_protocols.py | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index ba826a77..e8217e1e 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -5629,9 +5629,9 @@ def launchProtocol(self, mapper, projectId, protocolId, protocolClassName, param Supported execute modes: launch, restart, schedule, stop. """ modeAliases = { + None: "launch", "resume": "launch", } - executeMode = modeAliases.get(executeMode, executeMode) allowedModes = {"launch", "restart", "schedule", "stop"} if executeMode not in allowedModes: diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 60f37a0c..360c1b6b 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -132,6 +132,26 @@ def test_LaunchProtocolDelegatesToService(projectClient, fakeProjectService): } +def test_LaunchProtocolDefaultsMissingModeToLaunch(projectClient, fakeProjectService): + response = projectClient.post( + "/projects/1/launch", + json={ + "protocolId": "10", + "protocolClassName": "ProtClass", + "params": {"a": 1}, + }, + ) + + assert response.status_code == 200 + assert response.json() == { + "status": 0, + "errors": [], + "workflow": [], + } + + assert fakeProjectService.lastLaunchProtocolCall["executeMode"] is None + + def test_LaunchProtocolReturnsSyncCounts( projectClient, fakeProjectService, diff --git a/tests/unit/backend/api/services/test_project_service_protocols.py b/tests/unit/backend/api/services/test_project_service_protocols.py index fb3b622d..25801e11 100644 --- a/tests/unit/backend/api/services/test_project_service_protocols.py +++ b/tests/unit/backend/api/services/test_project_service_protocols.py @@ -1229,6 +1229,7 @@ def test_LaunchProtocolRaises422WhenValidationFails(service, mapper, monkeypatch @pytest.mark.parametrize( "executeMode, expectedRunMode", [ + (None, "resume-mode"), ("launch", "resume-mode"), ("resume", "resume-mode"), ("restart", "restart-mode"), From 316392394516ddee0e063e9734b1014cb3ff5972 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:04:31 +0200 Subject: [PATCH 241/278] fix(protocols): mark launch protocol errors as failed --- app/backend/api/routers/project_router.py | 6 +++--- tests/integration/api/test_projects_router_protocol_ops.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 6ffdc10e..61c794d1 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -483,7 +483,7 @@ async def launchProtocol( return JSONResponse( status_code=status.HTTP_404_NOT_FOUND, content={ - "status": 0, + "status": 1, "errors": ["Project not found"], "workflow": [], }, @@ -510,7 +510,7 @@ async def launchProtocol( return JSONResponse( status_code=e.status_code, content={ - "status": 0, + "status": 1, "errors": _normalizeErrors(e.detail), "workflow": [], }, @@ -520,7 +520,7 @@ async def launchProtocol( return JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={ - "status": 0, + "status": 1, "errors": ["Internal server error"], "workflow": [], }, diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 360c1b6b..af2656c9 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -98,7 +98,7 @@ def test_LaunchProtocolReturns404EnvelopeWhenProjectMissing(projectClient, fakeP assert response.status_code == 404 assert response.json() == { - "status": 0, + "status": 1, "errors": ["Project not found"], "workflow": [], } @@ -215,7 +215,7 @@ def test_LaunchProtocolWrapsHttpException(projectClient, fakeProjectService): assert response.status_code == 409 assert response.json() == { - "status": 0, + "status": 1, "errors": ["conflict", "busy"], "workflow": [], } From 7ffb3349369f3c5774932c69339f58c3ade2d34a Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:07:07 +0200 Subject: [PATCH 242/278] test(protocols): cover launch protocol unexpected errors --- .../api/test_projects_router_protocol_ops.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index af2656c9..c8b2b990 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -221,6 +221,41 @@ def test_LaunchProtocolWrapsHttpException(projectClient, fakeProjectService): } +def test_LaunchProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeLaunchProtocol( + mapper, + projectId, + protocolId, + protocolClassName, + params, + executeMode, + ): + raise RuntimeError("boom") + + monkeypatch.setattr(fakeProjectService, "launchProtocol", fakeLaunchProtocol) + + response = projectClient.post( + "/projects/1/launch", + json={ + "protocolId": "10", + "protocolClassName": "ProtClass", + "params": {"a": 1}, + "mode": "resume", + }, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["Internal server error"], + "workflow": [], + } + + def test_SaveProtocolReturnsSuccessWhenNoErrors(projectClient, fakeProjectService): fakeProjectService.saveProtocolResult = ({"protocolId": "10"}, []) From 15e57ddb989b08bcd35f0f3d3d4db02d67bac90a Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:10:43 +0200 Subject: [PATCH 243/278] test(protocols): cover save protocol http errors --- .../api/test_projects_router_protocol_ops.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index c8b2b990..a8205e38 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -307,6 +307,29 @@ def test_SaveProtocolReturnsStatusOneWhenServiceReturnsErrors(projectClient, fak } +def test_SaveProtocolWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.saveProtocolError = HTTPException( + status_code=500, + detail="Protocol was saved in Scipion but graph sync to PostgreSQL failed", + ) + + response = projectClient.post( + "/projects/1/save", + json={ + "protocolId": "10", + "protocolClassName": "ProtClass", + "params": {"a": 1}, + }, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["Protocol was saved in Scipion but graph sync to PostgreSQL failed"], + "workflow": [], + } + + def test_SuggestionProtocolReturns404EnvelopeWhenProjectMissing(projectClient, fakeProjectService): fakeProjectService.projectByIdResult = None From f24554726872c53dccc488cb05bdf923497743c4 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:16:18 +0200 Subject: [PATCH 244/278] test(protocols): cover save protocol unexpected errors --- .../api/test_projects_router_protocol_ops.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index a8205e38..2c2c86cc 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -330,6 +330,39 @@ def test_SaveProtocolWrapsHttpException(projectClient, fakeProjectService): } +def test_SaveProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeSaveProtocol( + mapper, + projectId, + protocolId, + protocolClassName, + params, + ): + raise RuntimeError("boom") + + monkeypatch.setattr(fakeProjectService, "saveProtocol", fakeSaveProtocol) + + response = projectClient.post( + "/projects/1/save", + json={ + "protocolId": "10", + "protocolClassName": "ProtClass", + "params": {"a": 1}, + }, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_SuggestionProtocolReturns404EnvelopeWhenProjectMissing(projectClient, fakeProjectService): fakeProjectService.projectByIdResult = None From 4e2385beddf551cdd5a02ff86f0b620a3229ade6 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:18:38 +0200 Subject: [PATCH 245/278] test(protocols): cover save protocol missing project --- .../api/test_projects_router_protocol_ops.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 2c2c86cc..bed3d2b4 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -284,6 +284,29 @@ def test_SaveProtocolReturnsSuccessWhenNoErrors(projectClient, fakeProjectServic } +def test_SaveProtocolReturns404EnvelopeWhenProjectMissing( + projectClient, + fakeProjectService, +): + fakeProjectService.projectByIdResult = None + + response = projectClient.post( + "/projects/1/save", + json={ + "protocolId": "10", + "protocolClassName": "ProtClass", + "params": {"a": 1}, + }, + ) + + assert response.status_code == 404 + assert response.json() == { + "status": 1, + "errors": ["Project not found"], + "workflow": [], + } + + def test_SaveProtocolReturnsStatusOneWhenServiceReturnsErrors(projectClient, fakeProjectService): fakeProjectService.saveProtocolResult = ( {"protocolId": "10"}, From a787ce440bb0fdfc30fb68f818bf3fe20ef89a78 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:25:53 +0200 Subject: [PATCH 246/278] fix(protocols): normalize suggestion protocol errors --- app/backend/api/routers/project_router.py | 29 +++++++++++--- tests/conftest.py | 3 ++ .../api/test_projects_router_protocol_ops.py | 40 +++++++++++++++++++ 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 61c794d1..25bd24e6 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -595,11 +595,30 @@ def suggestionProtocol( "errors": ["Project not found"], "workflow": []}, ) - return service.getNextProtocolSuggestions( - mapper=mapper, - projectId=projectId, - protocolId=protocolId, - ) + try: + return service.getNextProtocolSuggestions( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + except HTTPException as e: + return JSONResponse( + status_code=e.status_code, + content={ + "status": 1, + "errors": _normalizeErrors(e.detail), + "workflow": [], + }, + ) + except Exception as e: + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={ + "status": 1, + "errors": [str(e)], + "workflow": [], + }, + ) @router.put("/{projectId}/protocols/{protocolId}/rename", response_model=Any, status_code=status.HTTP_200_OK) diff --git a/tests/conftest.py b/tests/conftest.py index e89de741..2c229af3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -260,6 +260,7 @@ def __init__(self): self.lastSaveProtocolCall = None self.nextProtocolSuggestionsResult = [{"id": "next-1", "name": "Next protocol"}] + self.nextProtocolSuggestionsError = None self.lastGetNextProtocolSuggestionsCall = None self.renameProtocolError = None @@ -627,6 +628,8 @@ def getNextProtocolSuggestions(self, protocolId, mapper=None, projectId=None): "mapper": mapper, "projectId": projectId, } + if self.nextProtocolSuggestionsError is not None: + raise self.nextProtocolSuggestionsError return self.nextProtocolSuggestionsResult def renameProtocol(self, mapper, projectId, protocolId, newName, newComment=""): diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index bed3d2b4..a82f3faf 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -412,6 +412,46 @@ def test_SuggestionProtocolReturnsSuggestions(projectClient, fakeProjectService) } +def test_SuggestionProtocolWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.nextProtocolSuggestionsError = HTTPException( + status_code=404, + detail="Protocol not found", + ) + + response = projectClient.get("/projects/1/protocols/10/suggestions/next") + + assert response.status_code == 404 + assert response.json() == { + "status": 1, + "errors": ["Protocol not found"], + "workflow": [], + } + + +def test_SuggestionProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeGetNextProtocolSuggestions(mapper, projectId, protocolId): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "getNextProtocolSuggestions", + fakeGetNextProtocolSuggestions, + ) + + response = projectClient.get("/projects/1/protocols/10/suggestions/next") + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_RenameProtocolRejectsBlankName(projectClient): response = projectClient.put( "/projects/1/protocols/10/rename", From 8dc5bf124e7a4fde613220bc05d04f08d36b3fcc Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:27:47 +0200 Subject: [PATCH 247/278] test(protocols): cover rename protocol unexpected errors --- .../api/test_projects_router_protocol_ops.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index a82f3faf..57367c84 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -503,6 +503,38 @@ def test_RenameProtocolDelegatesToService(projectClient, fakeProjectService): } +def test_RenameProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeRenameProtocol( + mapper, + projectId, + protocolId, + newName, + newComment, + ): + raise RuntimeError("boom") + + monkeypatch.setattr(fakeProjectService, "renameProtocol", fakeRenameProtocol) + + response = projectClient.put( + "/projects/1/protocols/10/rename", + json={ + "runName": "Renamed protocol", + "comment": "Updated comment", + }, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_DuplicateProtocolRejectsMissingItems(projectClient): response = projectClient.post( "/projects/1/protocols/duplicate", From 6f64c0e135fd92c363f18e5c673fbe752df198e6 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:29:25 +0200 Subject: [PATCH 248/278] test(protocols): cover rename protocol http errors --- .../api/test_projects_router_protocol_ops.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 57367c84..c00087ac 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -535,6 +535,31 @@ def fakeRenameProtocol( } +def test_RenameProtocolWrapsHttpException( + projectClient, + fakeProjectService, +): + patchRenameProtocolFake(fakeProjectService) + fakeProjectService.renameProtocolError = HTTPException( + status_code=404, + detail="Protocol not found", + ) + + response = projectClient.put( + "/projects/1/protocols/10/rename", + json={ + "runName": "Renamed protocol", + "comment": "Updated comment", + }, + ) + + assert response.status_code == 404 + assert response.json() == { + "status": 1, + "errors": ["Protocol not found"], + "workflow": [], + } + def test_DuplicateProtocolRejectsMissingItems(projectClient): response = projectClient.post( "/projects/1/protocols/duplicate", From 2b6f818b5b966188b9ddcdf523b61556e35fcb94 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:31:59 +0200 Subject: [PATCH 249/278] test(protocols): cover duplicate protocol errors --- .../api/test_projects_router_protocol_ops.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index c00087ac..51ebab95 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -560,6 +560,7 @@ def test_RenameProtocolWrapsHttpException( "workflow": [], } + def test_DuplicateProtocolRejectsMissingItems(projectClient): response = projectClient.post( "/projects/1/protocols/duplicate", @@ -634,6 +635,60 @@ def test_DuplicateProtocolReturnsSyncCounts( } +def test_DuplicateProtocolWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.duplicateProtocolError = HTTPException( + status_code=500, + detail="Failed to duplicate protocols: copy failed", + ) + + response = projectClient.post( + "/projects/1/protocols/duplicate", + json={ + "items": [ + {"id": "10", "name": "Copy 1"}, + ] + }, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["Failed to duplicate protocols: copy failed"], + "workflow": [], + } + + +def test_DuplicateProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeDuplicateProtocol(mapper, projectId, items): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "duplicateProtocol", + fakeDuplicateProtocol, + ) + + response = projectClient.post( + "/projects/1/protocols/duplicate", + json={ + "items": [ + {"id": "10", "name": "Copy 1"}, + ] + }, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_DeleteProtocolRejectsMissingProtocolIds(projectClient): response = projectClient.post( "/projects/1/protocols/delete", From 8137e47ec5d4f9a75c4dd151e2cb226d602749e9 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:33:46 +0200 Subject: [PATCH 250/278] test(protocols): cover delete protocol errors --- .../api/test_projects_router_protocol_ops.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 51ebab95..e267e8c3 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -723,6 +723,52 @@ def test_DeleteProtocolDelegatesToService(projectClient, fakeProjectService): } +def test_DeleteProtocolWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.deleteProtocolError = HTTPException( + status_code=500, + detail="Failed to delete protocols", + ) + + response = projectClient.post( + "/projects/1/protocols/delete", + json={"protocolIds": ["10"]}, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["Failed to delete protocols"], + "workflow": [], + } + + +def test_DeleteProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeDeleteProtocol(mapper, projectId, protocolIds): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "deleteProtocol", + fakeDeleteProtocol, + ) + + response = projectClient.post( + "/projects/1/protocols/delete", + json={"protocolIds": ["10"]}, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_RestartProtocolAllReturnsErrorsWhenServiceRaisesHttpException(projectClient, fakeProjectService): fakeProjectService.restartProtocolAllError = HTTPException( status_code=422, From 3d40bbaf6835eb4827747ffa695bebcb37d7075e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:35:52 +0200 Subject: [PATCH 251/278] test(protocols): cover restart protocol unexpected errors --- .../api/test_projects_router_protocol_ops.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index e267e8c3..d70d8dde 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -785,6 +785,30 @@ def test_RestartProtocolAllReturnsErrorsWhenServiceRaisesHttpException(projectCl } +def test_RestartProtocolAllWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeRestartProtocolAll(mapper, projectId, protocolId): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "restartProtocolAll", + fakeRestartProtocolAll, + ) + + response = projectClient.post("/projects/1/protocols/10/restart-all") + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_RestartProtocolAllReturnsSuccess(projectClient, fakeProjectService): fakeProjectService.restartProtocolAllResult = [] From 5e155cf8b40544c4ed0ae982dad03aab8fff39ba Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:37:23 +0200 Subject: [PATCH 252/278] test(protocols): cover continue protocol errors --- .../api/test_projects_router_protocol_ops.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index d70d8dde..ca76f315 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -837,6 +837,46 @@ def test_RestartProtocolAllReturnsSuccess(projectClient, fakeProjectService): } +def test_ContinueProtocolAllWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.continueProtocolAllError = HTTPException( + status_code=422, + detail=["cannot continue", "blocked"], + ) + + response = projectClient.post("/projects/1/protocols/10/continue-all") + + assert response.status_code == 422 + assert response.json() == { + "status": 1, + "errors": ["cannot continue", "blocked"], + "workflow": [], + } + + +def test_ContinueProtocolAllWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeContinueProtocolAll(mapper, projectId, protocolId, currentUser): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "continueProtocolAll", + fakeContinueProtocolAll, + ) + + response = projectClient.post("/projects/1/protocols/10/continue-all") + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_ContinueProtocolAllDelegatesToService(projectClient, fakeProjectService): response = projectClient.post("/projects/1/protocols/10/continue-all") From dcca6e7d537cc16dd37648fee5d0020c9c887a90 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:39:11 +0200 Subject: [PATCH 253/278] test(protocols): cover reset protocol errors --- .../api/test_projects_router_protocol_ops.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index ca76f315..ae4555af 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -909,6 +909,46 @@ def test_ContinueProtocolAllDelegatesToService(projectClient, fakeProjectService } +def test_ResetProtocolFromWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.resetProtocolFromError = HTTPException( + status_code=422, + detail=["cannot reset", "blocked"], + ) + + response = projectClient.post("/projects/1/protocols/10/reset-from") + + assert response.status_code == 422 + assert response.json() == { + "status": 1, + "errors": ["cannot reset", "blocked"], + "workflow": [], + } + + +def test_ResetProtocolFromWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeResetProtocolFrom(mapper, projectId, protocolId): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "resetProtocolFrom", + fakeResetProtocolFrom, + ) + + response = projectClient.post("/projects/1/protocols/10/reset-from") + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_ResetProtocolFromDelegatesToService(projectClient, fakeProjectService): response = projectClient.post("/projects/1/protocols/10/reset-from") From 907dae552394a448f553c2eadc6a73abbf840991 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:46:45 +0200 Subject: [PATCH 254/278] test(protocols): cover stop protocol errors --- .../api/test_projects_router_protocol_ops.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index ae4555af..22318c50 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -1075,6 +1075,52 @@ def test_ResetProtocolFromReturnsErrorWhenGraphSyncFails(projectClient, fakeProj } +def test_StopProtocolWrapsHttpException(projectClient, fakeProjectService): + fakeProjectService.stopProtocolError = HTTPException( + status_code=422, + detail=["cannot stop", "blocked"], + ) + + response = projectClient.post( + "/projects/1/protocols/stop", + json={"protocolIds": ["10"]}, + ) + + assert response.status_code == 422 + assert response.json() == { + "status": 1, + "errors": ["cannot stop", "blocked"], + "workflow": [], + } + + +def test_StopProtocolWrapsUnexpectedException( + projectClient, + fakeProjectService, + monkeypatch, +): + def fakeStopProtocol(mapper, projectId, protocolIds): + raise RuntimeError("boom") + + monkeypatch.setattr( + fakeProjectService, + "stopProtocol", + fakeStopProtocol, + ) + + response = projectClient.post( + "/projects/1/protocols/stop", + json={"protocolIds": ["10"]}, + ) + + assert response.status_code == 500 + assert response.json() == { + "status": 1, + "errors": ["boom"], + "workflow": [], + } + + def test_StopProtocolReturnsErrorWhenGraphSyncFails(projectClient, fakeProjectService): fakeProjectService.syncProjectGraphAfterMutationError = HTTPException( status_code=500, From 0fa316a6cff7ce6d370e8a82477ea2fb8c0c8769 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 16:50:35 +0200 Subject: [PATCH 255/278] test(protocols): cover missing project protocol operations --- .../api/test_projects_router_protocol_ops.py | 63 ++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 22318c50..4eb6d9d2 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -23,6 +23,7 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ****************************************************************************** +import pytest from fastapi import HTTPException @@ -83,6 +84,65 @@ def test_LoadNewProtocolReturnsParams(projectClient, fakeProjectService): } +@pytest.mark.parametrize( + ("method", "url", "payload"), + [ + ( + "post", + "/projects/1/protocols/duplicate", + {"items": [{"id": "10", "name": "Copy 1"}]}, + ), + ( + "post", + "/projects/1/protocols/delete", + {"protocolIds": ["10"]}, + ), + ( + "post", + "/projects/1/protocols/10/restart-all", + None, + ), + ( + "post", + "/projects/1/protocols/10/continue-all", + None, + ), + ( + "post", + "/projects/1/protocols/10/reset-from", + None, + ), + ( + "post", + "/projects/1/protocols/stop", + {"protocolIds": ["10"]}, + ), + ], +) +def test_ProtocolOperationsReturn404EnvelopeWhenProjectMissing( + projectClient, + fakeProjectService, + method, + url, + payload, +): + fakeProjectService.projectByIdResult = None + + request = getattr(projectClient, method) + kwargs = {} + if payload is not None: + kwargs["json"] = payload + + response = request(url, **kwargs) + + assert response.status_code == 404 + assert response.json() == { + "status": 1, + "errors": ["Project not found"], + "workflow": [], + } + + def test_LaunchProtocolReturns404EnvelopeWhenProjectMissing(projectClient, fakeProjectService): fakeProjectService.projectByIdResult = None @@ -1189,6 +1249,7 @@ def test_StopProtocolDelegatesToService(projectClient, fakeProjectService): "checkPid": True, } + def test_DeleteProtocolReturnsErrorsWhenServiceRaisesHttpException( projectClient, fakeProjectService, @@ -1253,4 +1314,4 @@ def fakeDeleteProtocol(mapper, projectId, protocolIds): "mapper": fakeProjectService.lastDeleteProtocolCall["mapper"], "projectId": 1, "protocolIds": ["10"], - } \ No newline at end of file + } From b61ea39a17ebb18571555cfee6af9ed078e82e43 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 17:15:55 +0200 Subject: [PATCH 256/278] refactor(projects): count project protocols from postgresql --- app/backend/api/services/project_service.py | 25 ++++++++++++++++----- app/backend/mapper/postgresql.py | 16 +++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index e8217e1e..4c7102d3 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -1908,6 +1908,8 @@ def listProjects(self, mapper: PostgresqlFlatMapper, currentUser) -> List[dict]: if not storedProjectPath: continue + projectId = dbProj["id"] + storedPathObj = Path(storedProjectPath).expanduser() if not storedPathObj.is_absolute(): storedPathObj = Path(self.manager.getProjectPath(str(storedPathObj))) @@ -1917,7 +1919,6 @@ def listProjects(self, mapper: PostgresqlFlatMapper, currentUser) -> List[dict]: try: realProjectPathObj = storedPathObj.resolve(strict=True) except FileNotFoundError: - # Broken entry or missing project on disk; keep it visible but degraded realProjectPathObj = storedPathObj except Exception: realProjectPathObj = storedPathObj @@ -1930,16 +1931,28 @@ def listProjects(self, mapper: PostgresqlFlatMapper, currentUser) -> List[dict]: except Exception: sizeGB = 0.0 - try: - protCount = self.countProtocols(runsPath) - except Exception: - protCount = 0 + protCount = None + + countProjectProtocols = getattr(mapper, "countProjectProtocols", None) + if callable(countProjectProtocols): + try: + protCount = int(countProjectProtocols(projectId) or 0) + except Exception: + logger.exception( + "Failed to count project protocols from PostgreSQL. projectId=%s", + projectId, + ) + + if protCount is None: + try: + protCount = self.countProtocols(runsPath) + except Exception: + protCount = 0 isOwner = dbProj.get("isOwner", dbProj.get("ownerId") == currentUser["id"]) isShared = dbProj.get("isShared", False) permission = dbProj.get("permission", "owner" if isOwner else "full") projectOwnerId = dbProj.get("ownerId") - projectId = dbProj["id"] updatedAt = dbProj.get("updatedAt") thumbnailVersion = self._buildProjectThumbnailVersion( diff --git a/app/backend/mapper/postgresql.py b/app/backend/mapper/postgresql.py index ba03e853..a6ced609 100644 --- a/app/backend/mapper/postgresql.py +++ b/app/backend/mapper/postgresql.py @@ -1303,6 +1303,22 @@ def getProtocols(self, projectId: Optional[int] = None) -> List[Dict]: (projectId,), ) + def countProjectProtocols(self, projectId: int) -> int: + row = self.db.fetchOne( + """ + SELECT COUNT(*) AS count + FROM protocols + WHERE "projectId" = %s + """, + (projectId,), + ) + + if not row: + return 0 + + value = row.get("count") if isinstance(row, dict) else row[0] + return int(value or 0) + def updateProtocol(self, protocol: Dict[str, Any]) -> None: """Update protocol fields dynamically.""" updates = [] From 25faf27c2bb0bd5b16e0864ad74b1e0bcddd1a2b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 17:20:22 +0200 Subject: [PATCH 257/278] test(projects): cover postgresql protocol counts --- .../services/test_project_service_projects.py | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/unit/backend/api/services/test_project_service_projects.py b/tests/unit/backend/api/services/test_project_service_projects.py index d6bee593..d3032b12 100644 --- a/tests/unit/backend/api/services/test_project_service_projects.py +++ b/tests/unit/backend/api/services/test_project_service_projects.py @@ -93,6 +93,8 @@ class FakeMapper: def __init__(self): self.projectsById = {} self.projectsListResult = [] + self.projectProtocolCounts = {} + self.lastCountProjectProtocolsCall = None self.insertProjectResult = 101 self.updateProjectResult = { "id": 1, @@ -137,6 +139,10 @@ def listProjects(self, ownerId=None): self.lastListProjectsCall = {"ownerId": ownerId} return self.projectsListResult + def countProjectProtocols(self, projectId): + self.lastCountProjectProtocolsCall = {"projectId": projectId} + return self.projectProtocolCounts.get(projectId, 0) + def insertProject(self, ownerId, name, description, status): self.lastInsertProjectCall = { "ownerId": ownerId, @@ -488,7 +494,8 @@ def test_ListProjectsBuildsComputedFields(service, mapper, currentUser, monkeypa ] monkeypatch.setattr(service, "getProjectSize", lambda path: 3 * 1024 ** 3) - monkeypatch.setattr(service, "countProtocols", lambda path: 7) + mapper.projectProtocolCounts[1] = 7 + monkeypatch.setattr(service, "countProtocols", lambda path: 999) monkeypatch.setattr(service, "_buildProjectThumbnailVersion", lambda **kwargs: "thumb-v1") result = service.listProjects(mapper, currentUser) @@ -513,6 +520,42 @@ def test_ListProjectsBuildsComputedFields(service, mapper, currentUser, monkeypa "thumbnailVersion": "thumb-v1", } ] + assert mapper.lastCountProjectProtocolsCall == {"projectId": 1} + + +def test_ListProjectsFallsBackToFilesystemProtocolCountWhenPostgresqlCountFails( + service, + mapper, + currentUser, + monkeypatch, +): + storedPath = Path(service.manager.PROJECTS) / "demo-project" + storedPath.mkdir(parents=True, exist_ok=True) + + mapper.projectsListResult = [ + { + "id": 1, + "name": str(storedPath), + "description": "demo description", + "createdAt": "2026-04-15T10:00:00", + "updatedAt": "2026-04-15T11:00:00", + "status": "active", + "ownerId": 1, + } + ] + + def failCountProjectProtocols(projectId): + raise RuntimeError("postgresql count failed") + + mapper.countProjectProtocols = failCountProjectProtocols + + monkeypatch.setattr(service, "getProjectSize", lambda path: 0) + monkeypatch.setattr(service, "countProtocols", lambda path: 7) + monkeypatch.setattr(service, "_buildProjectThumbnailVersion", lambda **kwargs: "thumb-v1") + + result = service.listProjects(mapper, currentUser) + + assert result[0]["protocolsCount"] == "7" def test_ListProjectsKeepsSharedFlagsFromMapper(service, mapper, currentUser, monkeypatch): From 9c7cc5fc5c90dee27fa92f30df9f63e867878f8c Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 18:14:11 +0200 Subject: [PATCH 258/278] refactor(protocols): avoid runtime load when listing protocol steps --- app/backend/api/routers/project_router.py | 2 +- .../api/test_project_router_protocol_steps.py | 22 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 25bd24e6..78ad1399 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -409,7 +409,7 @@ def listProtocolSteps( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") diff --git a/tests/integration/api/test_project_router_protocol_steps.py b/tests/integration/api/test_project_router_protocol_steps.py index 8e5597f6..01171e12 100644 --- a/tests/integration/api/test_project_router_protocol_steps.py +++ b/tests/integration/api/test_project_router_protocol_steps.py @@ -69,6 +69,8 @@ def __init__(self): } self.updateProtocolStepStatusError = None self.lastUpdateProtocolStepStatusCall = None + self.projectDbRowResult = {"id": 1, "name": "Demo project"} + self.lastGetProjectDbRowCall = None def getProjectById(self, mapper, projectId, currentUser, refresh=False, checkPid=False): self.lastGetProjectByIdCall = { @@ -88,6 +90,14 @@ def listProtocolStepsService(self, mapper, projectId, protocolId): } return self.protocolStepsResult + def getProjectDbRow(self, mapper, projectId, currentUser): + self.lastGetProjectDbRowCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + } + return self.projectDbRowResult + def updateProtocolStepStatusService( self, mapper, @@ -142,7 +152,7 @@ def test_ListProtocolStepsReturns404WhenProjectMissing( protocolStepsClient, fakeProtocolStepsProjectService, ): - fakeProtocolStepsProjectService.projectByIdResult = None + fakeProtocolStepsProjectService.projectDbRowResult = None response = protocolStepsClient.get("/projects/1/protocols/10/steps") @@ -164,6 +174,16 @@ def test_ListProtocolStepsDelegatesToService( "projectId": 1, "protocolId": 10, } + assert fakeProtocolStepsProjectService.lastGetProjectDbRowCall == { + "mapper": fakeProjectMapper, + "projectId": 1, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + } + assert fakeProtocolStepsProjectService.lastGetProjectByIdCall is None def test_UpdateProtocolStepStatusReturns404WhenProjectMissing( From 8a6539ee339ed909cb8f0a7f37a04f5bd26105cf Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 18:30:26 +0200 Subject: [PATCH 259/278] refactor(protocols): avoid runtime load when listing protocol classes --- app/backend/api/routers/project_router.py | 2 +- app/backend/api/services/project_service.py | 22 ++++++- tests/conftest.py | 24 +++++++- .../api/test_projects_router_protocol_ops.py | 58 +++++++++++++++++++ 4 files changed, 100 insertions(+), 6 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 78ad1399..5adf8347 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -365,7 +365,7 @@ def loadProtocols( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=True) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 4c7102d3..4444e6c3 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6031,13 +6031,31 @@ def walkAndReplaceProtocols(self, data: dict, resolverFn): if isinstance(item, dict): self.replaceDefaultProtocolText(item, resolverFn) + def _getProtocolClassForTreeLabel(self, protClassName: str): + try: + if self.currentProject is not None: + domain = self.currentProject.getDomain() + protocolClass = domain.getProtocols().get(protClassName, None) + if protocolClass is not None: + return protocolClass + except Exception: + pass + + try: + return Config.getDomain().getProtocols().get(protClassName, None) + except Exception: + logger.warning( + "Protocol className '%s' not found while resolving protocol tree label.", + protClassName, + ) + return None + def getProtocolName(self, node): text = node.get('text') if text: value = node.get('value') if node.get('value') is not None else text protClassName = value.split('.')[-1] - emProtocolsDict = self.currentProject.getDomain().getProtocols() - prot = emProtocolsDict.get(protClassName, None) + prot = self._getProtocolClassForTreeLabel(protClassName) if node.get('tag') == 'protocol' and text == 'default': if prot is None: diff --git a/tests/conftest.py b/tests/conftest.py index 2c229af3..c8e6958a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -97,6 +97,7 @@ def _importMainModule(): if str(ROOT_DIR) not in sys.path: sys.path.insert(0, str(ROOT_DIR)) + def makeProjectOut(projectId: int = 1, name: str = "Demo Project", **overrides): # makeProjectOut payload = { @@ -126,6 +127,10 @@ def __init__(self): self.projectByIdResult = makeProjectOut() self.lastGetProjectByIdCall = None + self.projectDbRowResult = makeProjectOut() + self.lastGetProjectDbRowCall = None + self.lastGetProtocolsCall = None + self.protocolsResult = [{"id": 11, "name": "Prot A"}] self.logChannelsResult = [{"id": "stdout", "label": "Output"}] @@ -316,6 +321,14 @@ def getProjectById( } return self.projectByIdResult + def getProjectDbRow(self, mapper, projectId, currentUser): + self.lastGetProjectDbRowCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + } + return self.projectDbRowResult + def validateProjectPostgresqlConsistency( self, mapper, @@ -333,9 +346,6 @@ def validateProjectPostgresqlConsistency( } return self.consistencyResult - def getProtocols(self, mapper, projectId, currentUser): - return self.protocolsResult - def listProjectLogChannelsService(self, projectId, protocolId): return self.logChannelsResult @@ -560,6 +570,14 @@ def listProjectShares(self, mapper, projectId, currentUser): } return self.projectSharesResult + def getProtocols(self, mapper, projectId, currentUser): + self.lastGetProtocolsCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + } + return self.protocolsResult + def syncProjectGraphAfterMutation(self, mapper, projectId, actionLabel, refresh=True, checkPid=True): self.lastSyncProjectGraphAfterMutationCall = { "mapper": mapper, diff --git a/tests/integration/api/test_projects_router_protocol_ops.py b/tests/integration/api/test_projects_router_protocol_ops.py index 4eb6d9d2..d39c71dc 100644 --- a/tests/integration/api/test_projects_router_protocol_ops.py +++ b/tests/integration/api/test_projects_router_protocol_ops.py @@ -69,6 +69,64 @@ def test_LoadProtocolReturnsParams(projectClient, fakeProjectService): } +def test_LoadProtocolsReturns404WhenProjectMissing( + projectClient, + fakeProjectService, + fakeProjectMapper, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get("/projects/1/protocols") + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + + assert fakeProjectService.lastGetProjectDbRowCall == { + "mapper": fakeProjectMapper, + "projectId": 1, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + } + + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_LoadProtocolsUsesProjectDbRow( + projectClient, + fakeProjectService, + fakeProjectMapper, +): + response = projectClient.get("/projects/1/protocols") + + assert response.status_code == 200 + assert response.json() == fakeProjectService.protocolsResult + + assert fakeProjectService.lastGetProjectDbRowCall == { + "mapper": fakeProjectMapper, + "projectId": 1, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + } + + assert fakeProjectService.lastGetProjectByIdCall is None + + assert fakeProjectService.lastGetProtocolsCall == { + "mapper": fakeProjectMapper, + "projectId": 1, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + } + + def test_LoadNewProtocolReturnsParams(projectClient, fakeProjectService): response = projectClient.get("/projects/1/protclass/MyProtClass") From 38a6b473683d98832481c64c1862f5f4a5a23344 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 18:32:57 +0200 Subject: [PATCH 260/278] refactor(protocols): avoid runtime load when listing protocol classes --- tests/integration/api/test_projects_router.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 35d7c095..2fa95f92 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -89,12 +89,13 @@ def test_CheckProjectPostgresqlConsistencyCallsService(projectClient, fakeProjec def test_LoadProtocolsReturns404WhenProjectDoesNotExist(projectClient, fakeProjectService): - fakeProjectService.projectByIdResult = None + fakeProjectService.projectDbRowResult = None response = projectClient.get("/projects/1/protocols") assert response.status_code == 404 assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectByIdCall is None def test_LoadProtocolsReturns404WhenProtocolsAreMissing(projectClient, fakeProjectService): @@ -113,6 +114,8 @@ def test_LoadProtocolsReturnsProtocols(projectClient, fakeProjectService): assert response.status_code == 200 assert response.json() == [{"id": 11, "name": "Prot A"}] + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None def test_ListProtocolLogChannelsNormalizesStringAndDictItems(projectClient, fakeProjectService): From 486bb87e8bf3b9b0c19255651a2fc948a5db8dd2 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 21:28:22 +0200 Subject: [PATCH 261/278] refactor(metadata): avoid runtime load for metadata reads --- app/backend/api/routers/project_router.py | 8 ++-- tests/integration/api/test_projects_router.py | 48 +++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 5adf8347..23470c82 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2707,7 +2707,7 @@ def listOutputMetadataTables( """ List logical metadata tables (blocks) associated with a given output. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2744,7 +2744,7 @@ def getMetadataTableSchema( """ Return logical schema for one metadata table: columns, renderers, flags. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2873,7 +2873,7 @@ def getMetadataTablePage( """ Return one logical page of rows for a metadata table. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2932,7 +2932,7 @@ def exportMetadataTable( """ Export a metadata table (full or subset) as CSV/XLSX. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 2fa95f92..defe484b 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -23,6 +23,46 @@ # * e-mail address 'scipion@cnb.csic.es' # * # ****************************************************************************** +import pytest + + +@pytest.mark.parametrize( + ("method", "url"), + [ + ( + "get", + "/projects/1/protocols/2/outputs/out/metadata/tables", + ), + ( + "get", + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/schema", + ), + ( + "get", + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/page", + ), + ( + "get", + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/export", + ), + ], +) +def test_MetadataReadEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + method, + url, +): + fakeProjectService.projectDbRowResult = None + + response = getattr(projectClient, method)(url) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + def test_ListProjectWorkflowsReturnsServiceResult(projectClient): response = projectClient.get("/projects/workflows") @@ -242,6 +282,8 @@ def test_ListMetadataTablesDelegatesMapperToService(projectClient, fakeProjectSe "outputName": "out", "mapper": fakeProjectService.lastListOutputMetadataTablesCall["mapper"], } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None def test_GetMetadataTableSchemaDelegatesMapperToService(projectClient, fakeProjectService): @@ -266,6 +308,8 @@ def test_GetMetadataTableSchemaDelegatesMapperToService(projectClient, fakeProje "tableName": "objects", "mapper": fakeProjectService.lastGetMetadataTableSchemaCall["mapper"], } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None def test_RunMetadataTableActionRejectsMissingIds(projectClient): @@ -343,6 +387,8 @@ def test_GetMetadataTablePageDelegatesMapperToService(projectClient, fakeProject "selectionOnly": True, "mapper": fakeProjectService.lastGetMetadataTablePageCall["mapper"], } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None def test_ExportMetadataTableRejectsInvalidIds(projectClient): @@ -372,6 +418,8 @@ def test_ExportMetadataTableParsesIdsAndDelegatesToService(projectClient, fakePr "ids": [1, 2, 3], "mapper": fakeProjectService.lastExportMetadataTableCall["mapper"], } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None def test_RenderMetadataImageCellDelegatesMapperToService(projectClient, fakeProjectService): From f50967b33c187772e0138209da4940a311e76621 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 21:33:20 +0200 Subject: [PATCH 262/278] refactor(metadata): avoid runtime load for metadata window reads --- app/backend/api/routers/project_router.py | 2 +- tests/conftest.py | 34 +++++++++++++++++++ tests/integration/api/test_projects_router.py | 31 +++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 23470c82..9d2a974d 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -3081,7 +3081,7 @@ def getMetadataTableWindow( """ Return a window (offset + limit) of rows for a metadata table. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Project not found") diff --git a/tests/conftest.py b/tests/conftest.py index c8e6958a..5368b8c2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -205,6 +205,13 @@ def __init__(self): } ], } + self.metadataTableWindowResult = { + "offset": 10, + "limit": 25, + "totalRows": 1, + "rows": [{"id": 1, "values": ["row-1"]}], + } + self.lastGetMetadataTableWindowCall = None self.lastGetMetadataTablePageCall = None self.renderMetadataImageCellResponse = PlainTextResponse( @@ -433,6 +440,33 @@ def getMetadataTablePageService( } return self.metadataTablePageResult + def getMetadataTableWindowService( + self, + projectId, + protocolId, + outputName, + tableName, + offset, + limit, + selectionOnly, + sortBy, + asc, + mapper, + ): + self.lastGetMetadataTableWindowCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tableName": tableName, + "offset": offset, + "limit": limit, + "selectionOnly": selectionOnly, + "sortBy": sortBy, + "asc": asc, + "mapper": mapper, + } + return self.metadataTableWindowResult + def renderMetadataImageCellService( self, projectId, diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index defe484b..ef15436f 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -63,6 +63,37 @@ def test_MetadataReadEndpointsReturn404WhenProjectMissing( assert fakeProjectService.lastGetProjectByIdCall is None +def test_GetMetadataTableWindowDelegatesMapperToService(projectClient, fakeProjectService): + fakeProjectService.metadataTableWindowResult = { + "offset": 10, + "limit": 25, + "totalRows": 1, + "rows": [{"id": 1, "values": ["row-1"]}], + } + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/rows" + "?offset=10&limit=25&sortBy=id&asc=false&selectionOnly=true" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.metadataTableWindowResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetMetadataTableWindowCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tableName": "objects", + "offset": 10, + "limit": 25, + "selectionOnly": True, + "sortBy": "id", + "asc": False, + "mapper": fakeProjectService.lastGetMetadataTableWindowCall["mapper"], + } + + def test_ListProjectWorkflowsReturnsServiceResult(projectClient): response = projectClient.get("/projects/workflows") From 31b881fc4df6ffc9aa09565f7321038fd97264fc Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 21:47:40 +0200 Subject: [PATCH 263/278] refactor(volumes): avoid runtime load for volume metadata reads --- app/backend/api/routers/project_router.py | 6 +- tests/conftest.py | 72 +++++++++++++++++ tests/integration/api/test_projects_router.py | 79 ++++++++++++++++++- 3 files changed, 153 insertions(+), 4 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 9d2a974d..1ad55768 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1657,7 +1657,7 @@ def listOutputVolumes( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1689,7 +1689,7 @@ def getVolumeInfo( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1730,7 +1730,7 @@ def getVolumeHistogram( """ Return intensity histogram for one volume. """ - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") diff --git a/tests/conftest.py b/tests/conftest.py index 5368b8c2..254bf891 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -301,6 +301,27 @@ def __init__(self): self.stopProtocolError = None self.lastStopProtocolCall = None + self.volumeItemsResult = [ + { + "id": "vol-1", + "label": "Volume 1", + "fileName": "/tmp/volume.mrc", + } + ] + self.lastListOutputVolumesCall = None + + self.volumeInfoResult = { + "id": "vol-1", + "label": "Volume 1", + "dimensions": [64, 64, 64], + } + self.lastGetVolumeInfoCall = None + + self.volumeHistogramResult = { + "binEdges": [0.0, 1.0], + "counts": [10], + } + self.lastGetVolumeHistogramCall = None def listProjectWorkflows(self): if self.listProjectWorkflowsError is not None: @@ -752,6 +773,57 @@ def stopProtocol(self, mapper, projectId, protocolIds): if self.stopProtocolError is not None: raise self.stopProtocolError + def listOutputVolumesService( + self, + projectId, + protocolId, + outputName, + mapper=None, + ): + self.lastListOutputVolumesCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + return self.volumeItemsResult + + def getVolumeInfoService( + self, + projectId, + protocolId, + outputName, + volumeId, + mapper=None, + ): + self.lastGetVolumeInfoCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "volumeId": volumeId, + "mapper": mapper, + } + return self.volumeInfoResult + + def getVolumeHistogramService( + self, + projectId, + protocolId, + outputName, + volumeId, + bins=128, + mapper=None, + ): + self.lastGetVolumeHistogramCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "volumeId": volumeId, + "bins": bins, + "mapper": mapper, + } + return self.volumeHistogramResult + @pytest.fixture def authTestEnv(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index ef15436f..7ddb8605 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -478,4 +478,81 @@ def test_RenderMetadataImageCellDelegatesMapperToService(projectClient, fakeProj "inline": False, "fmt": "jpeg", "mapper": call["mapper"], - } \ No newline at end of file + } + + +def test_ListOutputVolumesUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/volumes" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.volumeItemsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListOutputVolumesCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastListOutputVolumesCall["mapper"], + } + + +def test_GetVolumeInfoUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/volumes/vol-1/info" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.volumeInfoResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetVolumeInfoCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "volumeId": "vol-1", + "mapper": fakeProjectService.lastGetVolumeInfoCall["mapper"], + } + + +def test_GetVolumeHistogramUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/volumes/vol-1/histogram?bins=32" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.volumeHistogramResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetVolumeHistogramCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "volumeId": "vol-1", + "bins": 32, + "mapper": fakeProjectService.lastGetVolumeHistogramCall["mapper"], + } + + +@pytest.mark.parametrize( + "url", + [ + "/projects/1/protocols/2/outputs/out/volumes", + "/projects/1/protocols/2/outputs/out/volumes/vol-1/info", + "/projects/1/protocols/2/outputs/out/volumes/vol-1/histogram", + ], +) +def test_VolumeReadEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + url, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get(url) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None \ No newline at end of file From 1d364b20bd2f3daf80c9e4a2b6d22536173e0d7d Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 22:03:05 +0200 Subject: [PATCH 264/278] refactor(tomography): avoid runtime load for tomography metadata reads --- app/backend/api/routers/project_router.py | 8 +- tests/conftest.py | 102 ++++++++++++++++++ tests/integration/api/test_projects_router.py | 94 ++++++++++++++++ 3 files changed, 200 insertions(+), 4 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 1ad55768..863de9a3 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1908,7 +1908,7 @@ def listOutputTiltSeries( """ List tilt series for a SetOfTiltSeries-like output. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1945,7 +1945,7 @@ def getTiltSeriesFrames( """ Return metadata for all tilt images in one tilt series. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2178,7 +2178,7 @@ def listCtftomoSeries( """ List CTFTomoSeries entries for a CTFTomo output. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2206,7 +2206,7 @@ def getCtftomoSeriesViews( """ Return all CTF measurements for one tilt-series (identified by tiltSeriesId). """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") diff --git a/tests/conftest.py b/tests/conftest.py index 254bf891..16b5b592 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -323,6 +323,44 @@ def __init__(self): } self.lastGetVolumeHistogramCall = None + self.tiltSeriesResult = [ + { + "tiltSeriesId": "TS_001", + "label": "TS_001", + "nViews": 3, + } + ] + self.lastListOutputTiltSeriesCall = None + + self.tiltSeriesFramesResult = { + "tiltSeriesId": "TS_001", + "label": "TS_001", + "frames": [ + {"index": 0, "tiltAngle": -1.0}, + {"index": 1, "tiltAngle": 0.0}, + ], + } + self.lastGetTiltSeriesFramesCall = None + + self.ctftomoSeriesResult = [ + { + "tiltSeriesId": "TS_001", + "label": "TS_001", + "nViews": 3, + } + ] + self.lastListOutputCtftomoSeriesCall = None + + self.ctftomoSeriesViewsResult = { + "tiltSeriesId": "TS_001", + "label": "TS_001", + "frames": [ + {"index": 0, "defocusU": 10000.0}, + {"index": 1, "defocusU": 11000.0}, + ], + } + self.lastGetCtftomoSeriesViewsCall = None + def listProjectWorkflows(self): if self.listProjectWorkflowsError is not None: raise self.listProjectWorkflowsError @@ -824,6 +862,70 @@ def getVolumeHistogramService( } return self.volumeHistogramResult + def listOutputTiltSeriesService( + self, + projectId, + protocolId, + outputName, + mapper=None, + ): + self.lastListOutputTiltSeriesCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + return self.tiltSeriesResult + + def getTiltSeriesFramesService( + self, + projectId, + protocolId, + outputName, + tiltSeriesId, + mapper=None, + ): + self.lastGetTiltSeriesFramesCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tiltSeriesId": tiltSeriesId, + "mapper": mapper, + } + return self.tiltSeriesFramesResult + + def listOutputCtftomoSeriesService( + self, + projectId, + protocolId, + outputName, + mapper=None, + ): + self.lastListOutputCtftomoSeriesCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + return self.ctftomoSeriesResult + + def getCtftomoSeriesViewsService( + self, + projectId, + protocolId, + outputName, + tiltSeriesId, + mapper=None, + ): + self.lastGetCtftomoSeriesViewsCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tiltSeriesId": tiltSeriesId, + "mapper": mapper, + } + return self.ctftomoSeriesViewsResult + @pytest.fixture def authTestEnv(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 7ddb8605..f3c50441 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -552,6 +552,100 @@ def test_VolumeReadEndpointsReturn404WhenProjectMissing( response = projectClient.get(url) + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_ListOutputTiltSeriesUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/tiltseries" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.tiltSeriesResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListOutputTiltSeriesCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastListOutputTiltSeriesCall["mapper"], + } + + +def test_GetTiltSeriesFramesUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/tiltseries/TS_001/frames" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.tiltSeriesFramesResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetTiltSeriesFramesCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tiltSeriesId": "TS_001", + "mapper": fakeProjectService.lastGetTiltSeriesFramesCall["mapper"], + } + + +def test_ListCtftomoSeriesUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/ctftomo" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.ctftomoSeriesResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListOutputCtftomoSeriesCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastListOutputCtftomoSeriesCall["mapper"], + } + + +def test_GetCtftomoSeriesViewsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/ctftomo/TS_001/views" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.ctftomoSeriesViewsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetCtftomoSeriesViewsCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tiltSeriesId": "TS_001", + "mapper": fakeProjectService.lastGetCtftomoSeriesViewsCall["mapper"], + } + + +@pytest.mark.parametrize( + "url", + [ + "/projects/1/protocols/2/outputs/out/tiltseries", + "/projects/1/protocols/2/outputs/out/tiltseries/TS_001/frames", + "/projects/1/protocols/2/outputs/out/ctftomo", + "/projects/1/protocols/2/outputs/out/ctftomo/TS_001/views", + ], +) +def test_TomographyReadEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + url, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get(url) + assert response.status_code == 404 assert response.json()["detail"] == "Project not found" assert fakeProjectService.lastGetProjectDbRowCall is not None From c924925bd8b81aa235aaf3613fd2f3c24767f8a6 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 22:12:34 +0200 Subject: [PATCH 265/278] refactor(coords3d): avoid runtime load for coordinates metadata reads --- app/backend/api/routers/project_router.py | 6 +- tests/conftest.py | 84 ++++++++++++++++++- tests/integration/api/test_projects_router.py | 75 +++++++++++++++++ 3 files changed, 161 insertions(+), 4 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 863de9a3..3966d592 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2372,7 +2372,7 @@ def listCoordinates3dTomograms( """ List tomograms referenced by a SetOfCoordinates3D output. """ - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2411,7 +2411,7 @@ def getCoordinates3dPoints( """ Return all 3D coordinates for one tomogram inside a SetOfCoordinates3D. """ - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2588,7 +2588,7 @@ def getIntegratedAnalyzeContext( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") diff --git a/tests/conftest.py b/tests/conftest.py index 16b5b592..0a1ba0c4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -361,6 +361,40 @@ def __init__(self): } self.lastGetCtftomoSeriesViewsCall = None + self.coords3dTomogramsResult = [ + { + "id": "tomo-1", + "name": "Tomogram 1", + "label": "tomo-1", + "dims": [64, 64, 64], + "voxelSize": [1.0, 1.0, 1.0], + } + ] + self.lastListCoordinates3dTomogramsCall = None + + self.coords3dPointsResult = [ + { + "id": 1, + "x": 10.0, + "y": 20.0, + "z": 30.0, + "tomoId": "tomo-1", + } + ] + self.lastGetCoordinates3dPointsCall = None + + self.integratedAnalyzeContextResult = { + "root": { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + }, + "links": {}, + "summaries": {}, + "relations": {"items": []}, + } + self.lastGetIntegratedAnalyzeContextCall = None + def listProjectWorkflows(self): if self.listProjectWorkflowsError is not None: raise self.listProjectWorkflowsError @@ -926,6 +960,53 @@ def getCtftomoSeriesViewsService( } return self.ctftomoSeriesViewsResult + def listCoordinates3dTomogramsService( + self, + projectId, + protocolId, + outputName, + mapper=None, + ): + self.lastListCoordinates3dTomogramsCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + return self.coords3dTomogramsResult + + def getCoordinates3dPointsService( + self, + projectId, + protocolId, + outputName, + tomogramId, + mapper=None, + ): + self.lastGetCoordinates3dPointsCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tomogramId": tomogramId, + "mapper": mapper, + } + return self.coords3dPointsResult + + def getIntegratedAnalyzeContextService( + self, + projectId, + protocolId, + outputName, + mapper=None, + ): + self.lastGetIntegratedAnalyzeContextCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + return self.integratedAnalyzeContextResult + @pytest.fixture def authTestEnv(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): @@ -1181,4 +1262,5 @@ async def fakeSendVerificationEmail(email: str, code: str): with TestClient(app) as client: yield client - app.dependency_overrides.clear() \ No newline at end of file + app.dependency_overrides.clear() + diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index f3c50441..bd582b09 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -646,6 +646,81 @@ def test_TomographyReadEndpointsReturn404WhenProjectMissing( response = projectClient.get(url) + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_ListCoordinates3dTomogramsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/coords3d/tomograms" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.coords3dTomogramsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListCoordinates3dTomogramsCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastListCoordinates3dTomogramsCall["mapper"], + } + + +def test_GetCoordinates3dPointsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/coords3d/tomograms/tomo-1" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.coords3dPointsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetCoordinates3dPointsCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tomogramId": "tomo-1", + "mapper": fakeProjectService.lastGetCoordinates3dPointsCall["mapper"], + } + + +def test_GetIntegratedAnalyzeContextUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/integrated-context" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.integratedAnalyzeContextResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetIntegratedAnalyzeContextCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastGetIntegratedAnalyzeContextCall["mapper"], + } + + +@pytest.mark.parametrize( + "url", + [ + "/projects/1/protocols/2/outputs/out/coords3d/tomograms", + "/projects/1/protocols/2/outputs/out/coords3d/tomograms/tomo-1", + "/projects/1/protocols/2/outputs/out/integrated-context", + ], +) +def test_Coordinates3dReadEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + url, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get(url) + assert response.status_code == 404 assert response.json()["detail"] == "Project not found" assert fakeProjectService.lastGetProjectDbRowCall is not None From 4fc4bbf960979dd0999841aa94ce346c99e1ce8f Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 22:30:11 +0200 Subject: [PATCH 266/278] refactor(metadata): avoid runtime load for metadata image reads --- app/backend/api/routers/project_router.py | 2 +- tests/integration/api/test_projects_router.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 3966d592..9e78e4d7 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -3018,7 +3018,7 @@ def renderMetadataImageCell( """ Render one image cell from a metadata table using the same logic as ImageRenderer. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=True, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index bd582b09..a01b0206 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -45,6 +45,10 @@ "get", "/projects/1/protocols/2/outputs/out/metadata/tables/objects/export", ), + ( + "get", + "/projects/1/protocols/2/outputs/out/metadata/tables/objects/image?column=stack", + ), ], ) def test_MetadataReadEndpointsReturn404WhenProjectMissing( @@ -462,6 +466,8 @@ def test_RenderMetadataImageCellDelegatesMapperToService(projectClient, fakeProj assert response.status_code == 200 assert response.text == "image-bytes" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None call = fakeProjectService.lastRenderMetadataImageCellCall assert str(call["rowId"]) == "7" From fec54ff2f1ecc57fe22e26e1ad76382d07ca608b Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 22:36:59 +0200 Subject: [PATCH 267/278] refactor(tags): avoid runtime load for tag endpoints --- app/backend/api/routers/project_router.py | 14 +- tests/conftest.py | 109 +++++++++++++++ tests/integration/api/test_projects_router.py | 126 ++++++++++++++++++ 3 files changed, 242 insertions(+), 7 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 9e78e4d7..b6d67c2b 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -3220,7 +3220,7 @@ def listProjectTags( service: ProjectService = Depends(getProjectService), ): # ensureProjectExists - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -3240,7 +3240,7 @@ def createProjectTag( service: ProjectService = Depends(getProjectService), ): # ensureProjectExists - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -3272,7 +3272,7 @@ def updateProjectTag( service: ProjectService = Depends(getProjectService), ): # ensureProjectExists - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -3298,7 +3298,7 @@ def deleteProjectTag( service: ProjectService = Depends(getProjectService), ): # ensureProjectExists - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -3319,7 +3319,7 @@ def listProtocolTags( service: ProjectService = Depends(getProjectService), ): # ensureProjectExists - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -3345,7 +3345,7 @@ def setProtocolTags( service: ProjectService = Depends(getProjectService), ): # ensureProjectExists - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -3369,7 +3369,7 @@ async def getContextMenuVisibilityPolicy( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") return service.getContextMenuVisibilityPolicy() diff --git a/tests/conftest.py b/tests/conftest.py index 0a1ba0c4..4158138a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -395,6 +395,56 @@ def __init__(self): } self.lastGetIntegratedAnalyzeContextCall = None + self.projectTagsResult = [ + { + "id": "tag-1", + "title": "Good", + "description": "Good protocols", + "color": "#00ff00", + } + ] + self.lastListProjectTagsCall = None + + self.createProjectTagResult = { + "id": "tag-2", + "title": "New tag", + "description": None, + "color": "#ff0000", + } + self.lastCreateProjectTagCall = None + + self.updateProjectTagResult = { + "id": "tag-1", + "title": "Updated tag", + "description": "Updated", + "color": "#0000ff", + } + self.lastUpdateProjectTagCall = None + + self.deleteProjectTagResult = True + self.lastDeleteProjectTagCall = None + + self.protocolTagsResult = { + "protocolId": "2", + "protocolDbId": 22, + "tagIds": ["tag-1"], + } + self.lastListProtocolTagsCall = None + + self.setProtocolTagsResult = { + "protocolId": "2", + "protocolDbId": 22, + "tagIds": ["tag-1", "tag-2"], + } + self.lastSetProtocolTagsCall = None + + self.contextMenuVisibilityResult = { + "open": True, + "delete": True, + "manageTags": True, + } + self.lastGetContextMenuVisibilityPolicyCall = None + def listProjectWorkflows(self): if self.listProjectWorkflowsError is not None: raise self.listProjectWorkflowsError @@ -1007,6 +1057,65 @@ def getIntegratedAnalyzeContextService( } return self.integratedAnalyzeContextResult + def listProjectTags(self, mapper, projectId, currentUser): + self.lastListProjectTagsCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + } + return self.projectTagsResult + + def createProjectTag(self, mapper, projectId, currentUser, payload): + self.lastCreateProjectTagCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + "payload": payload, + } + return self.createProjectTagResult + + def updateProjectTag(self, mapper, projectId, tagId, currentUser, payload): + self.lastUpdateProjectTagCall = { + "mapper": mapper, + "projectId": projectId, + "tagId": tagId, + "currentUser": currentUser, + "payload": payload, + } + return self.updateProjectTagResult + + def deleteProjectTag(self, mapper, projectId, tagId, currentUser): + self.lastDeleteProjectTagCall = { + "mapper": mapper, + "projectId": projectId, + "tagId": tagId, + "currentUser": currentUser, + } + return self.deleteProjectTagResult + + def listProtocolTags(self, mapper, projectId, protocolId, currentUser): + self.lastListProtocolTagsCall = { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + "currentUser": currentUser, + } + return self.protocolTagsResult + + def setProtocolTags(self, mapper, projectId, protocolId, tagIds, currentUser): + self.lastSetProtocolTagsCall = { + "mapper": mapper, + "projectId": projectId, + "protocolId": protocolId, + "tagIds": tagIds, + "currentUser": currentUser, + } + return self.setProtocolTagsResult + + def getContextMenuVisibilityPolicy(self): + self.lastGetContextMenuVisibilityPolicyCall = {} + return self.contextMenuVisibilityResult + @pytest.fixture def authTestEnv(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index a01b0206..55ef57e1 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -727,6 +727,132 @@ def test_Coordinates3dReadEndpointsReturn404WhenProjectMissing( response = projectClient.get(url) + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_ListProjectTagsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get("/projects/1/tags") + + assert response.status_code == 200 + assert response.json() == fakeProjectService.projectTagsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListProjectTagsCall["projectId"] == 1 + + +def test_CreateProjectTagUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.post( + "/projects/1/tags", + json={ + "title": "New tag", + "description": None, + "color": "#ff0000", + }, + ) + + assert response.status_code == 201 + assert response.json() == fakeProjectService.createProjectTagResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastCreateProjectTagCall["projectId"] == 1 + assert fakeProjectService.lastCreateProjectTagCall["payload"].title == "New tag" + + +def test_UpdateProjectTagUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.put( + "/projects/1/tags/tag-1", + json={ + "title": "Updated tag", + "description": "Updated", + "color": "#0000ff", + }, + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.updateProjectTagResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastUpdateProjectTagCall["projectId"] == 1 + assert fakeProjectService.lastUpdateProjectTagCall["tagId"] == "tag-1" + + +def test_DeleteProjectTagUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.delete("/projects/1/tags/tag-1") + + assert response.status_code == 200 + assert response.json() == {"success": True} + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastDeleteProjectTagCall["projectId"] == 1 + assert fakeProjectService.lastDeleteProjectTagCall["tagId"] == "tag-1" + + +def test_ListProtocolTagsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get("/projects/1/protocols/2/tags") + + assert response.status_code == 200 + assert response.json() == fakeProjectService.protocolTagsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListProtocolTagsCall["projectId"] == 1 + assert fakeProjectService.lastListProtocolTagsCall["protocolId"] == 2 + + +def test_SetProtocolTagsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.put( + "/projects/1/protocols/2/tags", + json={"tagIds": ["tag-1", "tag-2"]}, + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.setProtocolTagsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastSetProtocolTagsCall["projectId"] == 1 + assert fakeProjectService.lastSetProtocolTagsCall["protocolId"] == 2 + assert fakeProjectService.lastSetProtocolTagsCall["tagIds"] == ["tag-1", "tag-2"] + + +def test_GetContextMenuVisibilityUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get("/projects/1/context-menu-visibility") + + assert response.status_code == 200 + assert response.json() == fakeProjectService.contextMenuVisibilityResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetContextMenuVisibilityPolicyCall == {} + + +@pytest.mark.parametrize( + ("method", "url", "payload"), + [ + ("get", "/projects/1/tags", None), + ("post", "/projects/1/tags", {"title": "New tag"}), + ("put", "/projects/1/tags/tag-1", {"title": "Updated"}), + ("delete", "/projects/1/tags/tag-1", None), + ("get", "/projects/1/protocols/2/tags", None), + ("put", "/projects/1/protocols/2/tags", {"tagIds": ["tag-1"]}), + ("get", "/projects/1/context-menu-visibility", None), + ], +) +def test_TagAndContextMenuEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + method, + url, + payload, +): + fakeProjectService.projectDbRowResult = None + + request = getattr(projectClient, method) + if payload is None: + response = request(url) + else: + response = request(url, json=payload) + assert response.status_code == 404 assert response.json()["detail"] == "Project not found" assert fakeProjectService.lastGetProjectDbRowCall is not None From 773aa3b2be80882f5fd8de591e1d404d137b4ac6 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 22:54:15 +0200 Subject: [PATCH 268/278] refactor(fsc): avoid runtime load for fsc rows --- app/backend/api/routers/project_router.py | 8 +---- tests/conftest.py | 28 +++++++++++++++++ tests/integration/api/test_projects_router.py | 30 +++++++++++++++++++ 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index b6d67c2b..4a45811b 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2641,13 +2641,7 @@ def getFscRows( "threshold": 0.143, } """ - project = service.getProjectById( - mapper, - projectId, - currentUser, - refresh=False, - checkPid=False, - ) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") diff --git a/tests/conftest.py b/tests/conftest.py index 4158138a..b0eed4fe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -445,6 +445,19 @@ def __init__(self): } self.lastGetContextMenuVisibilityPolicyCall = None + self.fscRowsResult = { + "threshold": 0.143, + "curves": [ + { + "label": "FSC 1", + "resolution": 3.2, + "x": [0.01, 0.02], + "y": [0.95, 0.87], + } + ], + } + self.lastGetFscRowsCall = None + def listProjectWorkflows(self): if self.listProjectWorkflowsError is not None: raise self.listProjectWorkflowsError @@ -1116,6 +1129,21 @@ def getContextMenuVisibilityPolicy(self): self.lastGetContextMenuVisibilityPolicyCall = {} return self.contextMenuVisibilityResult + def getFscRowsService( + self, + projectId, + protocolId, + outputName, + mapper=None, + ): + self.lastGetFscRowsCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "mapper": mapper, + } + return self.fscRowsResult + @pytest.fixture def authTestEnv(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 55ef57e1..5f9b5989 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -853,6 +853,36 @@ def test_TagAndContextMenuEndpointsReturn404WhenProjectMissing( else: response = request(url, json=payload) + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_GetFscRowsUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/fsc/rows" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.fscRowsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetFscRowsCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "mapper": fakeProjectService.lastGetFscRowsCall["mapper"], + } + + +def test_GetFscRowsReturns404WhenProjectMissing(projectClient, fakeProjectService): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/fsc/rows" + ) + assert response.status_code == 404 assert response.json()["detail"] == "Project not found" assert fakeProjectService.lastGetProjectDbRowCall is not None From 9f6e386ce48f594de626cbd359de6154091793a7 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 23:37:30 +0200 Subject: [PATCH 269/278] refactor(projects): avoid runtime load for effective settings --- app/backend/api/routers/project_router.py | 8 +--- app/backend/api/services/project_service.py | 8 +--- tests/conftest.py | 18 +++++++++ tests/integration/api/test_projects_router.py | 38 ++++++++++++++++++- 4 files changed, 57 insertions(+), 15 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 4a45811b..71934041 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -211,13 +211,7 @@ def getProjectEffectiveSettings( The project must be accessible by the authenticated user. """ - project = service.getProjectById( - mapper, - projectId, - currentUser, - refresh=False, - checkPid=False, - ) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 4444e6c3..7e165967 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -2145,13 +2145,7 @@ def getProjectEffectiveSettings( currentUser: Any, ) -> Dict[str, Any]: # getProjectEffectiveSettings - project = self.getProjectById( - mapper, - projectId, - currentUser, - refresh=False, - checkPid=False, - ) + project = self.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, diff --git a/tests/conftest.py b/tests/conftest.py index b0eed4fe..3e9a56e6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -458,6 +458,16 @@ def __init__(self): } self.lastGetFscRowsCall = None + self.projectEffectiveSettingsResult = { + "projectId": 1, + "settings": { + "user": {"protocolView": "tree"}, + "instance": {"executionMode": "local"}, + "host": {"queueSystem": "slurm"}, + }, + } + self.lastGetProjectEffectiveSettingsCall = None + def listProjectWorkflows(self): if self.listProjectWorkflowsError is not None: raise self.listProjectWorkflowsError @@ -1144,6 +1154,14 @@ def getFscRowsService( } return self.fscRowsResult + def getProjectEffectiveSettings(self, mapper, projectId, currentUser): + self.lastGetProjectEffectiveSettingsCall = { + "mapper": mapper, + "projectId": projectId, + "currentUser": currentUser, + } + return self.projectEffectiveSettingsResult + @pytest.fixture def authTestEnv(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 5f9b5989..01c65ba5 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -886,4 +886,40 @@ def test_GetFscRowsReturns404WhenProjectMissing(projectClient, fakeProjectServic assert response.status_code == 404 assert response.json()["detail"] == "Project not found" assert fakeProjectService.lastGetProjectDbRowCall is not None - assert fakeProjectService.lastGetProjectByIdCall is None \ No newline at end of file + assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_GetProjectEffectiveSettingsUsesProjectDbRow( + projectClient, + fakeProjectService, +): + response = projectClient.get("/projects/1/effective-settings") + + assert response.status_code == 200 + assert response.json() == fakeProjectService.projectEffectiveSettingsResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetProjectEffectiveSettingsCall == { + "mapper": fakeProjectService.lastGetProjectEffectiveSettingsCall["mapper"], + "projectId": 1, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + } + + +def test_GetProjectEffectiveSettingsReturns404WhenProjectMissing( + projectClient, + fakeProjectService, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get("/projects/1/effective-settings") + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetProjectEffectiveSettingsCall is None \ No newline at end of file From b69e144619fd0257e24c20ce6d0c4af68bcbe058 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 23:42:49 +0200 Subject: [PATCH 270/278] refactor(coords3d): avoid runtime load for tomogram slice render --- app/backend/api/routers/project_router.py | 2 +- tests/conftest.py | 45 +++++++++++++++- tests/integration/api/test_projects_router.py | 52 ++++++++++++++++++- 3 files changed, 96 insertions(+), 3 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 71934041..30d085ef 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2487,7 +2487,7 @@ def renderCoords3dTomogramSlice( """ Render a 2D slice from a tomogram referenced by a SetOfCoordinates3D. """ - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") diff --git a/tests/conftest.py b/tests/conftest.py index 3e9a56e6..09377be4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ import pytest from fastapi import APIRouter, FastAPI from fastapi.testclient import TestClient -from starlette.responses import PlainTextResponse +from starlette.responses import PlainTextResponse, Response ROOT_DIR = Path(__file__).resolve().parents[1] if str(ROOT_DIR) not in sys.path: @@ -468,6 +468,12 @@ def __init__(self): } self.lastGetProjectEffectiveSettingsCall = None + self.coords3dSliceResponse = Response( + content=b"slice-bytes", + media_type="image/png", + ) + self.lastRenderCoords3dTomogramSliceCall = None + def listProjectWorkflows(self): if self.listProjectWorkflowsError is not None: raise self.listProjectWorkflowsError @@ -1162,6 +1168,43 @@ def getProjectEffectiveSettings(self, mapper, projectId, currentUser): } return self.projectEffectiveSettingsResult + def renderCoords3dTomogramSliceService( + self, + projectId, + protocolId, + outputName, + tomogramId, + sliceIndex, + axis="z", + colormap=None, + normalize="minmax", + scale=1.0, + inline=True, + fmt="webp", + thumb=None, + fast=True, + quality=75, + mapper=None, + ): + self.lastRenderCoords3dTomogramSliceCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tomogramId": tomogramId, + "sliceIndex": sliceIndex, + "axis": axis, + "colormap": colormap, + "normalize": normalize, + "scale": scale, + "inline": inline, + "fmt": fmt, + "thumb": thumb, + "fast": fast, + "quality": quality, + "mapper": mapper, + } + return self.coords3dSliceResponse + @pytest.fixture def authTestEnv(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 01c65ba5..5d7fc4e3 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -922,4 +922,54 @@ def test_GetProjectEffectiveSettingsReturns404WhenProjectMissing( assert response.json()["detail"] == "Project not found" assert fakeProjectService.lastGetProjectDbRowCall is not None assert fakeProjectService.lastGetProjectByIdCall is None - assert fakeProjectService.lastGetProjectEffectiveSettingsCall is None \ No newline at end of file + assert fakeProjectService.lastGetProjectEffectiveSettingsCall is None + + +def test_RenderCoords3dTomogramSliceUsesProjectDbRow( + projectClient, + fakeProjectService, +): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/coords3d/tomograms/tomo-1/slice" + "?index=4&axis=y&cmap=gray&normalize=zscore&scale=1.5" + "&inline=false&format=png&thumb=128&fast=false&quality=80" + ) + + assert response.status_code == 200 + assert response.content == b"slice-bytes" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderCoords3dTomogramSliceCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tomogramId": "tomo-1", + "sliceIndex": 4, + "axis": "y", + "colormap": "gray", + "normalize": "zscore", + "scale": 1.5, + "inline": False, + "fmt": "png", + "thumb": 128, + "fast": False, + "quality": 80, + "mapper": fakeProjectService.lastRenderCoords3dTomogramSliceCall["mapper"], + } + + +def test_RenderCoords3dTomogramSliceReturns404WhenProjectMissing( + projectClient, + fakeProjectService, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/coords3d/tomograms/tomo-1/slice?index=4" + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderCoords3dTomogramSliceCall is None \ No newline at end of file From 57c1a65748b184cc128d141ba0e3d9b156a9f819 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Mon, 29 Jun 2026 23:59:45 +0200 Subject: [PATCH 271/278] refactor(viewers): avoid runtime load when resolving analyze viewer --- app/backend/api/routers/project_router.py | 2 +- tests/conftest.py | 41 ++++++++---- tests/integration/api/test_projects_router.py | 65 ++++++++++++++++++- 3 files changed, 95 insertions(+), 13 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 30d085ef..0ec09494 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1613,7 +1613,7 @@ def resolveAnalyzeViewer( service: ProjectService = Depends(getProjectService), ): # resolveAnalyzeViewer - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") diff --git a/tests/conftest.py b/tests/conftest.py index 09377be4..c8ab6c5f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -474,6 +474,14 @@ def __init__(self): ) self.lastRenderCoords3dTomogramSliceCall = None + self.resolveAnalyzeViewerDecisionResult = { + "handled": False, + } + self.resolveViewerError = None + + self.lastResolveViewerCall = None + self.lastResolveAnalyzeViewerDecisionCall = None + def listProjectWorkflows(self): if self.listProjectWorkflowsError is not None: raise self.listProjectWorkflowsError @@ -542,17 +550,6 @@ def pollProtocolLogsService(self, projectId, protocolId, offsets, maxBytes, maxL } return self.pollLogsResult - def resolveAnalyzeViewerDecision(self, projectId, protocolId, ctx, mapper=None): - self.lastResolveViewerCall = { - "projectId": projectId, - "protocolId": protocolId, - "ctx": ctx, - "mapper": mapper - } - if self.resolveViewerError is not None: - raise self.resolveViewerError - return self.resolveViewerResult - def listOutputMetadataTablesService( self, projectId, @@ -1205,6 +1202,28 @@ def renderCoords3dTomogramSliceService( } return self.coords3dSliceResponse + def resolveAnalyzeViewerDecision( + self, + projectId, + protocolId, + ctx, + mapper=None, + ): + call = { + "projectId": projectId, + "protocolId": protocolId, + "ctx": ctx, + "mapper": mapper, + } + + self.lastResolveViewerCall = call + self.lastResolveAnalyzeViewerDecisionCall = call + + if self.resolveViewerError is not None: + raise self.resolveViewerError + + return self.resolveAnalyzeViewerDecisionResult + @pytest.fixture def authTestEnv(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 5d7fc4e3..07ea6e93 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -281,6 +281,8 @@ def test_ResolveAnalyzeViewerReturnsHandledFalseOnUnexpectedError(projectClient, "handled": False, "message": "viewer failed", } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None assert fakeProjectService.lastResolveViewerCall == { "projectId": 1, "protocolId": 22, @@ -972,4 +974,65 @@ def test_RenderCoords3dTomogramSliceReturns404WhenProjectMissing( assert response.json()["detail"] == "Project not found" assert fakeProjectService.lastGetProjectDbRowCall is not None assert fakeProjectService.lastGetProjectByIdCall is None - assert fakeProjectService.lastRenderCoords3dTomogramSliceCall is None \ No newline at end of file + assert fakeProjectService.lastRenderCoords3dTomogramSliceCall is None + + +def resolveAnalyzeViewerDecision( + self, + projectId, + protocolId, + ctx, + mapper=None, +): + self.lastResolveAnalyzeViewerDecisionCall = { + "projectId": projectId, + "protocolId": protocolId, + "ctx": ctx, + "mapper": mapper, + } + return self.resolveAnalyzeViewerDecisionResult + + +def test_ResolveAnalyzeViewerUnwrapsCtxAndUsesProjectDbRow( + projectClient, + fakeProjectService, +): + response = projectClient.post( + "/projects/1/protocols/2/viewer/resolve", + json={ + "ctx": { + "outputName": "out", + "objectId": "tomo-1", + "objectKind": "tomogram", + } + }, + ) + + assert response.status_code == 200 + assert response.json() == {"handled": False} + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastResolveViewerCall == { + "projectId": 1, + "protocolId": 22, + "ctx": {"outputName": "particles", "outputClass": "SetOfParticles"}, + "mapper": fakeProjectService.lastResolveViewerCall["mapper"], + } + + +def test_ResolveAnalyzeViewerReturns404WhenProjectMissing( + projectClient, + fakeProjectService, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.post( + "/projects/1/protocols/2/viewer/resolve", + json={"outputName": "out"}, + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastResolveAnalyzeViewerDecisionCall is None \ No newline at end of file From 7afe58c046298b4c6544a2900058119d64177c58 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 30 Jun 2026 00:02:34 +0200 Subject: [PATCH 272/278] Fixing resolve analyze tests --- tests/integration/api/test_projects_router.py | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 07ea6e93..7c2911af 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -997,26 +997,32 @@ def test_ResolveAnalyzeViewerUnwrapsCtxAndUsesProjectDbRow( projectClient, fakeProjectService, ): + ctx = { + "outputName": "out", + "objectId": "tomo-1", + "objectKind": "tomogram", + } + response = projectClient.post( "/projects/1/protocols/2/viewer/resolve", - json={ - "ctx": { - "outputName": "out", - "objectId": "tomo-1", - "objectKind": "tomogram", - } - }, + json={"ctx": ctx}, ) assert response.status_code == 200 - assert response.json() == {"handled": False} + assert response.json() == fakeProjectService.resolveAnalyzeViewerDecisionResult assert fakeProjectService.lastGetProjectDbRowCall is not None assert fakeProjectService.lastGetProjectByIdCall is None - assert fakeProjectService.lastResolveViewerCall == { + + call = ( + fakeProjectService.lastResolveAnalyzeViewerDecisionCall + or fakeProjectService.lastResolveViewerCall + ) + + assert call == { "projectId": 1, - "protocolId": 22, - "ctx": {"outputName": "particles", "outputClass": "SetOfParticles"}, - "mapper": fakeProjectService.lastResolveViewerCall["mapper"], + "protocolId": 2, + "ctx": ctx, + "mapper": call["mapper"], } From 54dac0eadbbd04509e8fb5e67fb66af90f628333 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 30 Jun 2026 00:10:23 +0200 Subject: [PATCH 273/278] refactor(volumes): avoid runtime load for volume render endpoints --- app/backend/api/routers/project_router.py | 6 +- tests/conftest.py | 105 +++++++++++++++++ tests/integration/api/test_projects_router.py | 107 +++++++++++++++++- 3 files changed, 214 insertions(+), 4 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 0ec09494..4e5ca967 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1771,7 +1771,7 @@ def renderVolumeSlice( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1816,7 +1816,7 @@ def getVolumeData3d( currentUser: Dict[str, Any] = Depends(getCurrentUser), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1849,7 +1849,7 @@ def getVolumeSurfaceMesh( mapper: PostgresqlFlatMapper = Depends(getMapper), service: ProjectService = Depends(getProjectService), ): - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") diff --git a/tests/conftest.py b/tests/conftest.py index c8ab6c5f..59bc3495 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -482,6 +482,26 @@ def __init__(self): self.lastResolveViewerCall = None self.lastResolveAnalyzeViewerDecisionCall = None + self.volumeSliceResponse = Response( + content=b"volume-slice-bytes", + media_type="image/png", + ) + self.lastRenderVolumeSliceCall = None + + self.volumeData3dResult = { + "dims": [2, 2, 2], + "values": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0], + } + self.lastGetVolumeData3dCall = None + + self.volumeSurfaceMeshResult = { + "vertices": [[0.0, 0.0, 0.0]], + "faces": [[0, 0, 0]], + "level": 0.5, + "volumeId": "vol-1", + } + self.lastGetVolumeSurfaceMeshCall = None + def listProjectWorkflows(self): if self.listProjectWorkflowsError is not None: raise self.listProjectWorkflowsError @@ -1224,6 +1244,91 @@ def resolveAnalyzeViewerDecision( return self.resolveAnalyzeViewerDecisionResult + def renderVolumeSliceService( + self, + projectId, + protocolId, + outputName, + volumeId, + sliceIndex, + axis, + colormap, + normalize, + scale, + inline, + fmt="webp", + thumb=None, + fast=True, + quality=75, + mapper=None, + ): + self.lastRenderVolumeSliceCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "volumeId": volumeId, + "sliceIndex": sliceIndex, + "axis": axis, + "colormap": colormap, + "normalize": normalize, + "scale": scale, + "inline": inline, + "fmt": fmt, + "thumb": thumb, + "fast": fast, + "quality": quality, + "mapper": mapper, + } + return self.volumeSliceResponse + + def getVolumeData3dService( + self, + projectId, + protocolId, + outputName, + volumeId, + maxDim=160, + method="binning", + mapper=None, + ): + self.lastGetVolumeData3dCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "volumeId": volumeId, + "maxDim": maxDim, + "method": method, + "mapper": mapper, + } + return self.volumeData3dResult + + def getVolumeSurfaceMesh( + self, + projectId, + protocolId, + outputName, + volumeId, + level=None, + maxDim=192, + method="stride", + maxTriangles=350000, + currentUser=None, + mapper=None, + ): + self.lastGetVolumeSurfaceMeshCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "volumeId": volumeId, + "level": level, + "maxDim": maxDim, + "method": method, + "maxTriangles": maxTriangles, + "currentUser": currentUser, + "mapper": mapper, + } + return self.volumeSurfaceMeshResult + @pytest.fixture def authTestEnv(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 7c2911af..2d460948 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -1041,4 +1041,109 @@ def test_ResolveAnalyzeViewerReturns404WhenProjectMissing( assert response.json()["detail"] == "Project not found" assert fakeProjectService.lastGetProjectDbRowCall is not None assert fakeProjectService.lastGetProjectByIdCall is None - assert fakeProjectService.lastResolveAnalyzeViewerDecisionCall is None \ No newline at end of file + assert fakeProjectService.lastResolveAnalyzeViewerDecisionCall is None + + +def test_RenderVolumeSliceUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/volumes/vol-1/slice" + "?index=4&axis=y&cmap=gray&normalize=zscore&scale=1.5" + "&inline=false&format=png&thumb=128&fast=false&quality=80" + ) + + assert response.status_code == 200 + assert response.content == b"volume-slice-bytes" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderVolumeSliceCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "volumeId": "vol-1", + "sliceIndex": 4, + "axis": "y", + "colormap": "gray", + "normalize": "zscore", + "scale": 1.5, + "inline": False, + "fmt": "png", + "thumb": 128, + "fast": False, + "quality": 80, + "mapper": fakeProjectService.lastRenderVolumeSliceCall["mapper"], + } + + +def test_GetVolumeData3dUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/volumes/vol-1/data3d" + "?maxDim=96&method=stride" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.volumeData3dResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetVolumeData3dCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "volumeId": "vol-1", + "maxDim": 96, + "method": "stride", + "mapper": fakeProjectService.lastGetVolumeData3dCall["mapper"], + } + + +def test_GetVolumeSurfaceMeshUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/volumes/vol-1/surface" + "?level=0.5&maxDim=96&method=stride&maxTriangles=1234" + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.volumeSurfaceMeshResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastGetVolumeSurfaceMeshCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "volumeId": "vol-1", + "level": 0.5, + "maxDim": 96, + "method": "stride", + "maxTriangles": 1234, + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + "mapper": fakeProjectService.lastGetVolumeSurfaceMeshCall["mapper"], + } + + +@pytest.mark.parametrize( + "url", + [ + "/projects/1/protocols/2/outputs/out/volumes/vol-1/slice?index=4", + "/projects/1/protocols/2/outputs/out/volumes/vol-1/data3d", + "/projects/1/protocols/2/outputs/out/volumes/vol-1/surface", + ], +) +def test_VolumeRenderEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + url, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get(url) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderVolumeSliceCall is None + assert fakeProjectService.lastGetVolumeData3dCall is None + assert fakeProjectService.lastGetVolumeSurfaceMeshCall is None \ No newline at end of file From d4d563016829f2bb8f57d9288c035c1e3d8b5840 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 30 Jun 2026 00:18:49 +0200 Subject: [PATCH 274/278] refactor(tiltseries): avoid runtime load for tilt image renders --- app/backend/api/routers/project_router.py | 4 +- tests/conftest.py | 81 +++++++++++++++ tests/integration/api/test_projects_router.py | 99 ++++++++++++++++++- 3 files changed, 181 insertions(+), 3 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 4e5ca967..1c308e81 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2007,7 +2007,7 @@ def renderTiltSeriesImage( """ Render one tilt image from a tilt series. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2055,7 +2055,7 @@ def renderTiltSeriesImagesBatch( Render several tilt images from the same tilt series in one request. This is intended for smooth slider prefetching in the web viewer. """ - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") diff --git a/tests/conftest.py b/tests/conftest.py index 59bc3495..57f713e8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -502,6 +502,29 @@ def __init__(self): } self.lastGetVolumeSurfaceMeshCall = None + self.tiltSeriesImageResponse = Response( + content=b"tilt-image-bytes", + media_type="image/png", + ) + self.lastRenderTiltSeriesImageCall = None + + self.tiltSeriesBatchResult = { + "tiltSeriesId": "TS_001", + "size": 256, + "fmt": "webp", + "applyTransform": True, + "items": [ + { + "index": 0, + "contentType": "image/png", + "dataUrl": "data:image/png;base64,AAAA", + "cache": "MISS", + } + ], + "errors": [], + } + self.lastRenderTiltSeriesImagesBatchCall = None + def listProjectWorkflows(self): if self.listProjectWorkflowsError is not None: raise self.listProjectWorkflowsError @@ -1329,6 +1352,64 @@ def getVolumeSurfaceMesh( } return self.volumeSurfaceMeshResult + def renderTiltSeriesImageService( + self, + projectId, + protocolId, + outputName, + tiltSeriesId, + index=0, + size=1024, + fmt="png", + applyTransform=True, + inline=True, + requestHeaders=None, + mapper=None, + ): + self.lastRenderTiltSeriesImageCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tiltSeriesId": tiltSeriesId, + "index": index, + "size": size, + "fmt": fmt, + "applyTransform": applyTransform, + "inline": inline, + "requestHeaders": requestHeaders, + "mapper": mapper, + } + return self.tiltSeriesImageResponse + + def renderTiltSeriesImagesBatchService( + self, + projectId, + protocolId, + outputName, + tiltSeriesId, + indices, + size=512, + fmt="webp", + applyTransform=True, + inline=True, + requestHeaders=None, + mapper=None, + ): + self.lastRenderTiltSeriesImagesBatchCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "tiltSeriesId": tiltSeriesId, + "indices": indices, + "size": size, + "fmt": fmt, + "applyTransform": applyTransform, + "inline": inline, + "requestHeaders": requestHeaders, + "mapper": mapper, + } + return self.tiltSeriesBatchResult + @pytest.fixture def authTestEnv(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index 2d460948..acdcdd1a 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -1146,4 +1146,101 @@ def test_VolumeRenderEndpointsReturn404WhenProjectMissing( assert fakeProjectService.lastGetProjectByIdCall is None assert fakeProjectService.lastRenderVolumeSliceCall is None assert fakeProjectService.lastGetVolumeData3dCall is None - assert fakeProjectService.lastGetVolumeSurfaceMeshCall is None \ No newline at end of file + assert fakeProjectService.lastGetVolumeSurfaceMeshCall is None + + +def test_RenderTiltSeriesImageUsesProjectDbRow(projectClient, fakeProjectService): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/tiltseries/TS_001/tilt" + "?index=3&size=256&fmt=webp&applyTransform=true&inline=false" + ) + + assert response.status_code == 200 + assert response.content == b"tilt-image-bytes" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderTiltSeriesImageCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tiltSeriesId": "TS_001", + "index": 3, + "size": 256, + "fmt": "webp", + "applyTransform": True, + "inline": False, + "requestHeaders": None, + "mapper": fakeProjectService.lastRenderTiltSeriesImageCall["mapper"], + } + + +def test_RenderTiltSeriesImagesBatchUsesProjectDbRow( + projectClient, + fakeProjectService, +): + response = projectClient.post( + "/projects/1/protocols/2/outputs/out/tiltseries/TS_001/tilt/batch", + json={ + "indices": [0, 2, 4], + "size": 256, + "fmt": "webp", + "applyTransform": True, + "inline": False, + }, + ) + + assert response.status_code == 200 + assert response.json() == fakeProjectService.tiltSeriesBatchResult + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderTiltSeriesImagesBatchCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "tiltSeriesId": "TS_001", + "indices": [0, 2, 4], + "size": 256, + "fmt": "webp", + "applyTransform": True, + "inline": False, + "requestHeaders": None, + "mapper": fakeProjectService.lastRenderTiltSeriesImagesBatchCall["mapper"], + } + + +@pytest.mark.parametrize( + ("method", "url", "payload"), + [ + ( + "get", + "/projects/1/protocols/2/outputs/out/tiltseries/TS_001/tilt?index=3", + None, + ), + ( + "post", + "/projects/1/protocols/2/outputs/out/tiltseries/TS_001/tilt/batch", + {"indices": [0, 2]}, + ), + ], +) +def test_TiltSeriesRenderEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + method, + url, + payload, +): + fakeProjectService.projectDbRowResult = None + + request = getattr(projectClient, method) + if payload is None: + response = request(url) + else: + response = request(url, json=payload) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderTiltSeriesImageCall is None + assert fakeProjectService.lastRenderTiltSeriesImagesBatchCall is None \ No newline at end of file From 99eafab771261ed2c185ef4c8b4b995002728e6e Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 30 Jun 2026 00:25:44 +0200 Subject: [PATCH 275/278] refactor(ctftomo): avoid runtime load for psd renders --- app/backend/api/routers/project_router.py | 10 +-- tests/conftest.py | 44 ++++++++++++ tests/integration/api/test_projects_router.py | 72 ++++++++++++++++++- 3 files changed, 118 insertions(+), 8 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 1c308e81..0413c417 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -2310,13 +2310,7 @@ def renderCtftomoPsdImage( Render a PSD image for a CTF-tomography view, given a stack spec (for example '3@/path/to/TS_1.mrc'). """ - project = service.getProjectById( - mapper, - projectId, - currentUser, - refresh=False, - checkPid=False, - ) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -2338,6 +2332,8 @@ def renderCtftomoPsdImage( ) resp.headers["Vary"] = "Authorization" return resp + except HTTPException: + raise except Exception as e: logger.exception("Error in renderCtftomoPsdImage: %s", e) raise HTTPException( diff --git a/tests/conftest.py b/tests/conftest.py index 57f713e8..8225fa6c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -525,6 +525,13 @@ def __init__(self): } self.lastRenderTiltSeriesImagesBatchCall = None + self.ctftomoPsdResponse = Response( + content=b"ctftomo-psd-bytes", + media_type="image/png", + ) + self.lastRenderCtfTomoPsdImageCall = None + self.renderCtfTomoPsdImageError = None + def listProjectWorkflows(self): if self.listProjectWorkflowsError is not None: raise self.listProjectWorkflowsError @@ -1410,6 +1417,43 @@ def renderTiltSeriesImagesBatchService( } return self.tiltSeriesBatchResult + def renderCtfTomoPsdImageService( + self, + projectId, + protocolId, + outputName, + psdPath, + size=1024, + fmt="png", + inline=True, + index=0, + quality=75, + applyTransform=False, + rot=None, + shifts=None, + mapper=None, + ): + self.lastRenderCtfTomoPsdImageCall = { + "projectId": projectId, + "protocolId": protocolId, + "outputName": outputName, + "psdPath": psdPath, + "size": size, + "fmt": fmt, + "inline": inline, + "index": index, + "quality": quality, + "applyTransform": applyTransform, + "rot": rot, + "shifts": shifts, + "mapper": mapper, + } + + if self.renderCtfTomoPsdImageError is not None: + raise self.renderCtfTomoPsdImageError + + return self.ctftomoPsdResponse + @pytest.fixture def authTestEnv(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index acdcdd1a..b0dc8cbc 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -24,6 +24,7 @@ # * # ****************************************************************************** import pytest +from fastapi import HTTPException @pytest.mark.parametrize( @@ -1243,4 +1244,73 @@ def test_TiltSeriesRenderEndpointsReturn404WhenProjectMissing( assert fakeProjectService.lastGetProjectDbRowCall is not None assert fakeProjectService.lastGetProjectByIdCall is None assert fakeProjectService.lastRenderTiltSeriesImageCall is None - assert fakeProjectService.lastRenderTiltSeriesImagesBatchCall is None \ No newline at end of file + assert fakeProjectService.lastRenderTiltSeriesImagesBatchCall is None + + +def test_RenderCtftomoPsdImageUsesProjectDbRow( + projectClient, + fakeProjectService, +): + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/ctftomo/psd" + "?spec=3%40%2Ftmp%2Fpsd.mrc&size=256&fmt=webp" + "&applyTransform=true&inline=false" + ) + + assert response.status_code == 200 + assert response.content == b"ctftomo-psd-bytes" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderCtfTomoPsdImageCall == { + "projectId": 1, + "protocolId": 2, + "outputName": "out", + "psdPath": "3@/tmp/psd.mrc", + "size": 256, + "fmt": "webp", + "inline": False, + "index": 0, + "quality": 75, + "applyTransform": True, + "rot": None, + "shifts": None, + "mapper": fakeProjectService.lastRenderCtfTomoPsdImageCall["mapper"], + } + + +def test_RenderCtftomoPsdImageReturns404WhenProjectMissing( + projectClient, + fakeProjectService, +): + fakeProjectService.projectDbRowResult = None + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/ctftomo/psd" + "?spec=3%40%2Ftmp%2Fpsd.mrc" + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastRenderCtfTomoPsdImageCall is None + + +def test_RenderCtftomoPsdImagePreservesHttpException( + projectClient, + fakeProjectService, +): + fakeProjectService.renderCtfTomoPsdImageError = HTTPException( + status_code=404, + detail="PSD image file not found", + ) + + response = projectClient.get( + "/projects/1/protocols/2/outputs/out/ctftomo/psd" + "?spec=3%40%2Ftmp%2Fpsd.mrc" + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "PSD image file not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None From 4a15c82ba81896f2e03c708a9d480cce51b08337 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 30 Jun 2026 09:42:09 +0200 Subject: [PATCH 276/278] refactor(coords2d): add postgresql reader for coordinates metadata --- app/backend/api/services/coords2d_service.py | 99 ++++- .../viewers/postgresql_coords2d_reader.py | 366 ++++++++++++++++++ .../test_coords2d_service_postgresql.py | 116 ++++++ .../test_postgresql_coords2d_reader.py | 163 ++++++++ 4 files changed, 737 insertions(+), 7 deletions(-) create mode 100644 app/backend/viewers/postgresql_coords2d_reader.py create mode 100644 tests/unit/backend/api/services/test_coords2d_service_postgresql.py create mode 100644 tests/unit/backend/viewers/test_postgresql_coords2d_reader.py diff --git a/app/backend/api/services/coords2d_service.py b/app/backend/api/services/coords2d_service.py index 720a2350..1d1ee2a9 100644 --- a/app/backend/api/services/coords2d_service.py +++ b/app/backend/api/services/coords2d_service.py @@ -280,6 +280,45 @@ def _countCoordinatesByMicrograph(self, coordinatesSet: Any) -> Dict[str, int]: except Exception: return counts + def _getPostgresqlCoords2dReaderIfAvailable( + self, + mapper: PostgresqlFlatMapper, + projectId: int, + protocolId: int, + outputName: str, + ): + if mapper is None: + return None + + try: + from app.backend.viewers.postgresql_coords2d_reader import PostgresqlCoords2dReader + + readerProtocolId = self.projectService._resolvePostgresqlReaderProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + + reader = PostgresqlCoords2dReader( + db=mapper.db, + projectId=projectId, + protocolId=readerProtocolId, + outputName=outputName, + ) + + if reader.hasOutput(): + return reader + + except Exception: + logger.exception( + "Failed to initialize PostgreSQL Coords2D reader. projectId=%s protocolId=%s outputName=%s", + projectId, + protocolId, + outputName, + ) + + return None + def listMicrographs( self, mapper: PostgresqlFlatMapper, @@ -288,6 +327,29 @@ def listMicrographs( protocolId: int, outputName: str, ) -> Dict[str, Any]: + pgReader = self._getPostgresqlCoords2dReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.listMicrographs() + if payload is not None: + return payload + + if mapper is not None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=( + "Coordinates2D output is not available in PostgreSQL metadata" + if pgReader is None + else "Coordinates2D micrographs are not available in PostgreSQL metadata: %s" + % getattr(pgReader, "lastSkipReason", None) + ), + ) + _, coordinatesSet = self._loadCoordinatesOutput( mapper, projectId, @@ -342,14 +404,37 @@ def _findMicrograph(self, coordinatesSet: Any, micId: str) -> Any: return micrograph def listCoordinatesForMicrograph( - self, - mapper: PostgresqlFlatMapper, - projectId: int, - currentUser: Any, - protocolId: int, - outputName: str, - micId: str, + self, + mapper: PostgresqlFlatMapper, + projectId: int, + currentUser: Any, + protocolId: int, + outputName: str, + micId: str, ) -> Dict[str, Any]: + pgReader = self._getPostgresqlCoords2dReaderIfAvailable( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + outputName=outputName, + ) + + if pgReader is not None: + payload = pgReader.listCoordinatesForMicrograph(micId) + if payload is not None: + return payload + + if mapper is not None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=( + "Coordinates2D output is not available in PostgreSQL metadata" + if pgReader is None + else "Coordinates2D coordinates are not available in PostgreSQL metadata: %s" + % getattr(pgReader, "lastSkipReason", None) + ), + ) + _, coordinatesSet = self._loadCoordinatesOutput( mapper, projectId, diff --git a/app/backend/viewers/postgresql_coords2d_reader.py b/app/backend/viewers/postgresql_coords2d_reader.py new file mode 100644 index 00000000..2f3339ba --- /dev/null +++ b/app/backend/viewers/postgresql_coords2d_reader.py @@ -0,0 +1,366 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from typing import Any, Dict, List, Optional, Set + +from app.backend.mapper.scipion_set_mapper import ScipionSetPostgresqlMapper + + +class PostgresqlCoords2dReader: + def __init__(self, db, projectId: int, protocolId: int, outputName: str): + self.db = db + self.projectId = projectId + self.protocolId = protocolId + self.outputName = outputName + self.setMapper = ScipionSetPostgresqlMapper(db) + self._storedSet = None + self.lastSkipReason = None + + def hasOutput(self) -> bool: + storedSet = self._getStoredSet() + return storedSet is not None and self._isCoords2dStoredSet(storedSet) + + def listMicrographs(self) -> Optional[Dict[str, Any]]: + self.lastSkipReason = None + + storedSet = self._getStoredSet() + if storedSet is None: + self.lastSkipReason = "stored_set_not_found" + return None + + if not self._isCoords2dStoredSet(storedSet): + self.lastSkipReason = "stored_set_is_not_coordinates2d" + return None + + countsByMicId: Dict[str, int] = {} + micrographsById: Dict[str, Dict[str, Any]] = {} + + for item in storedSet.get("items") or []: + values = item.get("values") or {} + micId = self._extractMicId(item, values) + + if not micId: + continue + + countsByMicId[micId] = countsByMicId.get(micId, 0) + 1 + + if micId not in micrographsById: + micrographsById[micId] = self._buildMicrographSummary( + micId=micId, + item=item, + values=values, + ) + + if not micrographsById: + self.lastSkipReason = "micrographs_not_resolved_from_coordinates" + return None + + micrographs: List[Dict[str, Any]] = [] + for index, micId in enumerate(sorted(micrographsById.keys(), key=self._micrographSortKey), start=1): + summary = dict(micrographsById[micId]) + summary["index"] = index + summary["particles"] = int(countsByMicId.get(micId, 0)) + micrographs.append(summary) + + boxSize = self._extractBoxSize(storedSet) + totalPicks = sum(int(item.get("particles") or 0) for item in micrographs) + + return { + "micrographs": micrographs, + "totalMicrographs": len(micrographs), + "totalPicks": totalPicks, + "boxSize": boxSize, + } + + def listCoordinatesForMicrograph(self, micId: Any) -> Optional[Dict[str, Any]]: + self.lastSkipReason = None + + storedSet = self._getStoredSet() + if storedSet is None: + self.lastSkipReason = "stored_set_not_found" + return None + + if not self._isCoords2dStoredSet(storedSet): + self.lastSkipReason = "stored_set_is_not_coordinates2d" + return None + + targetMicId = str(micId) + coordinates: List[Dict[str, Any]] = [] + + for index, item in enumerate(storedSet.get("items") or []): + values = item.get("values") or {} + itemMicId = self._extractMicId(item, values) + + if str(itemMicId) != targetMicId: + continue + + point = self._buildCoordinatePoint( + item=item, + values=values, + micId=targetMicId, + fallbackIndex=index, + ) + + if point is not None: + coordinates.append(point) + + if not coordinates: + self.lastSkipReason = "coordinates_not_found_for_micrograph micId=%s" % targetMicId + return None + + return {"coordinates": coordinates} + + def _getStoredSet(self) -> Optional[Dict[str, Any]]: + if self._storedSet is None: + self._storedSet = self.setMapper.getStoredSet( + projectId=self.projectId, + protocolDbId=self.protocolId, + outputName=self.outputName, + ) + return self._storedSet + + def _isCoords2dStoredSet(self, storedSet: Dict[str, Any]) -> bool: + classText = ("%s %s" % ( + storedSet.get("setClassName") or "", + storedSet.get("itemClassName") or "", + )).replace(" ", "").lower() + + if "coordinate" not in classText: + return False + + if "coordinates3d" in classText or "coordinate3d" in classText: + return False + + if "tomogram" in classText or "tomo" in classText: + return False + + return True + + def _buildMicrographSummary( + self, + micId: str, + item: Dict[str, Any], + values: Dict[str, Any], + ) -> Dict[str, Any]: + label = ( + self._firstValueBySuffix(values, ["micname", "micrographname", "filename", "filepath"]) + or item.get("label") + or "Micrograph %s" % micId + ) + + fileName = self._firstValueBySuffix( + values, + ["filename", "filepath", "micfilename", "micrographfilename", "path"], + ) + + width = self._toOptionalInt( + self._firstValueBySuffix(values, ["width", "xdim", "dimx"]) + ) + height = self._toOptionalInt( + self._firstValueBySuffix(values, ["height", "ydim", "dimy"]) + ) + + return { + "id": str(micId), + "fileName": str(fileName) if fileName else "", + "label": str(label), + "particles": 0, + "updated": False, + "width": width, + "height": height, + "locationIndex": self._toOptionalInt( + self._firstValueBySuffix(values, ["locationindex", "imageindex", "index"]) + ), + "thumbnailUrl": None, + } + + def _buildCoordinatePoint( + self, + item: Dict[str, Any], + values: Dict[str, Any], + micId: str, + fallbackIndex: int, + ) -> Optional[Dict[str, Any]]: + x = self._extractCoordinateValue(values, ["_x", "x", "coordx", "coordinatex", "positionx"]) + y = self._extractCoordinateValue(values, ["_y", "y", "coordy", "coordinatey", "positiony"]) + + if x is None or y is None: + return None + + objId = item.get("scipionItemId") + if objId is None: + objId = item.get("id") + if objId is None: + objId = "%s:%s" % (micId, fallbackIndex) + + return { + "id": objId, + "micId": str(micId), + "x": x, + "y": y, + "score": self._toOptionalFloat( + self._firstValueBySuffix(values, ["score", "weight"]) + ), + "classLabel": self._optionalString( + self._firstValueBySuffix(values, ["classid", "classlabel", "objlabel"]) + ), + } + + def _extractMicId(self, item: Dict[str, Any], values: Dict[str, Any]) -> Optional[str]: + value = self._firstValue( + values, + [ + "_micId", + "micId", + "micrographId", + "micrograph.id", + "micrograph._objId", + "_mic._objId", + "_micrograph._objId", + ], + ) + + if value is None: + value = self._firstValueBySuffix( + values, + ["micid", "micrographid", "micobjid", "micrographobjid"], + ) + + if value is None: + value = self._findNestedValueBySuffix( + values, + ["micid", "micrographid", "objid"], + ) + + if value is None: + return None + + return str(value) + + def _extractBoxSize(self, storedSet: Dict[str, Any]) -> Optional[int]: + properties = storedSet.get("properties") or {} + setProperties = storedSet.get("setProperties") or [] + + value = None + if isinstance(properties, dict): + value = self._firstValueBySuffix(properties, ["boxsize", "box"]) + + if value is None: + for row in setProperties: + key = str((row or {}).get("key") or "").replace("_", "").lower() + if key.endswith("boxsize") or key == "box": + value = (row or {}).get("value") + break + + return self._toOptionalInt(value) + + def _extractCoordinateValue(self, values: Dict[str, Any], names: List[str]) -> Optional[float]: + value = self._firstValue(values, names) + if value is None: + value = self._firstValueBySuffix(values, names) + return self._toOptionalFloat(value) + + def _firstValue(self, values: Dict[str, Any], keys: List[str]) -> Any: + for key in keys: + if key in values: + return values.get(key) + return None + + def _firstValueBySuffix(self, values: Dict[str, Any], suffixes: List[str]) -> Any: + normalizedSuffixes = [ + str(suffix).replace("_", "").replace(".", "").lower() + for suffix in suffixes + ] + + for key, value in (values or {}).items(): + normalizedKey = str(key).replace("_", "").replace(".", "").lower() + for suffix in normalizedSuffixes: + if normalizedKey.endswith(suffix): + return value + + return None + + def _findNestedValueBySuffix(self, value: Any, suffixes: List[str]) -> Any: + normalizedSuffixes = [ + str(suffix).replace("_", "").replace(".", "").lower() + for suffix in suffixes + ] + + def walk(node: Any, path: str = "") -> Any: + if isinstance(node, dict): + for key, child in node.items(): + nextPath = "%s.%s" % (path, key) if path else str(key) + normalizedPath = nextPath.replace("_", "").replace(".", "").lower() + for suffix in normalizedSuffixes: + if normalizedPath.endswith(suffix): + return child + + found = walk(child, nextPath) + if found is not None: + return found + + if isinstance(node, list): + for index, child in enumerate(node): + found = walk(child, "%s.%s" % (path, index)) + if found is not None: + return found + + return None + + return walk(value) + + def _toOptionalFloat(self, value: Any) -> Optional[float]: + if value is None or value == "": + return None + + try: + return float(value) + except Exception: + return None + + def _toOptionalInt(self, value: Any) -> Optional[int]: + if value is None or value == "": + return None + + try: + return int(value) + except Exception: + try: + return int(float(value)) + except Exception: + return None + + def _optionalString(self, value: Any) -> Optional[str]: + if value is None: + return None + + text = str(value).strip() + return text or None + + def _micrographSortKey(self, micId: str): + try: + return 0, int(micId) + except Exception: + return 1, str(micId).lower() \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_coords2d_service_postgresql.py b/tests/unit/backend/api/services/test_coords2d_service_postgresql.py new file mode 100644 index 00000000..bcf16778 --- /dev/null +++ b/tests/unit/backend/api/services/test_coords2d_service_postgresql.py @@ -0,0 +1,116 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from fastapi import HTTPException +import pytest + +from app.backend.api.services.coords2d_service import Coords2dService + + +class FakeReader: + def __init__(self, micrographs=None, coordinates=None): + self.micrographs = micrographs + self.coordinates = coordinates + self.lastSkipReason = None + + def listMicrographs(self): + return self.micrographs + + def listCoordinatesForMicrograph(self, micId): + return self.coordinates + + +def test_Coords2dServiceListMicrographsUsesPostgresqlReader(monkeypatch): + service = Coords2dService() + expected = { + "micrographs": [{"id": "10", "particles": 2}], + "totalMicrographs": 1, + "totalPicks": 2, + "boxSize": 128, + } + + monkeypatch.setattr( + service, + "_getPostgresqlCoords2dReaderIfAvailable", + lambda **kwargs: FakeReader(micrographs=expected), + ) + + payload = service.listMicrographs( + mapper=object(), + projectId=1, + currentUser={"id": 1}, + protocolId=2, + outputName="coordinates", + ) + + assert payload == expected + + +def test_Coords2dServiceListCoordinatesUsesPostgresqlReader(monkeypatch): + service = Coords2dService() + expected = { + "coordinates": [ + {"id": 1, "micId": "10", "x": 11.5, "y": 22.5} + ] + } + + monkeypatch.setattr( + service, + "_getPostgresqlCoords2dReaderIfAvailable", + lambda **kwargs: FakeReader(coordinates=expected), + ) + + payload = service.listCoordinatesForMicrograph( + mapper=object(), + projectId=1, + currentUser={"id": 1}, + protocolId=2, + outputName="coordinates", + micId="10", + ) + + assert payload == expected + + +def test_Coords2dServiceRaisesWhenPostgresqlReaderMissing(monkeypatch): + service = Coords2dService() + + monkeypatch.setattr( + service, + "_getPostgresqlCoords2dReaderIfAvailable", + lambda **kwargs: None, + ) + + with pytest.raises(HTTPException) as exc: + service.listMicrographs( + mapper=object(), + projectId=1, + currentUser={"id": 1}, + protocolId=2, + outputName="coordinates", + ) + + assert exc.value.status_code == 404 + assert "Coordinates2D output is not available in PostgreSQL metadata" in str(exc.value.detail) \ No newline at end of file diff --git a/tests/unit/backend/viewers/test_postgresql_coords2d_reader.py b/tests/unit/backend/viewers/test_postgresql_coords2d_reader.py new file mode 100644 index 00000000..06d97405 --- /dev/null +++ b/tests/unit/backend/viewers/test_postgresql_coords2d_reader.py @@ -0,0 +1,163 @@ +# ****************************************************************************** +# * +# * Authors: Yunior C. Fonseca Reyna +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 3 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ****************************************************************************** +from app.backend.viewers.postgresql_coords2d_reader import PostgresqlCoords2dReader + + +def make_reader(stored_set): + reader = PostgresqlCoords2dReader( + db=None, + projectId=1, + protocolId=2, + outputName="coordinates", + ) + reader._storedSet = stored_set + return reader + + +def test_PostgresqlCoords2dReaderListsMicrographs(): + reader = make_reader( + { + "setClassName": "SetOfCoordinates", + "itemClassName": "Coordinate", + "properties": {"boxSize": 128}, + "items": [ + { + "scipionItemId": 1, + "values": { + "_micId": 10, + "_x": 11.5, + "_y": 22.5, + }, + }, + { + "scipionItemId": 2, + "values": { + "_micId": 10, + "_x": 12.5, + "_y": 23.5, + }, + }, + { + "scipionItemId": 3, + "values": { + "_micId": 20, + "_x": 5.0, + "_y": 6.0, + }, + }, + ], + } + ) + + payload = reader.listMicrographs() + + assert payload == { + "micrographs": [ + { + "id": "10", + "index": 1, + "fileName": "", + "label": "Micrograph 10", + "particles": 2, + "updated": False, + "width": None, + "height": None, + "locationIndex": None, + "thumbnailUrl": None, + }, + { + "id": "20", + "index": 2, + "fileName": "", + "label": "Micrograph 20", + "particles": 1, + "updated": False, + "width": None, + "height": None, + "locationIndex": None, + "thumbnailUrl": None, + }, + ], + "totalMicrographs": 2, + "totalPicks": 3, + "boxSize": 128, + } + + +def test_PostgresqlCoords2dReaderListsCoordinatesForMicrograph(): + reader = make_reader( + { + "setClassName": "SetOfCoordinates", + "itemClassName": "Coordinate", + "items": [ + { + "scipionItemId": 1, + "values": { + "_micId": 10, + "_x": 11.5, + "_y": 22.5, + "_score": 0.9, + "_classId": 2, + }, + }, + { + "scipionItemId": 2, + "values": { + "_micId": 20, + "_x": 5.0, + "_y": 6.0, + }, + }, + ], + } + ) + + payload = reader.listCoordinatesForMicrograph("10") + + assert payload == { + "coordinates": [ + { + "id": 1, + "micId": "10", + "x": 11.5, + "y": 22.5, + "score": 0.9, + "classLabel": "2", + } + ] + } + + +def test_PostgresqlCoords2dReaderRejectsCoords3dStoredSet(): + reader = make_reader( + { + "setClassName": "SetOfCoordinates3D", + "itemClassName": "Coordinate3D", + "items": [], + } + ) + + assert reader.hasOutput() is False \ No newline at end of file From 554922de3e2bdb0621faf9c70720ed19868176c9 Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 30 Jun 2026 10:18:07 +0200 Subject: [PATCH 277/278] Removing deprecated tests --- .../api/services/test_coords2d_service.py | 148 ------------------ 1 file changed, 148 deletions(-) diff --git a/tests/unit/backend/api/services/test_coords2d_service.py b/tests/unit/backend/api/services/test_coords2d_service.py index 0c028c62..a0172d47 100644 --- a/tests/unit/backend/api/services/test_coords2d_service.py +++ b/tests/unit/backend/api/services/test_coords2d_service.py @@ -430,151 +430,3 @@ def test_LoadCoordinatesOutputRaisesWhenOutputIsNotCoordinatesSet( } ] - -def test_ListMicrographsResolvesPostgresqlProtocolId( - service, - currentProject, - projectService, - mapper, - currentUser, -): - protocol = FakeProtocol(objId=10) - protocol.outputCoordinates = buildCoordinatesOutput() - - currentProject.protocols[10] = protocol - projectService.runtimeProtocolIdByDbId[500] = 10 - - result = service.listMicrographs( - mapper=mapper, - projectId=1, - currentUser=currentUser, - protocolId=500, - outputName="outputCoordinates", - ) - - assert result == { - "micrographs": [ - { - "id": "1", - "index": 1, - "fileName": "/data/micrograph-a.mrc", - "label": "Micrograph A", - "particles": 2, - "updated": False, - "width": 4096, - "height": 3072, - "locationIndex": 0, - "thumbnailUrl": None, - }, - { - "id": "2", - "index": 2, - "fileName": "/data/micrograph-b.mrc", - "label": "Micrograph B label", - "particles": 1, - "updated": False, - "width": 2048, - "height": 2048, - "locationIndex": 1, - "thumbnailUrl": None, - }, - ], - "totalMicrographs": 2, - "totalPicks": 3, - "boxSize": 96, - } - - assert projectService.runtimeCalls == [ - { - "mapper": mapper, - "projectId": 1, - "protocolId": 500, - } - ] - - -def test_ListCoordinatesForMicrographResolvesPostgresqlProtocolId( - service, - currentProject, - projectService, - mapper, - currentUser, -): - protocol = FakeProtocol(objId=10) - protocol.outputCoordinates = buildCoordinatesOutput() - - currentProject.protocols[10] = protocol - projectService.runtimeProtocolIdByDbId[500] = 10 - - result = service.listCoordinatesForMicrograph( - mapper=mapper, - projectId=1, - currentUser=currentUser, - protocolId=500, - outputName="outputCoordinates", - micId="1", - ) - - assert result == { - "coordinates": [ - { - "id": 101, - "micId": "1", - "x": 10.5, - "y": 20.5, - "score": 0.95, - "classLabel": "good", - }, - { - "id": 102, - "micId": "1", - "x": 30.0, - "y": 40.0, - "score": 0.75, - "classLabel": "bad", - }, - ], - } - - assert projectService.runtimeCalls == [ - { - "mapper": mapper, - "projectId": 1, - "protocolId": 500, - } - ] - - -def test_ListCoordinatesForMicrographRaisesWhenMicrographIsMissing( - service, - currentProject, - projectService, - mapper, - currentUser, -): - protocol = FakeProtocol(objId=10) - protocol.outputCoordinates = buildCoordinatesOutput() - - currentProject.protocols[10] = protocol - projectService.runtimeProtocolIdByDbId[500] = 10 - - with pytest.raises(HTTPException) as exc: - service.listCoordinatesForMicrograph( - mapper=mapper, - projectId=1, - currentUser=currentUser, - protocolId=500, - outputName="outputCoordinates", - micId="999", - ) - - assert exc.value.status_code == 404 - assert exc.value.detail == "Micrograph '999' not found in coordinates output" - - assert projectService.runtimeCalls == [ - { - "mapper": mapper, - "projectId": 1, - "protocolId": 500, - } - ] \ No newline at end of file From e5193ac3d1873c5c8286ddb8336147d4322e378c Mon Sep 17 00:00:00 2001 From: "Yunior C. Fonseca Reyna" Date: Tue, 30 Jun 2026 10:35:51 +0200 Subject: [PATCH 278/278] refactor(logs): resolve protocol logs from postgresql paths first --- app/backend/api/routers/project_router.py | 6 +- app/backend/api/services/project_service.py | 411 ++++++++++++++++++ tests/conftest.py | 27 +- tests/integration/api/test_projects_router.py | 100 +++++ .../services/test_project_service_logs_fs.py | 162 +++++++ 5 files changed, 702 insertions(+), 4 deletions(-) diff --git a/app/backend/api/routers/project_router.py b/app/backend/api/routers/project_router.py index 0413c417..ecb79ecf 100644 --- a/app/backend/api/routers/project_router.py +++ b/app/backend/api/routers/project_router.py @@ -1115,7 +1115,7 @@ def listProtocolLogChannels( service: ProjectService = Depends(getProjectService), ): # listProtocolLogChannels - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1124,6 +1124,7 @@ def listProtocolLogChannels( projectId=projectId, protocolId=protocolId, mapper=mapper, + currentUser=currentUser, ) # normalizeChannels @@ -1166,7 +1167,7 @@ def pollProtocolLogs( service: ProjectService = Depends(getProjectService), ): # pollProtocolLogs - project = service.getProjectById(mapper, projectId, currentUser, refresh=False, checkPid=False) + project = service.getProjectDbRow(mapper, projectId, currentUser) if not project: raise HTTPException(status_code=404, detail="Project not found") @@ -1188,6 +1189,7 @@ def pollProtocolLogs( maxBytes=maxBytes, maxLines=maxLines, mapper=mapper, + currentUser=currentUser, ) # normalizePollResponse diff --git a/app/backend/api/services/project_service.py b/app/backend/api/services/project_service.py index 7e165967..a2eeed69 100644 --- a/app/backend/api/services/project_service.py +++ b/app/backend/api/services/project_service.py @@ -6063,15 +6063,404 @@ def getProtocolName(self, node): return 'default' + def _resolvePostgresqlProjectPathForFilesystem( + self, + mapper, + projectId: int, + ) -> Optional[str]: + if mapper is None or not hasattr(mapper, "db"): + return None + + try: + row = mapper.db.fetchOne( + """ + SELECT name + FROM projects + WHERE id = %s + """, + (projectId,), + ) + except Exception: + logger.debug( + "Could not resolve project path from PostgreSQL. projectId=%s", + projectId, + exc_info=True, + ) + return None + + if not row: + return None + + rawPath = row.get("name") if isinstance(row, dict) else row[0] + if not rawPath: + return None + + try: + path = self._normalizeProjectPath(str(rawPath)) + except Exception: + path = str(rawPath) + + if not os.path.isabs(path): + try: + path = self.manager.getProjectPath(path) + except Exception: + pass + + return os.path.realpath(path) + + def _resolvePostgresqlProtocolRunPath( + self, + mapper, + projectId: int, + scipionProtocolId: Union[int, str], + ) -> Optional[str]: + projectPath = self._resolvePostgresqlProjectPathForFilesystem( + mapper=mapper, + projectId=projectId, + ) + if not projectPath: + return None + + runsPath = os.path.join(projectPath, "Runs") + if not os.path.isdir(runsPath): + return None + + runtimeIdText = str(scipionProtocolId).strip() + runtimeIdInt = None + try: + runtimeIdInt = int(runtimeIdText) + except Exception: + pass + + matches: List[str] = [] + + try: + for entry in os.scandir(runsPath): + if not entry.is_dir(): + continue + + name = entry.name + + if name.startswith("%s_" % runtimeIdText): + matches.append(entry.path) + continue + + if runtimeIdInt is not None: + match = re.match(r"^0*(\d+)_", name) + if match and int(match.group(1)) == runtimeIdInt: + matches.append(entry.path) + continue + except Exception: + logger.debug( + "Could not scan Runs folder for protocol logs. runsPath=%s", + runsPath, + exc_info=True, + ) + return None + + if not matches: + return None + + return sorted(matches)[0] + + @staticmethod + def _firstExistingLogPath( + protocolPath: str, + candidates: List[str], + ) -> Optional[str]: + for candidate in candidates: + path = os.path.join(protocolPath, candidate) + if os.path.exists(path): + return path + + return None + + def _resolvePostgresqlProtocolLogPaths( + self, + mapper, + projectId: int, + protocolId: int, + ) -> Optional[Dict[str, Any]]: + if mapper is None: + return None + + try: + scipionProtocolId = self._resolveScipionProtocolId( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + except Exception: + logger.debug( + "Could not resolve PostgreSQL protocol id for logs. projectId=%s protocolId=%s", + projectId, + protocolId, + exc_info=True, + ) + return None + + protocolPath = self._resolvePostgresqlProtocolRunPath( + mapper=mapper, + projectId=projectId, + scipionProtocolId=scipionProtocolId, + ) + if not protocolPath: + return None + + logCandidates = { + "stdout": [ + "logs/run.stdout", + "logs/stdout.log", + "logs/stdout.txt", + "run.stdout", + "stdout.log", + "stdout.txt", + ], + "stderr": [ + "logs/run.stderr", + "logs/stderr.log", + "logs/stderr.txt", + "run.stderr", + "stderr.log", + "stderr.txt", + ], + "schedule": [ + "logs/schedule.log", + "logs/schedule.txt", + "logs/run.schedule", + "schedule.log", + "schedule.txt", + "run.schedule", + ], + } + + paths = { + channelId: self._firstExistingLogPath(protocolPath, candidates) + for channelId, candidates in logCandidates.items() + } + + # If we cannot find any known log file, do not guess. Let runtime fallback + # resolve the exact Scipion log paths. + if not any(paths.values()): + return None + + return { + "protocolId": scipionProtocolId, + "protocolPath": protocolPath, + "paths": paths, + } + + def _ensureRuntimeProjectForLogs( + self, + mapper, + projectId: int, + currentUser: Optional[dict], + ) -> None: + if getattr(self, "currentProject", None) is not None: + return + + if currentUser is None: + return + + self.getProjectById( + mapper, + projectId, + currentUser, + refresh=False, + checkPid=False, + ) + + @staticmethod + def _buildProtocolLogChannel( + channelId: str, + label: str, + order: int, + filePath: Optional[str], + ) -> Dict[str, Any]: + return { + "id": channelId, + "label": label, + "order": order, + } + + def _buildProtocolLogChannelsPayload( + self, + projectId: int, + protocolId: Union[int, str], + logPaths: Dict[str, Optional[str]], + ) -> Dict[str, Any]: + return { + "projectId": projectId, + "protocolId": int(protocolId), + "channels": [ + self._buildProtocolLogChannel("stdout", "Output", 1, logPaths.get("stdout")), + self._buildProtocolLogChannel("stderr", "Errors", 2, logPaths.get("stderr")), + self._buildProtocolLogChannel("schedule", "Schedule", 3, logPaths.get("schedule")), + ], + } + + @staticmethod + def _normalizeProtocolLogOffsets(rawOffsets: Dict[str, int]) -> Dict[str, int]: + if not isinstance(rawOffsets, dict): + return {"stdout": 0, "stderr": 0, "schedule": 0} + + keyMap = { + "stdout": "stdout", + "stdoutLog": "stdout", + "out": "stdout", + "stderr": "stderr", + "stderrLog": "stderr", + "err": "stderr", + "schedule": "schedule", + "scheduleLog": "schedule", + } + + normalized = {"stdout": 0, "stderr": 0, "schedule": 0} + for key, value in rawOffsets.items(): + canonical = keyMap.get(str(key), None) + if canonical is None: + continue + + try: + normalized[canonical] = max(0, int(value)) + except Exception: + normalized[canonical] = 0 + + return normalized + + @staticmethod + def _readProtocolLogChunk( + filePath: Optional[str], + startOffset: int, + maxBytes: Optional[int] = 65536, + maxLines: Optional[int] = 2000, + ) -> Dict[str, Any]: + if not filePath or not os.path.exists(filePath): + return { + "content": "", + "offset": int(startOffset or 0), + } + + try: + sizeBytes = int(os.path.getsize(filePath)) + except Exception: + sizeBytes = 0 + + safeOffset = int(startOffset or 0) + if safeOffset < 0: + safeOffset = 0 + + if safeOffset > sizeBytes: + safeOffset = 0 + + bytesCap = None if maxBytes is None else max(1, int(maxBytes)) + linesCap = None if maxLines is None else max(1, int(maxLines)) + + contentParts: List[str] = [] + bytesRead = 0 + linesRead = 0 + + try: + with open(filePath, "rb") as handle: + handle.seek(safeOffset) + + while True: + if linesCap is not None and linesRead >= linesCap: + break + if bytesCap is not None and bytesRead >= bytesCap: + break + + posBefore = handle.tell() + lineBytes = handle.readline() + if not lineBytes: + break + + if bytesCap is not None and (bytesRead + len(lineBytes)) > bytesCap: + handle.seek(posBefore) + break + + contentParts.append(lineBytes.decode("utf-8", errors="ignore")) + bytesRead += len(lineBytes) + linesRead += 1 + + newOffset = handle.tell() + + except Exception as e: + return { + "content": "", + "offset": safeOffset, + "error": str(e), + } + + return { + "content": "".join(contentParts), + "offset": int(newOffset), + } + + def _pollProtocolLogPaths( + self, + projectId: int, + protocolId: Union[int, str], + logPaths: Dict[str, Optional[str]], + offsets: Dict[str, int], + maxBytes: Optional[int] = 65536, + maxLines: Optional[int] = 2000, + ) -> Dict[str, Any]: + normalizedOffsets = self._normalizeProtocolLogOffsets(offsets or {}) + + return { + "projectId": projectId, + "protocolId": int(protocolId), + "channels": { + "stdout": self._readProtocolLogChunk( + logPaths.get("stdout"), + normalizedOffsets.get("stdout", 0), + maxBytes=maxBytes, + maxLines=maxLines, + ), + "stderr": self._readProtocolLogChunk( + logPaths.get("stderr"), + normalizedOffsets.get("stderr", 0), + maxBytes=maxBytes, + maxLines=maxLines, + ), + "schedule": self._readProtocolLogChunk( + logPaths.get("schedule"), + normalizedOffsets.get("schedule", 0), + maxBytes=maxBytes, + maxLines=maxLines, + ), + }, + } + def listProtocolLogChannelsService( self, projectId: int, protocolId: int, mapper=None, + currentUser: Optional[dict] = None, ): """ Return available log channels for a protocol, including paths and basic file stats. """ + pgLogs = self._resolvePostgresqlProtocolLogPaths( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + if pgLogs is not None: + return self._buildProtocolLogChannelsPayload( + projectId=projectId, + protocolId=pgLogs["protocolId"], + logPaths=pgLogs["paths"], + ) + + self._ensureRuntimeProjectForLogs( + mapper=mapper, + projectId=projectId, + currentUser=currentUser, + ) + scipionProtocolId = self._resolveScipionProtocolId( mapper=mapper, projectId=projectId, @@ -6133,10 +6522,32 @@ def pollProtocolLogsService( maxBytes: Optional[int] = 65536, maxLines: Optional[int] = 2000, mapper=None, + currentUser: Optional[dict] = None, ): """ Incrementally read protocol logs from the given offsets, applying maxBytes and maxLines limits. """ + pgLogs = self._resolvePostgresqlProtocolLogPaths( + mapper=mapper, + projectId=projectId, + protocolId=protocolId, + ) + if pgLogs is not None: + return self._pollProtocolLogPaths( + projectId=projectId, + protocolId=pgLogs["protocolId"], + logPaths=pgLogs["paths"], + offsets=offsets, + maxBytes=maxBytes, + maxLines=maxLines, + ) + + self._ensureRuntimeProjectForLogs( + mapper=mapper, + projectId=projectId, + currentUser=currentUser, + ) + scipionProtocolId = self._resolveScipionProtocolId( mapper=mapper, projectId=projectId, diff --git a/tests/conftest.py b/tests/conftest.py index 8225fa6c..1e5690de 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -140,6 +140,7 @@ def __init__(self): } } self.lastPollLogsCall = None + self.lastListProtocolLogChannelsCall = None self.resolveViewerResult = {"handled": False} self.resolveViewerError = None @@ -586,10 +587,31 @@ def validateProjectPostgresqlConsistency( def listProjectLogChannelsService(self, projectId, protocolId): return self.logChannelsResult - def listProtocolLogChannelsService(self, projectId, protocolId, mapper=None): + def listProtocolLogChannelsService( + self, + projectId, + protocolId, + mapper=None, + currentUser=None, + ): + self.lastListProtocolLogChannelsCall = { + "projectId": projectId, + "protocolId": protocolId, + "mapper": mapper, + "currentUser": currentUser, + } return self.logChannelsResult - def pollProtocolLogsService(self, projectId, protocolId, offsets, maxBytes, maxLines, mapper=None): + def pollProtocolLogsService( + self, + projectId, + protocolId, + offsets, + maxBytes, + maxLines, + mapper=None, + currentUser=None, + ): self.lastPollLogsCall = { "projectId": projectId, "protocolId": protocolId, @@ -597,6 +619,7 @@ def pollProtocolLogsService(self, projectId, protocolId, offsets, maxBytes, maxL "maxBytes": maxBytes, "maxLines": maxLines, "mapper": mapper, + "currentUser": currentUser, } return self.pollLogsResult diff --git a/tests/integration/api/test_projects_router.py b/tests/integration/api/test_projects_router.py index b0dc8cbc..c74a6653 100644 --- a/tests/integration/api/test_projects_router.py +++ b/tests/integration/api/test_projects_router.py @@ -239,6 +239,8 @@ def test_PollProtocolLogsNormalizesOffsetsAndIncludesDynamicChannels(projectClie ] } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None assert fakeProjectService.lastPollLogsCall == { "projectId": 1, "protocolId": 22, @@ -250,6 +252,11 @@ def test_PollProtocolLogsNormalizesOffsetsAndIncludesDynamicChannels(projectClie "maxBytes": 123, "maxLines": 45, "mapper": fakeProjectService.lastPollLogsCall["mapper"], + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, } @@ -1314,3 +1321,96 @@ def test_RenderCtftomoPsdImagePreservesHttpException( assert response.json()["detail"] == "PSD image file not found" assert fakeProjectService.lastGetProjectDbRowCall is not None assert fakeProjectService.lastGetProjectByIdCall is None + + +def test_ListProtocolLogChannelsUsesProjectDbRow( + projectClient, + fakeProjectService, +): + response = projectClient.get("/projects/1/protocols/2/logs/channels") + + assert response.status_code == 200 + assert response.json() == { + "channels": fakeProjectService.logChannelsResult, + } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListProtocolLogChannelsCall == { + "projectId": 1, + "protocolId": 2, + "mapper": fakeProjectService.lastListProtocolLogChannelsCall["mapper"], + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + } + + +def test_PollProtocolLogsUsesProjectDbRow( + projectClient, + fakeProjectService, +): + response = projectClient.post( + "/projects/1/protocols/2/logs/chunk", + json={ + "offsets": { + "stdout": 0, + }, + "maxBytes": 32, + "maxLines": 10, + }, + ) + + assert response.status_code == 200 + assert response.json() == { + "chunks": [ + { + "channel": "stdout", + "content": "hello", + "offset": 5, + } + ] + } + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastPollLogsCall == { + "projectId": 1, + "protocolId": 2, + "offsets": {"stdout": 0}, + "maxBytes": 32, + "maxLines": 10, + "mapper": fakeProjectService.lastPollLogsCall["mapper"], + "currentUser": { + "id": 1, + "email": "user@example.com", + "role": "user", + }, + } + + +@pytest.mark.parametrize( + ("method", "url", "payload"), + [ + ("get", "/projects/1/protocols/2/logs/channels", None), + ("post", "/projects/1/protocols/2/logs/chunk", {"offsets": {"stdout": 0}}), + ], +) +def test_ProtocolLogEndpointsReturn404WhenProjectMissing( + projectClient, + fakeProjectService, + method, + url, + payload, +): + fakeProjectService.projectDbRowResult = None + + request = getattr(projectClient, method) + response = request(url) if payload is None else request(url, json=payload) + + assert response.status_code == 404 + assert response.json()["detail"] == "Project not found" + assert fakeProjectService.lastGetProjectDbRowCall is not None + assert fakeProjectService.lastGetProjectByIdCall is None + assert fakeProjectService.lastListProtocolLogChannelsCall is None + assert fakeProjectService.lastPollLogsCall is None \ No newline at end of file diff --git a/tests/unit/backend/api/services/test_project_service_logs_fs.py b/tests/unit/backend/api/services/test_project_service_logs_fs.py index b8916b43..ebbf1602 100644 --- a/tests/unit/backend/api/services/test_project_service_logs_fs.py +++ b/tests/unit/backend/api/services/test_project_service_logs_fs.py @@ -144,6 +144,35 @@ def previewProtocolImageFile(self, protocolId, path, inline): "inline": inline, } +class FakePgDb: + def __init__(self, projectPath, runtimeProtocolIdByDbId=None): + self.projectPath = projectPath + self.runtimeProtocolIdByDbId = runtimeProtocolIdByDbId or {} + self.fetchCalls = [] + + def fetchOne(self, query, params): + self.fetchCalls.append({"query": query, "params": params}) + + if "FROM projects" in query: + return {"name": str(self.projectPath)} + + if "FROM protocols" in query: + protocolDbId = params[1] + runtimeProtocolId = self.runtimeProtocolIdByDbId.get(int(protocolDbId)) + if runtimeProtocolId is None: + return None + return {"protocolId": runtimeProtocolId} + + return None + + +class FakePgMapper: + def __init__(self, projectPath, runtimeProtocolIdByDbId=None): + self.db = FakePgDb( + projectPath=projectPath, + runtimeProtocolIdByDbId=runtimeProtocolIdByDbId, + ) + @pytest.fixture def projectServiceModule(authTestEnv): @@ -766,4 +795,137 @@ def test_GetProtocolLogsNormalizesNegativeOffsets(service, tmp_path): "stderrOffset": 4, "scheduleLog": "SCH\n", "scheduleOffset": 4, + } + + +def test_ListProtocolLogChannelsServiceUsesPostgresqlPathsBeforeRuntime( + service, + tmp_path, +): + projectPath = tmp_path / "DemoProject" + protocolPath = projectPath / "Runs" / "000010_ProtImport" + logsPath = protocolPath / "logs" + logsPath.mkdir(parents=True, exist_ok=True) + + (logsPath / "run.stdout").write_text("hello\n", encoding="utf-8") + (logsPath / "run.stderr").write_text("error\n", encoding="utf-8") + (logsPath / "schedule.log").write_text("schedule\n", encoding="utf-8") + + mapper = FakePgMapper( + projectPath=projectPath, + runtimeProtocolIdByDbId={500: 10}, + ) + + def failRuntime(*args, **kwargs): + raise AssertionError("runtime should not be used") + + service._getScipionProtocolByRuntimeId = failRuntime + + result = service.listProtocolLogChannelsService( + projectId=1, + protocolId=500, + mapper=mapper, + ) + + assert result == { + "projectId": 1, + "protocolId": 10, + "channels": [ + {"id": "stdout", "label": "Output", "order": 1}, + {"id": "stderr", "label": "Errors", "order": 2}, + {"id": "schedule", "label": "Schedule", "order": 3}, + ], + } + + +def test_PollProtocolLogsServiceUsesPostgresqlPathsBeforeRuntime( + service, + tmp_path, +): + projectPath = tmp_path / "DemoProject" + protocolPath = projectPath / "Runs" / "000010_ProtImport" + logsPath = protocolPath / "logs" + logsPath.mkdir(parents=True, exist_ok=True) + + (logsPath / "run.stdout").write_text("line1\nline2\nline3\n", encoding="utf-8") + (logsPath / "run.stderr").write_text("err1\nerr2\n", encoding="utf-8") + (logsPath / "schedule.log").write_text("sched1\nsched2\n", encoding="utf-8") + + mapper = FakePgMapper( + projectPath=projectPath, + runtimeProtocolIdByDbId={500: 10}, + ) + + def failRuntime(*args, **kwargs): + raise AssertionError("runtime should not be used") + + service._getScipionProtocolByRuntimeId = failRuntime + + result = service.pollProtocolLogsService( + projectId=1, + protocolId=500, + offsets={ + "stdoutLog": 6, + "err": 0, + "schedule": 7, + }, + maxBytes=64, + maxLines=1, + mapper=mapper, + ) + + assert result == { + "projectId": 1, + "protocolId": 10, + "channels": { + "stdout": { + "content": "line2\n", + "offset": 12, + }, + "stderr": { + "content": "err1\n", + "offset": 5, + }, + "schedule": { + "content": "sched2\n", + "offset": 14, + }, + }, + } + + +def test_ProtocolLogsFallbackToRuntimeWhenPostgresqlLogsAreMissing( + service, + tmp_path, +): + projectPath = tmp_path / "DemoProject" + protocolPath = projectPath / "Runs" / "000010_ProtImport" + protocolPath.mkdir(parents=True, exist_ok=True) + + runtimeStdout = tmp_path / "runtime.stdout" + runtimeStdout.write_text("runtime\n", encoding="utf-8") + + service.currentProject.protocols[10] = FakeProtocol( + stdoutLog=str(runtimeStdout), + stderrLog=str(tmp_path / "missing.stderr"), + scheduleLog=str(tmp_path / "missing.schedule"), + ) + + mapper = FakePgMapper( + projectPath=projectPath, + runtimeProtocolIdByDbId={500: 10}, + ) + + result = service.pollProtocolLogsService( + projectId=1, + protocolId=500, + offsets={"stdout": 0}, + maxBytes=64, + maxLines=10, + mapper=mapper, + ) + + assert result["channels"]["stdout"] == { + "content": "runtime\n", + "offset": len("runtime\n"), } \ No newline at end of file