feat: display Jupyter notebook in cluster detail page

backend/pyproject.toml

@@ -59,6 +59,7 @@ target-version = "py312"
 extend-exclude = [
     "*/migrations/*.py",
     "staticfiles/*",
+    "target/metadata/notebooks/*.ipynb",
 ]
 
 [tool.ruff.lint]

backend/requirements/base.txt

@@ -15,6 +15,10 @@ pandas==2.2.3 # https://github.com/pandas-dev/pandas
 astropy>=6.1 # https://github.com/astropy/astropy
 psycopg2==2.9.10 # https://github.com/psycopg/psycopg2
 psycopg==3.2.3 # https://github.com/psycopg/psycopg
+nbconvert==7.17.1
+nbformat==5.10.4
+ipykernel==7.3.0
+matplotlib==3.11.0
 # Django
 # ------------------------------------------------------------------------------
 django==5.0.10  # pyup: < 5.1  # https://www.djangoproject.com/

backend/target/metadata/api/serializers.py

@@ -113,6 +113,7 @@ class Meta:
 class NestedTableSerializer(serializers.ModelSerializer[Table]):
     qs_columns = Table.objects.prefetch_related("columns")
     columns = ResumedColumnSerializer(qs_columns, many=True, read_only=True)
+    ucds = serializers.SerializerMethodField()
 
     qs_settings = Table.objects.prefetch_related("settings")
     settings = SettingsSerializer(qs_settings, many=False, read_only=True)
@@ -153,6 +154,7 @@ class Meta:
             "property_ra",
             "property_dec",
             "columns",
+            "ucds",
             "settings",
         ]
 
@@ -198,3 +200,7 @@ def get_related_property_id(self, obj):
             if col:
                 return col.name
         return None
+
+    def get_ucds(self, obj):
+        columns = obj.columns.filter(ucd__isnull=False)
+        return {c.ucd: c.name for c in columns if c.ucd and c.name}

backend/target/metadata/api/views.py

@@ -1,6 +1,15 @@
+import base64
+import io
+import json
 import math
+import traceback as tb
+from contextlib import redirect_stderr
+from contextlib import redirect_stdout
+from pathlib import Path
 
+import nbformat
 from django.conf import settings
+from nbconvert import HTMLExporter
 from rest_framework import status
 from rest_framework.decorators import action
 from rest_framework.response import Response
@@ -19,6 +28,78 @@
 from .serializers import TableSerializer
 
 
+def _execute_notebook_inprocess(nb):
+    """Execute notebook code cells in-process, capturing outputs."""
+    namespace = {}
+
+    # Ensure matplotlib uses a non-interactive backend before any import
+    exec("import matplotlib; matplotlib.use('Agg')", namespace)  # noqa: S102
+
+    for execution_count, cell in enumerate(nb.cells, start=1):
+        if cell.cell_type != "code":
+            continue
+
+        cell.outputs = []
+        cell.execution_count = execution_count
+
+        stdout_buf = io.StringIO()
+        stderr_buf = io.StringIO()
+
+        try:
+            with redirect_stdout(stdout_buf), redirect_stderr(stderr_buf):
+                exec(cell.source, namespace)  # noqa: S102
+
+            # Capture any matplotlib figures produced by this cell
+            try:
+                import matplotlib.pyplot as plt
+
+                for fig_num in plt.get_fignums():
+                    fig = plt.figure(fig_num)
+                    buf = io.BytesIO()
+                    fig.savefig(buf, format="png", bbox_inches="tight")
+                    img_b64 = base64.b64encode(buf.getvalue()).decode()
+                    cell.outputs.append(
+                        nbformat.v4.new_output(
+                            output_type="display_data",
+                            data={"image/png": img_b64, "text/plain": "<Figure>"},
+                            metadata={},
+                        ),
+                    )
+                plt.close("all")
+            except ImportError:
+                pass
+
+        except Exception as exc:  # noqa: BLE001
+            cell.outputs.append(
+                nbformat.v4.new_output(
+                    output_type="error",
+                    ename=type(exc).__name__,
+                    evalue=str(exc),
+                    traceback=tb.format_exception(type(exc), exc, exc.__traceback__),
+                ),
+            )
+
+        stdout_val = stdout_buf.getvalue()
+        stderr_val = stderr_buf.getvalue()
+
+        if stdout_val:
+            cell.outputs.append(
+                nbformat.v4.new_output(
+                    output_type="stream",
+                    name="stdout",
+                    text=stdout_val,
+                ),
+            )
+        if stderr_val:
+            cell.outputs.append(
+                nbformat.v4.new_output(
+                    output_type="stream",
+                    name="stderr",
+                    text=stderr_val,
+                ),
+            )
+
+
 class TableRegistrationError(Exception):
     """Raised when table registration fails"""
 
@@ -463,6 +544,52 @@ def sanitize_rows(self, data, bigint_columns=None):
             return result
         return data
 
+    def query_data(  # noqa: PLR0913
+        self,
+        table,
+        limit,
+        offset,
+        url_filters,
+        ordering,
+        ucds,
+    ):
+        db = MyDB(username=self.request.user.username)
+        rows, count = db.query(
+            tablename=table.name,
+            limit=limit,
+            offset=offset,
+            url_filters=url_filters,
+            ordering=ordering,
+        )
+
+        # match the column names with the table columns ucd
+        for row in rows:
+            row.update(
+                {
+                    "meta_catalog_id": table.id,
+                    "meta_id": None
+                    if (v := row.get(ucds.get("meta.id;meta.main"))) is None
+                    else str(v),
+                    "meta_ra": None
+                    if (v := row.get(ucds.get("pos.eq.ra;meta.main"))) is None
+                    else str(v),
+                    "meta_dec": None
+                    if (v := row.get(ucds.get("pos.eq.dec;meta.main"))) is None
+                    else str(v),
+                    "meta_radius_arcmin": row.get(ucds.get("phys.angSize;src")),
+                },
+            )
+
+        # Convert bigints to string to avoid JS issues
+        bigint_columns = []
+        for col in table.columns.all().order_by("name"):
+            datatype = col.datatype.lower().split("(")[0]
+            if datatype in ["bigint", "int8"]:
+                bigint_columns.append(col.name)
+
+        parsed_rows = self.sanitize_rows(rows, bigint_columns)
+        return parsed_rows, count
+
     @action(detail=True, methods=["get"])
     def data(self, request, pk=None):
         # IMPORTANTE: Não pode ser utilizado o self.get_object()
@@ -494,46 +621,117 @@ def data(self, request, pk=None):
 
         ordering = request.query_params.get("ordering", None)
 
-        db = MyDB(username=request.user.username)
-        rows, count = db.query(
-            tablename=table.name,
+        parsed_rows, count = self.query_data(
+            table=table,
             limit=limit,
             offset=offset,
             url_filters=url_filters,
             ordering=ordering,
+            ucds=ucds,
         )
 
-        # match the column names with the table columns ucd
-        for row in rows:
-            row.update(
-                {
-                    "meta_catalog_id": table.id,
-                    "meta_id": None
-                    if (v := row.get(ucds.get("meta.id;meta.main"))) is None
-                    else str(v),
-                    "meta_ra": None
-                    if (v := row.get(ucds.get("pos.eq.ra;meta.main"))) is None
-                    else str(v),
-                    "meta_dec": None
-                    if (v := row.get(ucds.get("pos.eq.dec;meta.main"))) is None
-                    else str(v),
-                    "meta_radius_arcmin": row.get(ucds.get("phys.angSize;src")),
-                },
-            )
-
-        # Convert bigints to string to avoid JS issues
-        bigint_columns = []
-        for col in table.columns.all().order_by("name"):
-            datatype = col.datatype.lower().split("(")[0]
-            if datatype in ["bigint", "int8"]:
-                bigint_columns.append(col.name)
-
-        parsed_rows = self.sanitize_rows(rows, bigint_columns)
-
         results = {
             "results": parsed_rows,
             "count": count,
             "has_more": (offset + limit) < count,
         }
 
         return Response(results, status=status.HTTP_200_OK)
+
+    @action(detail=True, methods=["get"])
+    def notebook(self, request, pk=None):
+        """Generate a notebook for the given table and record ID."""
+
+        queryset = self.get_queryset()
+
+        main_table = queryset.prefetch_related("columns").get(pk=pk)
+
+        # Main table metadata ( cluster table metadata )
+        main_table_metadata = NestedTableSerializer(
+            main_table,
+            context={"request": request},
+        ).data
+
+        ucds = self.get_table_ucds(main_table)
+        url_filters = self.parse_filters(request.query_params)
+
+        main_record, _ = self.query_data(
+            table=main_table,
+            limit=1,
+            offset=0,
+            url_filters=url_filters,
+            ordering=None,
+            ucds=ucds,
+        )
+        if len(main_record) != 1:
+            return Response(
+                {"error": "Record not found"},
+                status=status.HTTP_404_NOT_FOUND,
+            )
+
+        # Main record data ( cluster data for the given filters )
+        main_record = main_record[0]
+
+        # IF cluster
+        related_table_metadata = None
+        related_table_data = None
+
+        if main_table.catalog_type == Table.CATALOG_TYPE_CLUSTER:
+            if not main_table.related_table:
+                return Response(
+                    {"error": "Related table must be set for cluster catalogs."},
+                    status=status.HTTP_400_BAD_REQUEST,
+                )
+
+            # Related table metadata ( members table metadata )
+            related_table_metadata = NestedTableSerializer(
+                main_table.related_table,
+                context={"request": self.request},
+            ).data
+
+            # Related table data ( members data for the given cluster )
+            cross_id_property = main_table_metadata.get("related_property_id")
+
+            related_filters = {}
+            related_filters[cross_id_property] = main_record[
+                (main_table_metadata.get("property_id"))
+            ]
+
+            # Related table data ( members data for the given cluster )
+            related_table_data, count = self.query_data(
+                table=main_table.related_table,
+                limit=None,
+                offset=0,
+                url_filters=related_filters,
+                ordering=None,
+                ucds=related_table_metadata.get("ucds"),
+            )
+
+        template_path = (
+            Path(__file__).parent.parent / "notebooks" / "cluster_detail_template.ipynb"
+        )
+        with template_path.open() as f:
+            nb = nbformat.read(f, as_version=4)
+
+        def to_python_literal(value):
+            return json.dumps(value)
+
+        replacements = {
+            "main_table_metadata": to_python_literal(main_table_metadata),
+            "main_record": to_python_literal(main_record),
+            "related_table_metadata": to_python_literal(related_table_metadata),
+            "related_table_data": to_python_literal(related_table_data),
+        }
+
+        for cell in nb.cells:
+            if cell.source.strip().startswith("# canvas-variables"):
+                for key, value in replacements.items():
+                    cell.source = cell.source.replace(f"{{{{{key}}}}}", value)
+                break
+
+        _execute_notebook_inprocess(nb)
+
+        exporter = HTMLExporter()
+        html, _ = exporter.from_notebook_node(nb)
+
+        return Response({"html": html}, status=status.HTTP_200_OK)