Update main to v0.6.5 of python-parakeet

.github/workflows/publish-to-pypi.yml

@@ -12,7 +12,7 @@ jobs:
     - name: setup-python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10.4"
+        python-version: "3.10.20"
     - name: install-pypa/build
       run: >-
         python3 -m
@@ -22,7 +22,7 @@ jobs:
     - name: build-bin-whl
       run: python3 -m build
     - name: grab-build
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: pkg-dist
         path: dist/
@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: download-dists
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: pkg-dist
         path: dist/
@@ -62,7 +62,7 @@ jobs:
 
     steps:
     - name: download-dists
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: pkg-dist
         path: dist/
@@ -104,7 +104,7 @@ jobs:
 
     steps:
     - name: download-dists
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: pkg-dist
         path: dist/

.pre-commit-config.yaml

@@ -3,13 +3,13 @@
 # from manual and automatic code checks.
 repos:
   - repo: https://github.com/psf/black
-    rev: 23.1.0  
+    rev: 26.3.1 
     hooks:
       - id: black
         args:
         - --line-length=79
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v6.0.0
     hooks:
       # - id: check-added-large-files
         # args: ['maxkb=500']
@@ -20,13 +20,13 @@ repos:
       - id: check-toml
       - id: check-yaml
   - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 7.3.0
     hooks:
       - id: flake8
         args:
         - "--ignore=E203,W503"
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.1.1
+    rev: v1.20.0
     hooks:
       - id: mypy
         additional_dependencies: [pandas-stubs, types-Pillow, types-setuptools, types-PyYAML]

README.md

@@ -111,3 +111,50 @@ cd ccpem-pipeliner
 pip install -e .
 ```
 After doing so, an editable install of roodmus should be set up with all requisite packages.
+
+# Modern (May 2025) developer install on V100
+
+```
+conda create -n roodmus python=3.10.4
+conda activate roodmus
+
+git clone https://gitlab.com/ccpem/ccpem-pipeliner.git
+cd ccpem-pipeliner
+git checkout bedbedbe183ad497dbaa82a638f210d316ba9bae
+cd ../
+
+git clone git@github.com:ccpem/roodmus.git
+cd roodmus
+export CXX=/usr/bin/g++
+export CUDACXX=/usr/local/cuda-12.3/bin/nvcc
+export CMAKE_CUDA_ARCHITECTURES=70
+#USE CMAKE COMPATIBLE WITH CMAKE<3.5
+pip install --upgrade pip
+pip install -e . --no-cache-dir
+pre-commit install
+cd ../
+
+cd ccpem-pipeliner
+pip install -e .
+```
+
+# Modern (July 2026) developer install on V100
+
+```
+conda create -n roodmus python=3.10.20
+conda activate roodmus
+
+git clone git@github.com:ccpem/roodmus.git
+cd roodmus
+export CXX=/usr/bin/g++
+export CUDACXX=/usr/bin/nvcc
+export CMAKE_CUDA_ARCHITECTURES=70
+#USE CMAKE COMPATIBLE WITH CMAKE<3.5????
+pip install --upgrade pip
+pip install -e .[dev,gpu,docs] --no-cache-dir
+pre-commit install
+cd ../
+
+cd ccpem-pipeliner
+pip install -e .
+```

notebooks/make_starfile_with_GT.ipynb

@@ -0,0 +1,161 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Creating STAR file with Ground Truth Particle Positions And Defoci\n",
+    "\n",
+    "Uses roodmus' simulation/write_starfile utilities:\n",
+    "\n",
+    "* read ground truth information from YAML files\n",
+    "* save the dataframe of truth information to file as csv\n",
+    "* use CLI roodmus write_starfile with csv file as input to create a particles.star file\n",
+    "* user particles.star file to extract (or re-extract?) particles to get extracted particle stack\n",
+    "* utilise cryodrgn to train on particle stack in doppio\n",
+    "* interpret latent space representation in another notebook"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "# read ground truth from yaml files\n",
+    "# data loading for DE-Shaw covid spike partially open set\n",
+    "project_dir = \"/mnt/dinaMISMO/synthetic_datasets/MapReconstruction/PreP/extract_frames_6xou_unsteered_0-200ns\"\n",
+    "mrc_dir = os.path.join(project_dir, \"Micrographs\")\n",
+    "figures_dir = os.path.join(project_dir, \"figures\")\n",
+    "\n",
+    "verbose=True\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "loading micrographs: 100%|██████████| 67/67 [00:14<00:00,  4.59it/s, micrograph=000058.mrc]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loaded ground-truth particle positions from config files\n",
+      "Dictionaries now contain 0 particles and 20100 true particles\n",
+      "Added 20100 particles from /mnt/dinaMISMO/synthetic_datasets/MapReconstruction/PreP/extract_frames_6xou_unsteered_0-200ns/Micrographs\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# save ground truth dataframe to csv format\n",
+    "\n",
+    "# load into dataframe\n",
+    "from roodmus.analysis.utils import load_data\n",
+    "\n",
+    "loaded_data = load_data(\n",
+    "    meta_file=None, # no reconstruction file(s)\n",
+    "    config_dir=mrc_dir, # path to roodmus run_parakeet output directory\n",
+    "    particle_diameter=100, # approx\n",
+    "    ugraph_shape=(4000, 4000), \n",
+    "    ignore_missing_files=False,\n",
+    "    verbose=True,\n",
+    "    enable_tqdm=True,\n",
+    ")\n",
+    "\n",
+    "# taking most of this code from src/roodmus/analysis/extract_particles.py?????\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'\\npd.read_csv(\\n    truth_csv_name,\\n    index_col=0,\\n)\\n'"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# make dict into df and save\n",
+    "import pandas as pd\n",
+    "\n",
+    "# make df and save to file\n",
+    "truth_csv_fname = os.path.join(project_dir, \"truth.csv\") \n",
+    "truth_df = pd.DataFrame(\n",
+    "    loaded_data.results_truth,\n",
+    "    columns=loaded_data.results_truth.keys(),\n",
+    ")\n",
+    "truth_df.to_csv(truth_csv_fname)\n",
+    "\n",
+    "# can load with:\n",
+    "\"\"\"\n",
+    "pd.read_csv(\n",
+    "    truth_csv_name,\n",
+    "    index_col=0,\n",
+    ")\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# run starfile creation script\n",
+    "!bash make_starfile_with_GT.sh\n",
+    "\n",
+    "\n",
+    "\n",
+    "# After this move to doppio to extract particles and run cryodrgn\n",
+    "# Then use notebook to trace conformations through latent space embeddings\n",
+    "# And then create output maps and models to compute metrics on"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "roodmus",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/make_starfile_with_GT.sh

@@ -0,0 +1,33 @@
+# take imported micrgraphs and make a starfile with the ground truth coordinates as well as the other optics
+# group information necessary to allow particles to be extracted into particle stacks straight from the
+# micrographs
+
+# in this case we need no motion correction!
+# and can/should use ground truth per-particle defoci?
+# WILL USE PER-UGRAPH DEFOCUS, simply because already coded in. MAYBE REVISE THIS?
+
+INPUT_CSV="truth.csv"
+TYPE="data_star" #? check
+OUTPUT_DIR="GT_starfile"
+UGRAPH_DIR="Micrographs" # the dir of imported micrographs
+PIXEL_SIZE=1.0
+
+# optics group arguments
+OPTICS_GROUP_NAME="opticsGroup1"
+OPTICS_GROUP="1"
+MTF_FILENAME="Micrographs/relion/mtf_300kV.star"
+MICROGRAPH_ORIGINAL_PIXEL_SIZE="1.0"
+VOLTAGE="300"
+SPHERICAL_ABERRATION="2.7"
+AMPLITUDE_CONTRAST="0.1"
+IMAGE_PIXEL_SIZE=1.0
+IMAGE_SIZE=256
+IMAGE_DIMENSIONALITY=2
+CTF_DATA_ARE_CTF_PREMULTIPLIED=0
+
+# EXTRACT_DIR="" not used in this case
+
+roodmus write_starfile --tqdm --verbose --input_csv $INPUT_CSV --type $TYPE --output_dir $OUTPUT_DIR --ugraph_dir $UGRAPH_DIR --pixel_size $PIXEL_SIZE --optics_group_name $OPTICS_GROUP_NAME --optics_group $OPTICS_GROUP --mtf_filename $MTF_FILENAME --micrograph_original_pixel_size $MICROGRAPH_ORIGINAL_PIXEL_SIZE --voltage $VOLTAGE --spherical_aberration $SPHERICAL_ABERRATION --amplitude_contrast $AMPLITUDE_CONTRAST --image_pixel_size $IMAGE_PIXEL_SIZE --image_size $IMAGE_SIZE --image_dimensionality $IMAGE_DIMENSIONALITY --ctf_data_are_ctf_premultiplied $CTF_DATA_ARE_CTF_PREMULTIPLIED 
+
+# if you want per-particle defoci instead of per ugraph defoci:
+# roodmus write_starfile --pp_defoci --tqdm --verbose --input_csv $INPUT_CSV --type $TYPE --output_dir $OUTPUT_DIR --ugraph_dir $UGRAPH_DIR --pixel_size $PIXEL_SIZE --optics_group_name $OPTICS_GROUP_NAME --optics_group $OPTICS_GROUP --mtf_filename $MTF_FILENAME --micrograph_original_pixel_size $MICROGRAPH_ORIGINAL_PIXEL_SIZE --voltage $VOLTAGE --spherical_aberration $SPHERICAL_ABERRATION --amplitude_contrast $AMPLITUDE_CONTRAST --image_pixel_size $IMAGE_PIXEL_SIZE --image_size $IMAGE_SIZE --image_dimensionality $IMAGE_DIMENSIONALITY --ctf_data_are_ctf_premultiplied $CTF_DATA_ARE_CTF_PREMULTIPLIED