diff --git a/docs/diagrams/bioloop_architecture.dot b/docs/diagrams/bioloop_architecture.dot new file mode 100644 index 000000000..a99ae1df7 --- /dev/null +++ b/docs/diagrams/bioloop_architecture.dot @@ -0,0 +1,87 @@ +digraph bioloop_architecture { + rankdir=TB; + node [shape=box, style=filled, fontname="Helvetica", fontsize=11]; + edge [fontname="Helvetica", fontsize=9]; + compound=true; + nodesep=0.6; + ranksep=0.8; + splines=ortho; + + // ── Row 1: User ── + user [label="Researcher /\nOperator", shape=ellipse, style=filled, fillcolor="#e0e0e0"]; + + // ── Row 2: Frontend ── + subgraph cluster_frontend { + label="Frontend"; + style=filled; fillcolor="#dce8f8"; + ui [label="UI\n(Vue 3 / Vite)", fillcolor="#b3d4f5"]; + } + + // ── Row 3: Core Services + Auth side by side ── + subgraph cluster_core { + label="Core Services"; + style=filled; fillcolor="#fff3cd"; + api [label="API\n(Node / Express)", fillcolor="#ffe082"]; + secure_dl [label="Secure Download\n(Node.js)", fillcolor="#ffe082"]; + } + + subgraph cluster_auth { + label="Auth"; + style=filled; fillcolor="#fde8e8"; + signet [label="Signet\n(OAuth 2.0)", fillcolor="#f5b8b8"]; + signet_db [label="PostgreSQL\n(Signet)", fillcolor="#f5b8b8"]; + } + + // ── Row 4: Rhythm (left) + Workers (right) ── + subgraph cluster_workflow { + label="Workflow Engine (Rhythm)"; + style=filled; fillcolor="#e8f5e8"; + rhythm [label="Rhythm API\n(FastAPI)", fillcolor="#a5d6a7"]; + queue [label="RabbitMQ", fillcolor="#a5d6a7"]; + mongo [label="MongoDB\n(Task State)", fillcolor="#a5d6a7"]; + } + + subgraph cluster_workers { + label="Workers (Python / Celery)"; + style=filled; fillcolor="#f0e8ff"; + celery [label="celery_worker\n(Task Executor)", fillcolor="#ce93d8"]; + watch [label="watch\n(File Monitor)", fillcolor="#ce93d8"]; + } + + // ── Row 5: Storage ── + subgraph cluster_storage { + label="Storage"; + style=filled; fillcolor="#fff8e1"; + postgres [label="PostgreSQL\n(App Data)", fillcolor="#ffe57f"]; + landing [label="Shared File Volume\n(/opt/sca/data)", shape=cylinder, fillcolor="#ffe57f"]; + } + + // ── Rank constraints for portrait layout ── + { rank=same; user } + { rank=same; ui } + { rank=same; api; secure_dl; signet } + { rank=same; rhythm; queue; celery; watch } + { rank=same; postgres; landing; mongo } + + // ── Edges ── + user -> ui; + ui -> api; + ui -> secure_dl; + + api -> signet; + signet -> signet_db; + + api -> rhythm [label="dispatch"]; + api -> postgres; + + rhythm -> queue; + rhythm -> mongo; + queue -> celery [label="tasks"]; + celery -> mongo [label="results"]; + celery -> landing; + + watch -> landing; + watch -> api [label="trigger\ningest"]; + + secure_dl -> landing; +} diff --git a/docs/diagrams/bioloop_architecture.svg b/docs/diagrams/bioloop_architecture.svg new file mode 100644 index 000000000..26f29e61f --- /dev/null +++ b/docs/diagrams/bioloop_architecture.svg @@ -0,0 +1,244 @@ + + + + + + +bioloop_architecture + + +cluster_user + +User + + +cluster_frontend + +Frontend + + +cluster_core + +Core Services + + +cluster_auth + +Auth (Signet) + + +cluster_workflow + +Workflow Engine (Rhythm) + + +cluster_workers + +Workers (Python / Celery) + + +cluster_storage + +Storage + + + +user + +Researcher / +Operator + + + +ui + +UI +(Vue 3 / Vite) +:443 + + + +user->ui + + +HTTPS + + + +api + +API +(Node.js / Express) +:3030 + + + +ui->api + + +proxy /api + + + +secure_dl + +Secure Download +(Node.js) +:3060 + + + +ui->secure_dl + + +proxy /secure_download + + + +signet + +Signet +(OAuth 2.0) +:5050 + + + +api->signet + + +JWKS / token exchange + + + +rhythm + +Rhythm API +(FastAPI) +:5001 + + + +api->rhythm + + +workflow dispatch + + + +postgres + +PostgreSQL +(App Data) + + + +api->postgres + + + + + +landing + + +Shared File Volume +(/opt/sca/data) + + + +secure_dl->landing + + +read + + + +signet_db + +PostgreSQL +(Signet) + + + +signet->signet_db + + + + + +queue + +RabbitMQ + + + +rhythm->queue + + + + + +mongo + +MongoDB +(Task State) + + + +rhythm->mongo + + + + + +celery + +celery_worker +(Task Executor) + + + +queue->celery + + +tasks + + + +celery->mongo + + +results + + + +celery->landing + + + + + +watch + +watch +(File Monitor) + + + +watch->api + + +trigger ingest + + + +watch->landing + + +monitor + + + diff --git a/docs/diagrams/dataflow_pipeline.dot b/docs/diagrams/dataflow_pipeline.dot new file mode 100644 index 000000000..62c9bb828 --- /dev/null +++ b/docs/diagrams/dataflow_pipeline.dot @@ -0,0 +1,49 @@ +digraph dataflow_pipeline { + rankdir=TB; + node [shape=box, style=filled, fontname="Helvetica"]; + compound=true; + + subgraph cluster_lab { + label="Laboratory"; + style=filled; fillcolor="#f0f0f0"; + inst1 [label="Instrument\n(legacy OS)", fillcolor="#d0d0d0"]; + inst2 [label="Instrument\n(modern)", fillcolor="#d0d0d0"]; + } + + subgraph cluster_vm { + label="Hosted Instance"; + style=filled; fillcolor="#dce8f8"; + + samba [label="Samba\n(SMB Gateway)", fillcolor="#b3d4f5"]; + + subgraph cluster_bioloop { + label="Bioloop"; + style=filled; fillcolor="#e8f0fc"; + watch [label="watch worker\n(file monitor)", fillcolor="#90caf9"]; + bioloop [label="API + celery_worker\n(orchestration)", fillcolor="#90caf9"]; + } + + tape [label="Tape Archive\n(HPSS / SDA)", shape=cylinder, fillcolor="#b3d4f5"]; + hpc [label="HPC Filesystem\n(Lustre / Slate)", shape=cylinder, fillcolor="#b3d4f5"]; + } + + subgraph cluster_access { + label="Researcher Access"; + style=filled; fillcolor="#e8f5e8"; + webui [label="Web Browser\n(Bioloop UI)", fillcolor="#a5d6a7"]; + hpclogin [label="HPC Systems\n(direct access)", fillcolor="#a5d6a7"]; + } + + inst1 -> samba [label="SMB write"]; + inst2 -> samba [label="SMB write"]; + samba -> watch [label="new files detected"]; + watch -> bioloop [label="trigger ingest"]; + + bioloop -> tape [label="1. archive\n(tar + checksum)"]; + tape -> bioloop [label="confirmed"]; + bioloop -> hpc [label="2. stage\n(untar to Lustre)"]; + + hpc -> webui [label="secure download"]; + hpc -> hpclogin [label="direct I/O"]; + webui -> bioloop [label="manage / notify"]; +} diff --git a/docs/diagrams/dataflow_pipeline.svg b/docs/diagrams/dataflow_pipeline.svg new file mode 100644 index 000000000..3d1ce7f97 --- /dev/null +++ b/docs/diagrams/dataflow_pipeline.svg @@ -0,0 +1,170 @@ + + + + + + +dataflow_pipeline + + +cluster_lab + +Laboratory + + +cluster_vm + +Hosted Instance + + +cluster_bioloop + +Bioloop + + +cluster_access + +Researcher Access + + + +inst1 + +Instrument +(legacy OS) + + + +samba + +Samba +(SMB Gateway) + + + +inst1->samba + + +SMB write + + + +inst2 + +Instrument +(modern) + + + +inst2->samba + + +SMB write + + + +watch + +watch worker +(file monitor) + + + +samba->watch + + +new files detected + + + +bioloop + +API + celery_worker +(orchestration) + + + +watch->bioloop + + +trigger ingest + + + +tape + + +Tape Archive +(HPSS / SDA) + + + +bioloop->tape + + +1. archive +(tar + checksum) + + + +hpc + + +HPC Filesystem +(Lustre / Slate) + + + +bioloop->hpc + + +2. stage +(untar to Lustre) + + + +tape->bioloop + + +confirmed + + + +webui + +Web Browser +(Bioloop UI) + + + +hpc->webui + + +secure download + + + +hpclogin + +HPC Systems +(direct access) + + + +hpc->hpclogin + + +direct I/O + + + +webui->bioloop + + +manage / notify + + + diff --git a/docs/diagrams/render-diagrams.md b/docs/diagrams/render-diagrams.md new file mode 100644 index 000000000..26a178996 --- /dev/null +++ b/docs/diagrams/render-diagrams.md @@ -0,0 +1,15 @@ +# Diagram rendering + +## Graphviz + +``` +sudo apt update +sudo apt install -y graphviz +``` + +``` +cd docs/diagrams/ +dot -Tsvg bioloop_architecture.dot -o bioloop_architecture.svg +dot -Tsvg dataflow_pipeline.dot -o dataflow_pipeline.svg + +```