From 3e8b4a31321bbc9e07db7e6325e0e0f300adff8b Mon Sep 17 00:00:00 2001 From: Charles Brandt Date: Thu, 26 Mar 2026 12:09:40 -0400 Subject: [PATCH 1/2] Add initial architecture diagrams --- docs/diagrams/bioloop_architecture.dot | 73 ++++++++ docs/diagrams/bioloop_architecture.svg | 244 +++++++++++++++++++++++++ docs/diagrams/dataflow_pipeline.dot | 49 +++++ docs/diagrams/dataflow_pipeline.svg | 170 +++++++++++++++++ docs/diagrams/render-diagrams.md | 15 ++ 5 files changed, 551 insertions(+) create mode 100644 docs/diagrams/bioloop_architecture.dot create mode 100644 docs/diagrams/bioloop_architecture.svg create mode 100644 docs/diagrams/dataflow_pipeline.dot create mode 100644 docs/diagrams/dataflow_pipeline.svg create mode 100644 docs/diagrams/render-diagrams.md diff --git a/docs/diagrams/bioloop_architecture.dot b/docs/diagrams/bioloop_architecture.dot new file mode 100644 index 000000000..3ef3ad8aa --- /dev/null +++ b/docs/diagrams/bioloop_architecture.dot @@ -0,0 +1,73 @@ +digraph bioloop_architecture { + rankdir=LR; + node [shape=box, style=filled, fontname="Helvetica"]; + compound=true; + + subgraph cluster_user { + label="User"; + style=filled; fillcolor="#f5f5f5"; + user [label="Researcher /\nOperator", shape=ellipse, fillcolor="#e0e0e0"]; + } + + subgraph cluster_frontend { + label="Frontend"; + style=filled; fillcolor="#dce8f8"; + ui [label="UI\n(Vue 3 / Vite)\n:443", fillcolor="#b3d4f5"]; + } + + subgraph cluster_core { + label="Core Services"; + style=filled; fillcolor="#fff3cd"; + api [label="API\n(Node.js / Express)\n:3030", fillcolor="#ffe082"]; + secure_dl [label="Secure Download\n(Node.js)\n:3060", fillcolor="#ffe082"]; + } + + subgraph cluster_auth { + label="Auth (Signet)"; + style=filled; fillcolor="#fde8e8"; + signet [label="Signet\n(OAuth 2.0)\n:5050", fillcolor="#f5b8b8"]; + signet_db [label="PostgreSQL\n(Signet)", fillcolor="#f5b8b8"]; + } + + subgraph cluster_workflow { + label="Workflow Engine (Rhythm)"; + style=filled; fillcolor="#e8f5e8"; + rhythm [label="Rhythm API\n(FastAPI)\n:5001", fillcolor="#a5d6a7"]; + queue [label="RabbitMQ", fillcolor="#a5d6a7"]; + mongo [label="MongoDB\n(Task State)", fillcolor="#a5d6a7"]; + } + + subgraph cluster_workers { + label="Workers (Python / Celery)"; + style=filled; fillcolor="#f0e8ff"; + celery [label="celery_worker\n(Task Executor)", fillcolor="#ce93d8"]; + watch [label="watch\n(File Monitor)", fillcolor="#ce93d8"]; + } + + subgraph cluster_storage { + label="Storage"; + style=filled; fillcolor="#fff8e1"; + postgres [label="PostgreSQL\n(App Data)", fillcolor="#ffe57f"]; + landing [label="Shared File Volume\n(/opt/sca/data)", shape=cylinder, fillcolor="#ffe57f"]; + } + + user -> ui [label="HTTPS"]; + ui -> api [label="proxy /api"]; + ui -> secure_dl [label="proxy /secure_download"]; + + api -> postgres; + api -> signet [label="JWKS / token exchange"]; + signet -> signet_db; + api -> rhythm [label="workflow dispatch"]; + + rhythm -> queue; + rhythm -> mongo; + queue -> celery [label="tasks"]; + celery -> mongo [label="results"]; + celery -> landing; + + watch -> landing [label="monitor"]; + watch -> api [label="trigger ingest"]; + + secure_dl -> landing [label="read"]; +} diff --git a/docs/diagrams/bioloop_architecture.svg b/docs/diagrams/bioloop_architecture.svg new file mode 100644 index 000000000..26f29e61f --- /dev/null +++ b/docs/diagrams/bioloop_architecture.svg @@ -0,0 +1,244 @@ + + + + + + +bioloop_architecture + + +cluster_user + +User + + +cluster_frontend + +Frontend + + +cluster_core + +Core Services + + +cluster_auth + +Auth (Signet) + + +cluster_workflow + +Workflow Engine (Rhythm) + + +cluster_workers + +Workers (Python / Celery) + + +cluster_storage + +Storage + + + +user + +Researcher / +Operator + + + +ui + +UI +(Vue 3 / Vite) +:443 + + + +user->ui + + +HTTPS + + + +api + +API +(Node.js / Express) +:3030 + + + +ui->api + + +proxy /api + + + +secure_dl + +Secure Download +(Node.js) +:3060 + + + +ui->secure_dl + + +proxy /secure_download + + + +signet + +Signet +(OAuth 2.0) +:5050 + + + +api->signet + + +JWKS / token exchange + + + +rhythm + +Rhythm API +(FastAPI) +:5001 + + + +api->rhythm + + +workflow dispatch + + + +postgres + +PostgreSQL +(App Data) + + + +api->postgres + + + + + +landing + + +Shared File Volume +(/opt/sca/data) + + + +secure_dl->landing + + +read + + + +signet_db + +PostgreSQL +(Signet) + + + +signet->signet_db + + + + + +queue + +RabbitMQ + + + +rhythm->queue + + + + + +mongo + +MongoDB +(Task State) + + + +rhythm->mongo + + + + + +celery + +celery_worker +(Task Executor) + + + +queue->celery + + +tasks + + + +celery->mongo + + +results + + + +celery->landing + + + + + +watch + +watch +(File Monitor) + + + +watch->api + + +trigger ingest + + + +watch->landing + + +monitor + + + diff --git a/docs/diagrams/dataflow_pipeline.dot b/docs/diagrams/dataflow_pipeline.dot new file mode 100644 index 000000000..54e9303a8 --- /dev/null +++ b/docs/diagrams/dataflow_pipeline.dot @@ -0,0 +1,49 @@ +digraph dataflow_pipeline { + rankdir=LR; + node [shape=box, style=filled, fontname="Helvetica"]; + compound=true; + + subgraph cluster_lab { + label="Laboratory"; + style=filled; fillcolor="#f0f0f0"; + inst1 [label="Instrument\n(legacy OS)", fillcolor="#d0d0d0"]; + inst2 [label="Instrument\n(modern)", fillcolor="#d0d0d0"]; + } + + subgraph cluster_vm { + label="Hosted Instance"; + style=filled; fillcolor="#dce8f8"; + + samba [label="Samba\n(SMB Gateway)", fillcolor="#b3d4f5"]; + + subgraph cluster_bioloop { + label="Bioloop"; + style=filled; fillcolor="#e8f0fc"; + watch [label="watch worker\n(file monitor)", fillcolor="#90caf9"]; + bioloop [label="API + celery_worker\n(orchestration)", fillcolor="#90caf9"]; + } + + tape [label="Tape Archive\n(HPSS / SDA)", shape=cylinder, fillcolor="#b3d4f5"]; + hpc [label="HPC Filesystem\n(Lustre / Slate)", shape=cylinder, fillcolor="#b3d4f5"]; + } + + subgraph cluster_access { + label="Researcher Access"; + style=filled; fillcolor="#e8f5e8"; + webui [label="Web Browser\n(Bioloop UI)", fillcolor="#a5d6a7"]; + hpclogin [label="HPC Systems\n(direct access)", fillcolor="#a5d6a7"]; + } + + inst1 -> samba [label="SMB write"]; + inst2 -> samba [label="SMB write"]; + samba -> watch [label="new files detected"]; + watch -> bioloop [label="trigger ingest"]; + + bioloop -> tape [label="1. archive\n(tar + checksum)"]; + tape -> bioloop [label="confirmed"]; + bioloop -> hpc [label="2. stage\n(untar to Lustre)"]; + + hpc -> webui [label="secure download"]; + hpc -> hpclogin [label="direct I/O"]; + webui -> bioloop [label="manage / notify"]; +} diff --git a/docs/diagrams/dataflow_pipeline.svg b/docs/diagrams/dataflow_pipeline.svg new file mode 100644 index 000000000..3d1ce7f97 --- /dev/null +++ b/docs/diagrams/dataflow_pipeline.svg @@ -0,0 +1,170 @@ + + + + + + +dataflow_pipeline + + +cluster_lab + +Laboratory + + +cluster_vm + +Hosted Instance + + +cluster_bioloop + +Bioloop + + +cluster_access + +Researcher Access + + + +inst1 + +Instrument +(legacy OS) + + + +samba + +Samba +(SMB Gateway) + + + +inst1->samba + + +SMB write + + + +inst2 + +Instrument +(modern) + + + +inst2->samba + + +SMB write + + + +watch + +watch worker +(file monitor) + + + +samba->watch + + +new files detected + + + +bioloop + +API + celery_worker +(orchestration) + + + +watch->bioloop + + +trigger ingest + + + +tape + + +Tape Archive +(HPSS / SDA) + + + +bioloop->tape + + +1. archive +(tar + checksum) + + + +hpc + + +HPC Filesystem +(Lustre / Slate) + + + +bioloop->hpc + + +2. stage +(untar to Lustre) + + + +tape->bioloop + + +confirmed + + + +webui + +Web Browser +(Bioloop UI) + + + +hpc->webui + + +secure download + + + +hpclogin + +HPC Systems +(direct access) + + + +hpc->hpclogin + + +direct I/O + + + +webui->bioloop + + +manage / notify + + + diff --git a/docs/diagrams/render-diagrams.md b/docs/diagrams/render-diagrams.md new file mode 100644 index 000000000..26a178996 --- /dev/null +++ b/docs/diagrams/render-diagrams.md @@ -0,0 +1,15 @@ +# Diagram rendering + +## Graphviz + +``` +sudo apt update +sudo apt install -y graphviz +``` + +``` +cd docs/diagrams/ +dot -Tsvg bioloop_architecture.dot -o bioloop_architecture.svg +dot -Tsvg dataflow_pipeline.dot -o dataflow_pipeline.svg + +``` From 5f4c158c450c36e288f06dc74d336d01847eb391 Mon Sep 17 00:00:00 2001 From: Charles Brandt Date: Mon, 30 Mar 2026 18:10:02 -0400 Subject: [PATCH 2/2] Update diagram layout to portrait orientation with styling improvements --- docs/diagrams/bioloop_architecture.dot | 58 ++++++++++++++++---------- docs/diagrams/dataflow_pipeline.dot | 2 +- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/docs/diagrams/bioloop_architecture.dot b/docs/diagrams/bioloop_architecture.dot index 3ef3ad8aa..a99ae1df7 100644 --- a/docs/diagrams/bioloop_architecture.dot +++ b/docs/diagrams/bioloop_architecture.dot @@ -1,38 +1,42 @@ digraph bioloop_architecture { - rankdir=LR; - node [shape=box, style=filled, fontname="Helvetica"]; + rankdir=TB; + node [shape=box, style=filled, fontname="Helvetica", fontsize=11]; + edge [fontname="Helvetica", fontsize=9]; compound=true; + nodesep=0.6; + ranksep=0.8; + splines=ortho; - subgraph cluster_user { - label="User"; - style=filled; fillcolor="#f5f5f5"; - user [label="Researcher /\nOperator", shape=ellipse, fillcolor="#e0e0e0"]; - } + // ── Row 1: User ── + user [label="Researcher /\nOperator", shape=ellipse, style=filled, fillcolor="#e0e0e0"]; + // ── Row 2: Frontend ── subgraph cluster_frontend { label="Frontend"; style=filled; fillcolor="#dce8f8"; - ui [label="UI\n(Vue 3 / Vite)\n:443", fillcolor="#b3d4f5"]; + ui [label="UI\n(Vue 3 / Vite)", fillcolor="#b3d4f5"]; } + // ── Row 3: Core Services + Auth side by side ── subgraph cluster_core { label="Core Services"; style=filled; fillcolor="#fff3cd"; - api [label="API\n(Node.js / Express)\n:3030", fillcolor="#ffe082"]; - secure_dl [label="Secure Download\n(Node.js)\n:3060", fillcolor="#ffe082"]; + api [label="API\n(Node / Express)", fillcolor="#ffe082"]; + secure_dl [label="Secure Download\n(Node.js)", fillcolor="#ffe082"]; } subgraph cluster_auth { - label="Auth (Signet)"; + label="Auth"; style=filled; fillcolor="#fde8e8"; - signet [label="Signet\n(OAuth 2.0)\n:5050", fillcolor="#f5b8b8"]; + signet [label="Signet\n(OAuth 2.0)", fillcolor="#f5b8b8"]; signet_db [label="PostgreSQL\n(Signet)", fillcolor="#f5b8b8"]; } + // ── Row 4: Rhythm (left) + Workers (right) ── subgraph cluster_workflow { label="Workflow Engine (Rhythm)"; style=filled; fillcolor="#e8f5e8"; - rhythm [label="Rhythm API\n(FastAPI)\n:5001", fillcolor="#a5d6a7"]; + rhythm [label="Rhythm API\n(FastAPI)", fillcolor="#a5d6a7"]; queue [label="RabbitMQ", fillcolor="#a5d6a7"]; mongo [label="MongoDB\n(Task State)", fillcolor="#a5d6a7"]; } @@ -44,6 +48,7 @@ digraph bioloop_architecture { watch [label="watch\n(File Monitor)", fillcolor="#ce93d8"]; } + // ── Row 5: Storage ── subgraph cluster_storage { label="Storage"; style=filled; fillcolor="#fff8e1"; @@ -51,14 +56,23 @@ digraph bioloop_architecture { landing [label="Shared File Volume\n(/opt/sca/data)", shape=cylinder, fillcolor="#ffe57f"]; } - user -> ui [label="HTTPS"]; - ui -> api [label="proxy /api"]; - ui -> secure_dl [label="proxy /secure_download"]; + // ── Rank constraints for portrait layout ── + { rank=same; user } + { rank=same; ui } + { rank=same; api; secure_dl; signet } + { rank=same; rhythm; queue; celery; watch } + { rank=same; postgres; landing; mongo } - api -> postgres; - api -> signet [label="JWKS / token exchange"]; + // ── Edges ── + user -> ui; + ui -> api; + ui -> secure_dl; + + api -> signet; signet -> signet_db; - api -> rhythm [label="workflow dispatch"]; + + api -> rhythm [label="dispatch"]; + api -> postgres; rhythm -> queue; rhythm -> mongo; @@ -66,8 +80,8 @@ digraph bioloop_architecture { celery -> mongo [label="results"]; celery -> landing; - watch -> landing [label="monitor"]; - watch -> api [label="trigger ingest"]; + watch -> landing; + watch -> api [label="trigger\ningest"]; - secure_dl -> landing [label="read"]; + secure_dl -> landing; } diff --git a/docs/diagrams/dataflow_pipeline.dot b/docs/diagrams/dataflow_pipeline.dot index 54e9303a8..62c9bb828 100644 --- a/docs/diagrams/dataflow_pipeline.dot +++ b/docs/diagrams/dataflow_pipeline.dot @@ -1,5 +1,5 @@ digraph dataflow_pipeline { - rankdir=LR; + rankdir=TB; node [shape=box, style=filled, fontname="Helvetica"]; compound=true;