From 09324a0a93da8c82cbbc18acc1f05e645e679ce5 Mon Sep 17 00:00:00 2001 From: Jae-Seung Yeom Date: Fri, 5 Jun 2026 16:08:29 -0700 Subject: [PATCH 1/3] Doc: enable search --- docs/_fragments/design_overview.rst | 6 ++--- docs/_templates/search.html | 37 +++++++++++++++++++++++++++++ docs/conf.py | 3 +++ docs/getting_started.rst | 17 ++++++++----- docs/index.rst | 8 ++++++- 5 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 docs/_templates/search.html diff --git a/docs/_fragments/design_overview.rst b/docs/_fragments/design_overview.rst index 7c64dd38..51edaf02 100644 --- a/docs/_fragments/design_overview.rst +++ b/docs/_fragments/design_overview.rst @@ -9,12 +9,12 @@ Overview of Synchronization and File Transfer DYAD consists of two components: a :ref:`**wrapper**` library that is injected into the application process via ``LD_PRELOAD`` or C++ stream wrapper -classes to transparently intercept I/O calls, and a :ref:`**service**` that +classes to transparently intercept I/O calls, and a :ref:`**service**` that runs as a plugin module to the Flux broker on each node. Together they coordinate file access and transfer between nodes. The wrapper intercepts I/O only for files in the directory it manages, -or :ref:`DYAD managed directory (DMD)`. If the file shared between producer and +or :ref:`DYAD managed directory (DMD)`. If the file shared between producer and consumer under DMD resides on the producer's local storage that is not visible to the consumer, DYAD both synchronizes accesses and transfers files to the DMD on the consumer's local storage. If it is visible to @@ -52,7 +52,7 @@ occur in order (see **c.1–c.3** in the diagram): query when it sees the matching entry, which propagates from the producer's local HFL through the hierarchy. 2. **c.2** ``rpc_get(producer_rank, filename)`` — the wrapper sends an - RPC to the producer's :ref:`DYAD service` requesting the file. The service + RPC to the producer's :ref:`DYAD service` requesting the file. The service transfers the file over the selected :ref:`DTL backend` (Flux RPC, Margo, or UCX) and stores it on the consumer's local storage. 3. **c.3** ``read(managed_dir/filename)`` — the application reads the diff --git a/docs/_templates/search.html b/docs/_templates/search.html new file mode 100644 index 00000000..fbbbf946 --- /dev/null +++ b/docs/_templates/search.html @@ -0,0 +1,37 @@ +{%- extends "!search.html" %} +{% block body %} +

{{ _('Search') }}

+ +
+ + + +
+ {% if search_performed %} +

{{ _('Search Results') }}

+ {% if not search_results %} +

{{ _('Your search did not match any documents.') }}

+ {% endif %} + {% endif %} +
+ {% if search_results %} +
    + {% for href, caption, context in search_results %} +
  • + {{ caption }} +

    {{ context|e }}

    +
    • + {% endfor %} +
+ {% endif %} +
+{% endblock %} diff --git a/docs/conf.py b/docs/conf.py index febffaee..338458a3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,6 +28,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + "sphinx_rtd_theme", "sphinx.ext.autosectionlabel", 'myst_parser', ] @@ -56,6 +57,8 @@ raw_enabled = True html_allow_unsafe = True +autosectionlabel_prefix_document = True +suppress_warnings = ["autosectionlabel.*"] # -- Options for HTML output ------------------------------------------------- diff --git a/docs/getting_started.rst b/docs/getting_started.rst index c00fb0dd..8953c86d 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -362,10 +362,12 @@ Running DYAD There are three steps to running DYAD-enabled applications: -1. :ref:`Create a Flux key-value store (KVS) namespace ` -2. :ref:`Determine the managed directories for each application ` -3. :ref:`Load DYAD's Flux module ` -4. :ref:`Configure and run the DYAD-enabled applications ` +1. :ref:`Create a Flux key-value store (KVS) namespace ` +2. :ref:`Determine the managed directories for each application ` +3. :ref:`Load DYAD's Flux module ` +4. :ref:`Configure and run the DYAD-enabled applications ` + +.. _DYAD_Flux_KVS_NS: Create a Flux KVS Namespace *************************** @@ -384,7 +386,7 @@ consumer. To create this namespace, run the following: The namespace can be whatever string value you want. -.. _dyad_managed_dir: +.. _DYAD_managed_dir: Determine the Managed Directories for Each Application ****************************************************** @@ -416,7 +418,7 @@ for each application. These directories will need to be provided to the commands files in the managed directories. -.. _dyad_user_service: +.. _DYAD_user_service: Load DYAD's Flux Module *********************** @@ -443,6 +445,9 @@ do this by running: $ flux exec -r all flux module load path/to/dyad.so + +.. _DYAD_enabling_apps: + Configure and Run the DYAD-Enabled Applications *********************************************** diff --git a/docs/index.rst b/docs/index.rst index 7139ad9f..3542b686 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -84,10 +84,16 @@ In resolving these challenges, DYAD aims to provide the following to users: dyad_logos +.. toctree:: + :hidden: + :caption: Indices + + genindex + search + Indices and tables ================== * :ref:`genindex` -* :ref:`modindex` * :ref:`search` From 290d1d1026d98426ce24fdebc2805e4c1a9fb63b Mon Sep 17 00:00:00 2001 From: Jae-Seung Yeom Date: Fri, 5 Jun 2026 17:37:45 -0700 Subject: [PATCH 2/3] Doc:enlarge the diagram by reducing margins --- docs/_static/figs/dyad_without_vs_with.svg | 393 +++++++++++---------- docs/_static/img_width.css | 31 ++ docs/conf.py | 3 +- docs/design.rst | 2 + 4 files changed, 233 insertions(+), 196 deletions(-) create mode 100644 docs/_static/img_width.css diff --git a/docs/_static/figs/dyad_without_vs_with.svg b/docs/_static/figs/dyad_without_vs_with.svg index acc49ae2..fc4c35ab 100644 --- a/docs/_static/figs/dyad_without_vs_with.svg +++ b/docs/_static/figs/dyad_without_vs_with.svg @@ -1,11 +1,13 @@ + showgrid="false" + inkscape:clip-to-page="false" /> @@ -41,7 +44,7 @@ refY="3" orient="auto"> @@ -54,8 +57,8 @@ refY="3" orient="auto"> @@ -64,75 +67,75 @@ height="387.66046" fill="#ffffff" id="rect2" - x="105.53488" - y="161.39536" + x="0" + y="0" style="stroke-width:0.685529" /> Without DYAD explicit sync (dependent jobs, API, or polling) P1 P2 Producer W R Consumer Shared FS SharedFS: from bottom-center of W (347.6,333.5) to top of FS (379.5,437.6) --> /lustre/${USER}/file.txt With DYAD P1 P2 Producer W R Consumer Local Storage Local Storage /ssd/${USER} /file.txt /ssd/${USER}/file.txt I/O intercepting wrapper lib + id="text30">I/O intercepting wrapper relying on KVS + RPC + id="text31">lib relying on KVS + RPC diff --git a/docs/_static/img_width.css b/docs/_static/img_width.css new file mode 100644 index 00000000..7d40e4c4 --- /dev/null +++ b/docs/_static/img_width.css @@ -0,0 +1,31 @@ +#dyad-without-vs-with a.reference { + display: block !important; +} + +#dyad-without-vs-with img { + width: 100% !important; + max-width: 100% !important; +} + +#dyad-without-vs-with.align-default { + margin-left: 0 !important; + margin-right: 0 !important; + width: 100% !important; +} + +#dyad-without-vs-with figcaption { + text-align: left !important; +} + +#dyad-without-vs-with { + width: 100% !important; +} + +#dyad-without-vs-with img { + width: 100% !important; + max-width: 100% !important; +} + +#dyad-without-vs-with figcaption { + text-align: left !important; +} diff --git a/docs/conf.py b/docs/conf.py index 338458a3..c61fab27 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -72,7 +72,8 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] html_logo = "_static/logo/dyad_logo_white_streamliner.svg" -html_css_files = ["logo/logo.css"] +html_css_files = ["logo/logo.css", "img_width.css"] + # -- Options for Breathe --------------------------------------------------- diff --git a/docs/design.rst b/docs/design.rst index c26a44ee..78281349 100644 --- a/docs/design.rst +++ b/docs/design.rst @@ -14,6 +14,8 @@ A producer and consumer scenario sharing a file .. figure:: _static/figs/dyad_without_vs_with.svg :alt: Without DYAD vs With DYAD + :name: dyad-without-vs-with + :align: center Without DYAD, producer and consumer share data via a shared file system (e.g., Lustre) with explicit synchronization through dependent jobs, APIs, From 07c9f8936e8c60058ce632eefa7fddea1e3149ae Mon Sep 17 00:00:00 2001 From: Jae-Seung Yeom Date: Fri, 5 Jun 2026 20:11:59 -0700 Subject: [PATCH 3/3] Docs: minor tweak --- README.md | 7 +++---- docs/design.rst | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3408ea56..8841e355 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,10 @@

- Flux + Flux - DYAD: DYnamic and Asynchronous Data Streamliner - + ** DYAD: DYnamic and Asynchronous Data Streamliner ** - DYAD + DYAD

--- diff --git a/docs/design.rst b/docs/design.rst index 78281349..a59009de 100644 --- a/docs/design.rst +++ b/docs/design.rst @@ -25,8 +25,7 @@ A producer and consumer scenario sharing a file RDMA-based data movement, eliminating both the shared file system bottleneck and the need for explicit synchronization. - Check out our use case with a molecular dynamics simulation workflow - :ref:`Lumsden et al. 2024 `. + Check out our use case with a molecular dynamics simulation workflow :ref:`Lumsden et al. 2024 `. Deep Learning Training by Distributed Stochastic Gradient Descent (SGD) =======================================================================