Draft: #45 - separate out dial_dataclass and dial_service into separate packages

.dockerignore

@@ -1,5 +1,6 @@
 *
-!src/**/*
+!packages/**/*
+!services/**/*
 !scripts/**/*
 !tests/**/*
 !pyproject.toml

.github/workflows/ci.yml

@@ -67,7 +67,7 @@ jobs:
         run: |
           uv sync --locked --all-extras --all-groups
         continue-on-error: ${{ matrix.os.continue-on-error || matrix.python-version.continue-on-error }}
-      - run: pytest tests/ --cov=src/ --cov-report=html:reports/htmlcov/ --cov-report=xml:reports/coverage_report.xml --junitxml=reports/junit.xml
+      - run: pytest tests/ --cov=packages/ --cov=services/ --cov-report=html:reports/htmlcov/ --cov-report=xml:reports/coverage_report.xml --junitxml=reports/junit.xml
         continue-on-error: ${{ matrix.os.continue-on-error || matrix.python-version.continue-on-error }}
 
   docker-test:

.github/workflows/pypi-release.yml

@@ -0,0 +1,43 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+*"
+  workflow_dispatch:
+
+jobs:
+  build-and-publish:
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+    environment:
+      name: pypi
+      url: https://pypi.org/p/intersect-sdk
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup UV
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.12"
+      # if the output of check_needs_publish is not "True" (only happens if dial_dataclass and dial_service have the same versions), do not try to publish to PyPI or Github Releases
+      - name: "Check if package should be published"
+        id: "check_needs_publish"
+        run: echo "match=$(python -c 'from importlib.metadata import version; print(version("intersect_dial_dataclass") == version("dial_service"))')" >> "$GITHUB_OUTPUT"
+      - name: "Build package"
+        if: ${{ steps.check_needs_publish.outputs.match == 'True' }}
+        run: uv build --no-create-gitignore packages/intersect_dial_dataclass
+      - name: Publish package to PyPI
+        if: ${{ steps.check_needs_publish.outputs.match == 'True' }}
+        run: uv publish
+      - name: upload to github release
+        if: ${{ steps.check_needs_publish.outputs.match == 'True' }}
+        uses: docker://antonyurchenko/git-release:v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # assume any version with "a" (gets "ALPHA") or "b" (gets BETA) or "rc" (release candidates) will be a prerelease
+          PRE_RELEASE: "${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }}"
+        with:
+          args: dist/*

.readthedocs.yaml

@@ -17,7 +17,7 @@ formats: all
 
 python:
   install:
-    - method: pip
-      path: .
-      extra_requirements:
+    - method: uv
+      command: sync
+      groups:
         - docs

Dockerfile

@@ -27,15 +27,20 @@ WORKDIR /app
 RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,source=uv.lock,target=uv.lock \
     --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
-    uv sync --locked --no-install-project --no-editable
+    --mount=type=bind,source=packages/intersect_dial_dataclass/pyproject.toml,target=packages/intersect_dial_dataclass/pyproject.toml \
+    --mount=type=bind,source=services/dial_service/pyproject.toml,target=services/dial_service/pyproject.toml \
+    uv sync --locked --package dial_service --no-install-workspace --no-editable
 
 # Sync the project
-COPY src src
+COPY packages packages
+COPY services services
 RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,source=uv.lock,target=uv.lock \
     --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=bind,source=packages/intersect_dial_dataclass/pyproject.toml,target=packages/intersect_dial_dataclass/pyproject.toml \
+    --mount=type=bind,source=services/dial_service/pyproject.toml,target=services/dial_service/pyproject.toml \
     --mount=type=bind,source=README.md,target=README.md \
-    uv sync --locked --no-editable
+    uv sync --locked --package dial_service --no-editable
 
 FROM --platform=$BUILDPLATFORM gcr.io/distroless/cc:nonroot AS runner
 

README.md

@@ -2,26 +2,39 @@
 
 # Dial
 
+This repository is a uv workspace with two members:
+
+- `packages/intersect_dial_dataclass` (package of the core DIAL types, available on PyPI)
+- `services/dial_service` (DIAL service runtime package, available as a Docker container and a Helm chart)
+
 ## Requirements
 
 - Python >= 3.10
 
 ## Installing (Non-developers)
 
-To install intersect-sdk from PyPI:
+When writing a Client, most users will only need to install the `intersect-sdk` and `intersect-dial-dataclass` .
+
+To install all dependencies from PyPI:
 
-`pip install intersect-sdk`
+```
+pip install intersect-sdk intersect-dial-dataclass
+```
 
-To install Dial from source, git clone this repository and run the following from the root directory (that is, within the neeter-active-learning directory):
+To install the publishable dataclass package from source, run:
 
-`pip install .`
+`pip install ./packages/dial_dataclass`
+
+For more advanced users, to install the service package from source, run:
+
+`pip install -e ./packages/dial_dataclass -e ./services/dial_service`
 
 By default, **only scikit-learn is available as a backend**. We also support GPax and Sable as backends. To install these:
 
-- for GPax: `pip install ".[gpax]"`
-- for Sable: `pip install ".[sable]"`
+- for GPax: `pip install "./packages/dial_dataclass[gpax]"`
+- for Sable: `pip install "./packages/dial_dataclass[sable]"`
 
-Use commas to add multiple backends: `pip install ".[gpax,sable]"`
+Use commas to add multiple backends: `pip install "./packages/dial_dataclass[gpax,sable]"`
 
 ## Installing (developers)
 
@@ -133,3 +146,8 @@ You will need `pytest` installed to run the tests.
 You will need to make sure the `docs` optional dependency group is installed. Then you can run `sphinx build -W --keep-going docs/ docs/_build/` .
 
 To quickly spin up the documentation web server on `http://localhost:8000` : `python -m http.server -d docs/_build/`
+
+## Version management
+
+- always bump `dial_service` whenever we want to tag a new version, and have the tag reference the `dial_service` version
+- when bumping `intersect_dial_dataclass` version, bump it to match the `dial_service` version (okay to skip versions)

docs/active_learning/client_guide.rst

@@ -12,7 +12,7 @@ When you return the ``IntersectClientCallback`` object from the Client callback
 .. codeblock:: python
   from intersect_sdk import IntersectClientCallback, IntersectDirectMessageParams, INTERSECT_RESPONSE_VALUE
 
-  from dial_dataclass import (
+  from intersect_dial_dataclass import (
     DialWorkflowCreationParamsClient,
     DialWorkflowDatasetUpdate,
     DialWorkflowDatasetUpdates,
@@ -76,7 +76,7 @@ To validate the response from a message, you can use the dial_dataclasses types
 .. codeblock:: python
   from intersect_sdk import IntersectClientCallback, IntersectDirectMessageParams, INTERSECT_RESPONSE_VALUE
 
-  from dial_dataclass import
+  from intersect_dial_dataclass import
 
   class CustomOrchestrator:
     # ...

docs/active_learning/dataclass_reference.rst

@@ -9,15 +9,15 @@ If using the iHub interface, these schemas and their descriptions should be visi
 Request Objects
 ===============
 
-.. automodule:: dial_dataclass.dial_dataclass
+.. automodule:: intersect_dial_dataclass.dial_dataclass
    :members:
    :undoc-members:
    :exclude-members: model_computed_fields, model_config, model_fields
 
 Response Objects
 ================
 
-.. automodule:: dial_dataclass.dial_dataclass_responses
+.. automodule:: intersect_dial_dataclass.dial_dataclass_responses
    :members:
    :undoc-members:
    :exclude-members: model_computed_fields, model_config, model_fields

packages/intersect_dial_dataclass/README.md

@@ -0,0 +1,13 @@
+# intersect_dial_dataclass
+
+Publishable data model package for DIAL clients and services.
+
+## Install
+
+From PyPI:
+
+`pip install intersect_dial_dataclass`
+
+Or from source:
+
+`pip install ./packages/intersect_dial_dataclass`

packages/intersect_dial_dataclass/pyproject.toml

@@ -0,0 +1,32 @@
+[project]
+name = "intersect-dial-dataclass"
+version = "0.1.7"
+authors = [
+  { name = "Stephen DeWitt", email = "dewittsj@ornl.gov" },
+  { name = "Lance Drane", email = "dranelt@ornl.gov" },
+  { name = "David Joy", email = "davidjoy022@gmail.com" },
+  { name = "Jacob Sweet", email = "jsweet@gatech.edu" },
+  { name = "Ankit Shrivastava", email = "shrivastavaa@ornl.gov" },
+  { name = "Konstantin Pieper", email = "pieperk@ornl.gov" },
+  { name = "Viktor Reshniak", email = "reshniakv@ornl.gov" },
+]
+description = "Dial dataclass models for client and service integration"
+readme = "README.md"
+requires-python = ">=3.10"
+keywords = ["active learning", "dataclass"]
+license = { text = "BSD-3-Clause" }
+classifiers = ["Programming Language :: Python :: 3"]
+dependencies = ["pydantic>=2.13.0"]
+
+[project.urls]
+Homepage = "https://github.com/INTERSECT-DIAL/dial/"
+Changelog = "https://github.com/INTERSECT-DIAL/dial/blob/main/CHANGELOG.md"
+Documentation = "https://intersect-dial.readthedocs.io/en/latest/"
+Issues = "https://github.com/INTERSECT-DIAL/dial/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/intersect_dial_dataclass"]

pyproject.toml

@@ -1,43 +1,11 @@
-[project]
-name = "dial"
-version = "0.1.6"
-authors = [
-  { name = "Stephen DeWitt", email = "dewittsj@ornl.gov" },
-  { name = "Lance Drane", email = "dranelt@ornl.gov" },
-  { name = "David Joy", email = "davidjoy022@gmail.com" },
-  { name = "Jacob Sweet", email = "jsweet@gatech.edu" },
-  { name = "Ankit Shrivastava", email = "shrivastavaa@ornl.gov" },
-  { name = "Konstantin Pieper", email = "pieperk@ornl.gov" },
-  { name = "Viktor Reshniak", email = "reshniakv@ornl.gov" },
+[tool.uv.workspace]
+members = [
+  "packages/intersect_dial_dataclass",
+  "services/dial_service",
 ]
-description = "Dial (Distributed INTERSECT Active Learning): An INTERSECT service that provides Bayesian optimization and active learning"
-readme = "README.md"
-requires-python = ">=3.10"
-keywords = ["active learning"]
-license = { text = "BSD-3-Clause" }
-classifiers = ["Programming Language :: Python :: 3"]
-# TODO move some dependencies into optional dependencies
-dependencies = [
-  "intersect_sdk>=0.9.3,<0.10.0",
-  "numpy",
-  "scikit-learn>=1.4.0,<2.0.0",   # TODO consider making an optional dependency group
-  "scipy>=1.12.0,<2.0.0",
-  "pymongo>=4.12.1",              # TODO - this is only needed for dial_service, dial_dataclass can use a simple fixture for ObjectID representation which allows it to skip this dependency
-]
-
-[project.urls]
-Homepage = "https://github.com/INTERSECT-DIAL/dial/"
-Changelog = "https://github.com/INTERSECT-DIAL/dial/blob/main/CHANGELOG.md"
-Documentation = "https://intersect-dial.readthedocs.io/en/latest/"
-Issues = "https://github.com/INTERSECT-DIAL/dial/issues"
 
-[project.optional-dependencies]
-docs = ["sphinx>=5.3.0", "furo>=2023.3.27"]
-gpax = [
-  "gpax>=0.1.8",
-  "numpyro<0.20.1", # depend on older numpyro for gpax (numpyro.contrib.module: random_haiku_module)
-]
-sable = ["sable @ git+https://code.ornl.gov/sable/sable.git"]
+[tool.uv.sources]
+intersect_dial_dataclass = { workspace = true }
 
 [dependency-groups]
 dev = [
@@ -50,25 +18,13 @@ dev = [
   "statsmodels>=0.14.4",
   "matplotlib>=3.8.2,<4.0.0",
 ]
-
-# TODO - consider swapping to scikit-build-core backend if we need C/C++ extensions
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[tool.hatch.build.targets.wheel]
-packages = ["src/dial_dataclass", "src/dial_service"]
-
-[tool.hatch.metadata]
-# TODO - remove when Sable is published to PyPI
-allow-direct-references = true
-
-#[tool.pdm.scripts]
-# add --cov-fail-under=<NUMBER> when ready to require a minimum coverage amount
-#test-all = "pytest tests/ --cov=src/ --cov-report=html:reports/htmlcov/ --cov-report=xml:reports/coverage_report.xml --junitxml=reports/junit.xml"
-#test-unit = "pytest tests/unit/"
+docs = [
+  "sphinx>=5.3.0",
+  "furo>=2023.3.27",
+]
 
 [tool.ruff]
+target-version = "py310"
 line-length = 100
 format = { quote-style = 'single' }
 namespace-packages = ["scripts/", "tests/"]

scripts/1d_sable_client.py

@@ -8,6 +8,13 @@
 import matplotlib as mpl
 import matplotlib.pyplot as plt
 import numpy as np
+from intersect_dial_dataclass import (
+    DialInputPredictions,
+    DialInputSingleOtherStrategy,
+    DialWorkflowCreationParamsClient,
+    DialWorkflowDatasetUpdate,
+    Normal,
+)
 from intersect_sdk import (
     INTERSECT_RESPONSE_VALUE,
     HierarchyConfig,
@@ -18,14 +25,6 @@
     default_intersect_lifecycle_loop,
 )
 
-from dial_dataclass import (
-    DialInputPredictions,
-    DialInputSingleOtherStrategy,
-    DialWorkflowCreationParamsClient,
-    DialWorkflowDatasetUpdate,
-    Normal,
-)
-
 mpl.use('agg')
 
 logging.basicConfig(level=logging.INFO)
@@ -121,7 +120,11 @@ def __init__(self, service_destination: str):
         self.service_destination = service_destination
 
     def __call__(
-        self, _source: str, operation: str, _has_error: bool, payload: INTERSECT_RESPONSE_VALUE
+        self,
+        _source: str,
+        operation: str,
+        _has_error: bool,
+        payload: INTERSECT_RESPONSE_VALUE,
     ) -> IntersectClientCallback:
         if _has_error:
             print('============ERROR==============', file=sys.stderr)
@@ -278,7 +281,12 @@ def graph(self):
 
         axs[1].plot(self.x_grid[:, 0], acquisition_values)
         if self.x_next is not None:
-            axs[1].axvline(x=self.x_next[0], color='red', linestyle='--', label='Next Point')
+            axs[1].axvline(
+                x=self.x_next[0],
+                color='red',
+                linestyle='--',
+                label='Next Point',
+            )
         axs[1].set_xlabel('Features, x')
         axs[1].set_ylabel('Acquisition Value')
         axs[1].legend()