Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions ai4papi/routers/v1/catalog/modules.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from ai4papi import quotas, nomad, utils
import ai4papi.conf as papiconf
from .common import Catalog, retrieve_docker_tags, fmt_map
import ai4papi.routers.v1.stats.deployments as stats_router


gpu_specs = utils.gpu_specs()
Expand Down Expand Up @@ -44,6 +45,12 @@ def get_config(
conf["general"]["docker_tag"]["options"] = tags
conf["general"]["docker_tag"]["value"] = tags[0]

# Add available datacenters
cluster_stats = stats_router.get_cluster_stats(vo)
conf["general"]["datacenter"]["options"] += list(
cluster_stats["datacenters"].keys()
)

# Modify the resources limits for a given user or VO
conf["hardware"] = quotas.limit_resources(
item_name=item_name,
Expand Down
7 changes: 7 additions & 0 deletions ai4papi/routers/v1/catalog/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from ai4papi import quotas, utils, nomad
import ai4papi.conf as papiconf
from .common import Catalog, retrieve_docker_tags, fmt_map
import ai4papi.routers.v1.stats.deployments as stats_router


security = HTTPBearer()
Expand Down Expand Up @@ -43,6 +44,12 @@ def get_config(
vo=vo,
)

# Add available datacenters
cluster_stats = stats_router.get_cluster_stats(vo)
conf["general"]["datacenter"]["options"] += list(
cluster_stats["datacenters"].keys()
)

# Fill with correct Docker image and tags
if item_name in ["ai4os-federated-server", "ai4os-ai4life-loader", "ai4os-dev-env"]:
# Parse docker registry
Expand Down
8 changes: 8 additions & 0 deletions ai4papi/routers/v1/deployments/modules.py
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,7 @@ def create_deployment(
{
"JOB_UUID": job_uuid,
"NAMESPACE": papiconf.MAIN_CONF["nomad"]["namespaces"][vo],
"DATACENTER": user_conf["general"]["datacenter"],
"PRIORITY": priority,
"OWNER": auth_info["id"],
"OWNER_NAME": auth_info["name"],
Expand Down Expand Up @@ -297,6 +298,13 @@ def create_deployment(
user_conf["general"]["docker_image"], usertask
)

# If the user didn't provide a datacenter in the conf, remove that constraint from
# the job
if not user_conf["general"]["datacenter"]:
nomad_conf["Constraints"] = [
c for c in nomad_conf["Constraints"] if c["LTarget"] != "${node.datacenter}"
]

# Modify the GPU section
if user_conf["hardware"]["gpu_num"] <= 0:
# Delete GPU section in CPU deployments
Expand Down
10 changes: 10 additions & 0 deletions ai4papi/routers/v1/deployments/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,7 @@ def create_deployment(
{
"JOB_UUID": job_uuid,
"NAMESPACE": papiconf.MAIN_CONF["nomad"]["namespaces"][vo],
"DATACENTER": user_conf["general"]["datacenter"],
"PRIORITY": priority,
"OWNER": auth_info["id"],
"OWNER_NAME": auth_info["name"],
Expand Down Expand Up @@ -333,6 +334,15 @@ def create_deployment(
# Convert template to Nomad conf
nomad_conf = nomad.load_job_conf(nomad_conf)

# If the user didn't provide a datacenter in the conf, remove that constraint from
# the job
if not user_conf["general"]["datacenter"]:
nomad_conf["Constraints"] = [
c
for c in nomad_conf["Constraints"]
if c["LTarget"] != "${node.datacenter}"
]

tasks = nomad_conf["TaskGroups"][0]["Tasks"]
usertask = [t for t in tasks if t["Name"] == "main"][0]

Expand Down
8 changes: 8 additions & 0 deletions etc/modules/nomad.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,14 @@ job "module-${JOB_UUID}" {
weight = -100 # anti-affinity for ai4eosc clients
}

# Sometimes the user wants the deployment to land in a particular datacenter to comply
# with ISO or privacy requirements (i.e. data must not leave the datacenter)
constraint {
attribute = "${node.datacenter}"
operator = "="
value = "${DATACENTER}"
}

# CPU-only jobs should deploy *preferably* on CPU clients (affinity) to avoid
# overloading GPU clients with CPU-only jobs.
affinity {
Expand Down
6 changes: 6 additions & 0 deletions etc/modules/user.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ general:
value: ''
description: Select a password for your IDE (JupyterLab or VS Code). It should have at least 9 characters.

datacenter:
name: Datacenter
value: ''
options: ['']
description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter.

hardware:
cpu_num:
name: Number of CPUs
Expand Down
8 changes: 8 additions & 0 deletions etc/tools/ai4os-ai4life-loader/nomad.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,14 @@ job "tool-ai4life-${JOB_UUID}" {
value = "${NAMESPACE}"
}

# Sometimes the user wants the deployment to land in a particular datacenter to comply
# with ISO or privacy requirements (i.e. data must not leave the datacenter)
constraint {
attribute = "${node.datacenter}"
operator = "="
value = "${DATACENTER}"
}

# Try to deploy iMagine jobs on nodes that are iMagine-exclusive
# In this way, we leave AI4EOSC nodes for AI4EOSC users and for iMagine users only
# when iMagine nodes are fully booked.
Expand Down
7 changes: 6 additions & 1 deletion etc/tools/ai4os-ai4life-loader/user.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ general:
value: ''
description: Provide some additional extended information about this deployment.


docker_image:
name: Docker image
value: 'ai4oshub/ai4os-ai4life-loader'
Expand All @@ -48,6 +47,12 @@ general:
description: AI4Life model ID.
options: []

datacenter:
name: Datacenter
value: ''
options: ['']
description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter.

hardware:
cpu_num:
name: Number of CPUs
Expand Down
8 changes: 8 additions & 0 deletions etc/tools/ai4os-cvat/nomad.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,14 @@ job "tool-cvat-${JOB_UUID}" {
value = "${NAMESPACE}"
}

# Sometimes the user wants the deployment to land in a particular datacenter to comply
# with ISO or privacy requirements (i.e. data must not leave the datacenter)
constraint {
attribute = "${node.datacenter}"
operator = "="
value = "${DATACENTER}"
}

# Try to deploy iMagine jobs on nodes that are iMagine-exclusive
# In this way, we leave AI4EOSC nodes for AI4EOSC users and for iMagine users only
# when iMagine nodes are fully booked.
Expand Down
6 changes: 6 additions & 0 deletions etc/tools/ai4os-cvat/user.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@ general:
value: ''
description: Select a password for your CVAT instance superuser.

datacenter:
name: Datacenter
value: ''
options: ['']
description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter.

storage:

rclone_conf:
Expand Down
8 changes: 8 additions & 0 deletions etc/tools/ai4os-dev-env/nomad.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,14 @@ job "tool-devenv-${JOB_UUID}" {
value = "${NAMESPACE}"
}

# Sometimes the user wants the deployment to land in a particular datacenter to comply
# with ISO or privacy requirements (i.e. data must not leave the datacenter)
constraint {
attribute = "${node.datacenter}"
operator = "="
value = "${DATACENTER}"
}

# Try to deploy iMagine jobs on nodes that are iMagine-exclusive
# In this way, we leave AI4EOSC nodes for AI4EOSC users and for iMagine users only
# when iMagine nodes are fully booked.
Expand Down
6 changes: 6 additions & 0 deletions etc/tools/ai4os-dev-env/user.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ general:
value: ''
description: Select a password for your IDE (JupyterLab or VS Code). It should have at least 9 characters.

datacenter:
name: Datacenter
value: ''
options: ['']
description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter.

hardware:
cpu_num:
name: Number of CPUs
Expand Down
8 changes: 8 additions & 0 deletions etc/tools/ai4os-federated-server/nomad.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,14 @@ job "tool-fl-${JOB_UUID}" {
weight = -100 # anti-affinity for ai4eosc clients
}

# Sometimes the user wants the deployment to land in a particular datacenter to comply
# with ISO or privacy requirements (i.e. data must not leave the datacenter)
constraint {
attribute = "${node.datacenter}"
operator = "="
value = "${DATACENTER}"
}

# CPU-only jobs should deploy *preferably* on CPU clients (affinity) to avoid
# overloading GPU clients with CPU-only jobs.
affinity {
Expand Down
6 changes: 6 additions & 0 deletions etc/tools/ai4os-federated-server/user.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ general:
value: ''
description: Select a password for your IDE (JupyterLab or VS Code). It should have at least 9 characters.

datacenter:
name: Datacenter
value: ''
options: ['']
description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter.

hardware:
cpu_num:
name: Number of CPUs
Expand Down
8 changes: 8 additions & 0 deletions etc/tools/ai4os-llm/nomad.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,14 @@ job "tool-llm-${JOB_UUID}" {
value = "${NAMESPACE}"
}

# Sometimes the user wants the deployment to land in a particular datacenter to comply
# with ISO or privacy requirements (i.e. data must not leave the datacenter)
constraint {
attribute = "${node.datacenter}"
operator = "="
value = "${DATACENTER}"
}

# Try to deploy iMagine jobs on nodes that are iMagine-exclusive
# In this way, we leave AI4EOSC nodes for AI4EOSC users and for iMagine users only
# when iMagine nodes are fully booked.
Expand Down
6 changes: 6 additions & 0 deletions etc/tools/ai4os-llm/user.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@ general:
value: ''
description: Provide some additional extended information about this deployment.

datacenter:
name: Datacenter
value: ''
options: ['']
description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter.

llm:
type:
name: Deployment type
Expand Down
8 changes: 8 additions & 0 deletions etc/tools/ai4os-nvflare/nomad.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,14 @@ job "tool-nvflare-${JOB_UUID}" {
value = "${NAMESPACE}"
}

# Sometimes the user wants the deployment to land in a particular datacenter to comply
# with ISO or privacy requirements (i.e. data must not leave the datacenter)
constraint {
attribute = "${node.datacenter}"
operator = "="
value = "${DATACENTER}"
}

# Try to deploy iMagine jobs on nodes that are iMagine-exclusive
# In this way, we leave AI4EOSC nodes for AI4EOSC users and for iMagine users only
# when iMagine nodes are fully booked.
Expand Down
6 changes: 6 additions & 0 deletions etc/tools/ai4os-nvflare/user.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@ general:
value: ''
description: Provide some additional extended information about this deployment.

datacenter:
name: Datacenter
value: ''
options: ['']
description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter.

nvflare:
username:
name: NVFlare Dashboard username
Expand Down