diff --git a/ai4papi/routers/v1/catalog/modules.py b/ai4papi/routers/v1/catalog/modules.py index 007d32c7..9784631e 100644 --- a/ai4papi/routers/v1/catalog/modules.py +++ b/ai4papi/routers/v1/catalog/modules.py @@ -6,6 +6,7 @@ from ai4papi import quotas, nomad, utils import ai4papi.conf as papiconf from .common import Catalog, retrieve_docker_tags, fmt_map +import ai4papi.routers.v1.stats.deployments as stats_router gpu_specs = utils.gpu_specs() @@ -44,6 +45,12 @@ def get_config( conf["general"]["docker_tag"]["options"] = tags conf["general"]["docker_tag"]["value"] = tags[0] + # Add available datacenters + cluster_stats = stats_router.get_cluster_stats(vo) + conf["general"]["datacenter"]["options"] += list( + cluster_stats["datacenters"].keys() + ) + # Modify the resources limits for a given user or VO conf["hardware"] = quotas.limit_resources( item_name=item_name, diff --git a/ai4papi/routers/v1/catalog/tools.py b/ai4papi/routers/v1/catalog/tools.py index e712befb..b6013968 100644 --- a/ai4papi/routers/v1/catalog/tools.py +++ b/ai4papi/routers/v1/catalog/tools.py @@ -8,6 +8,7 @@ from ai4papi import quotas, utils, nomad import ai4papi.conf as papiconf from .common import Catalog, retrieve_docker_tags, fmt_map +import ai4papi.routers.v1.stats.deployments as stats_router security = HTTPBearer() @@ -43,6 +44,12 @@ def get_config( vo=vo, ) + # Add available datacenters + cluster_stats = stats_router.get_cluster_stats(vo) + conf["general"]["datacenter"]["options"] += list( + cluster_stats["datacenters"].keys() + ) + # Fill with correct Docker image and tags if item_name in ["ai4os-federated-server", "ai4os-ai4life-loader", "ai4os-dev-env"]: # Parse docker registry diff --git a/ai4papi/routers/v1/deployments/modules.py b/ai4papi/routers/v1/deployments/modules.py index 329a7727..45a6854e 100644 --- a/ai4papi/routers/v1/deployments/modules.py +++ b/ai4papi/routers/v1/deployments/modules.py @@ -249,6 +249,7 @@ def create_deployment( { "JOB_UUID": job_uuid, "NAMESPACE": papiconf.MAIN_CONF["nomad"]["namespaces"][vo], + "DATACENTER": user_conf["general"]["datacenter"], "PRIORITY": priority, "OWNER": auth_info["id"], "OWNER_NAME": auth_info["name"], @@ -297,6 +298,13 @@ def create_deployment( user_conf["general"]["docker_image"], usertask ) + # If the user didn't provide a datacenter in the conf, remove that constraint from + # the job + if not user_conf["general"]["datacenter"]: + nomad_conf["Constraints"] = [ + c for c in nomad_conf["Constraints"] if c["LTarget"] != "${node.datacenter}" + ] + # Modify the GPU section if user_conf["hardware"]["gpu_num"] <= 0: # Delete GPU section in CPU deployments diff --git a/ai4papi/routers/v1/deployments/tools.py b/ai4papi/routers/v1/deployments/tools.py index 6169c85b..c547bc79 100644 --- a/ai4papi/routers/v1/deployments/tools.py +++ b/ai4papi/routers/v1/deployments/tools.py @@ -297,6 +297,7 @@ def create_deployment( { "JOB_UUID": job_uuid, "NAMESPACE": papiconf.MAIN_CONF["nomad"]["namespaces"][vo], + "DATACENTER": user_conf["general"]["datacenter"], "PRIORITY": priority, "OWNER": auth_info["id"], "OWNER_NAME": auth_info["name"], @@ -333,6 +334,15 @@ def create_deployment( # Convert template to Nomad conf nomad_conf = nomad.load_job_conf(nomad_conf) + # If the user didn't provide a datacenter in the conf, remove that constraint from + # the job + if not user_conf["general"]["datacenter"]: + nomad_conf["Constraints"] = [ + c + for c in nomad_conf["Constraints"] + if c["LTarget"] != "${node.datacenter}" + ] + tasks = nomad_conf["TaskGroups"][0]["Tasks"] usertask = [t for t in tasks if t["Name"] == "main"][0] diff --git a/etc/modules/nomad.hcl b/etc/modules/nomad.hcl index b33856da..b3051313 100644 --- a/etc/modules/nomad.hcl +++ b/etc/modules/nomad.hcl @@ -56,6 +56,14 @@ job "module-${JOB_UUID}" { weight = -100 # anti-affinity for ai4eosc clients } + # Sometimes the user wants the deployment to land in a particular datacenter to comply + # with ISO or privacy requirements (i.e. data must not leave the datacenter) + constraint { + attribute = "${node.datacenter}" + operator = "=" + value = "${DATACENTER}" + } + # CPU-only jobs should deploy *preferably* on CPU clients (affinity) to avoid # overloading GPU clients with CPU-only jobs. affinity { diff --git a/etc/modules/user.yaml b/etc/modules/user.yaml index c907bf35..ccef10ad 100644 --- a/etc/modules/user.yaml +++ b/etc/modules/user.yaml @@ -52,6 +52,12 @@ general: value: '' description: Select a password for your IDE (JupyterLab or VS Code). It should have at least 9 characters. + datacenter: + name: Datacenter + value: '' + options: [''] + description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter. + hardware: cpu_num: name: Number of CPUs diff --git a/etc/tools/ai4os-ai4life-loader/nomad.hcl b/etc/tools/ai4os-ai4life-loader/nomad.hcl index 213cfb67..e4c6921b 100644 --- a/etc/tools/ai4os-ai4life-loader/nomad.hcl +++ b/etc/tools/ai4os-ai4life-loader/nomad.hcl @@ -53,6 +53,14 @@ job "tool-ai4life-${JOB_UUID}" { value = "${NAMESPACE}" } + # Sometimes the user wants the deployment to land in a particular datacenter to comply + # with ISO or privacy requirements (i.e. data must not leave the datacenter) + constraint { + attribute = "${node.datacenter}" + operator = "=" + value = "${DATACENTER}" + } + # Try to deploy iMagine jobs on nodes that are iMagine-exclusive # In this way, we leave AI4EOSC nodes for AI4EOSC users and for iMagine users only # when iMagine nodes are fully booked. diff --git a/etc/tools/ai4os-ai4life-loader/user.yaml b/etc/tools/ai4os-ai4life-loader/user.yaml index 7dcaa027..c6a362cf 100644 --- a/etc/tools/ai4os-ai4life-loader/user.yaml +++ b/etc/tools/ai4os-ai4life-loader/user.yaml @@ -30,7 +30,6 @@ general: value: '' description: Provide some additional extended information about this deployment. - docker_image: name: Docker image value: 'ai4oshub/ai4os-ai4life-loader' @@ -48,6 +47,12 @@ general: description: AI4Life model ID. options: [] + datacenter: + name: Datacenter + value: '' + options: [''] + description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter. + hardware: cpu_num: name: Number of CPUs diff --git a/etc/tools/ai4os-cvat/nomad.hcl b/etc/tools/ai4os-cvat/nomad.hcl index ae3ed3ae..c9b19d87 100644 --- a/etc/tools/ai4os-cvat/nomad.hcl +++ b/etc/tools/ai4os-cvat/nomad.hcl @@ -103,6 +103,14 @@ job "tool-cvat-${JOB_UUID}" { value = "${NAMESPACE}" } + # Sometimes the user wants the deployment to land in a particular datacenter to comply + # with ISO or privacy requirements (i.e. data must not leave the datacenter) + constraint { + attribute = "${node.datacenter}" + operator = "=" + value = "${DATACENTER}" + } + # Try to deploy iMagine jobs on nodes that are iMagine-exclusive # In this way, we leave AI4EOSC nodes for AI4EOSC users and for iMagine users only # when iMagine nodes are fully booked. diff --git a/etc/tools/ai4os-cvat/user.yaml b/etc/tools/ai4os-cvat/user.yaml index 4d86a89b..d1d30334 100644 --- a/etc/tools/ai4os-cvat/user.yaml +++ b/etc/tools/ai4os-cvat/user.yaml @@ -32,6 +32,12 @@ general: value: '' description: Select a password for your CVAT instance superuser. + datacenter: + name: Datacenter + value: '' + options: [''] + description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter. + storage: rclone_conf: diff --git a/etc/tools/ai4os-dev-env/nomad.hcl b/etc/tools/ai4os-dev-env/nomad.hcl index 224e6f6b..3a60265c 100644 --- a/etc/tools/ai4os-dev-env/nomad.hcl +++ b/etc/tools/ai4os-dev-env/nomad.hcl @@ -53,6 +53,14 @@ job "tool-devenv-${JOB_UUID}" { value = "${NAMESPACE}" } + # Sometimes the user wants the deployment to land in a particular datacenter to comply + # with ISO or privacy requirements (i.e. data must not leave the datacenter) + constraint { + attribute = "${node.datacenter}" + operator = "=" + value = "${DATACENTER}" + } + # Try to deploy iMagine jobs on nodes that are iMagine-exclusive # In this way, we leave AI4EOSC nodes for AI4EOSC users and for iMagine users only # when iMagine nodes are fully booked. diff --git a/etc/tools/ai4os-dev-env/user.yaml b/etc/tools/ai4os-dev-env/user.yaml index ba06e1d7..1352c18f 100644 --- a/etc/tools/ai4os-dev-env/user.yaml +++ b/etc/tools/ai4os-dev-env/user.yaml @@ -52,6 +52,12 @@ general: value: '' description: Select a password for your IDE (JupyterLab or VS Code). It should have at least 9 characters. + datacenter: + name: Datacenter + value: '' + options: [''] + description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter. + hardware: cpu_num: name: Number of CPUs diff --git a/etc/tools/ai4os-federated-server/nomad.hcl b/etc/tools/ai4os-federated-server/nomad.hcl index 91ee547d..92ca1f74 100644 --- a/etc/tools/ai4os-federated-server/nomad.hcl +++ b/etc/tools/ai4os-federated-server/nomad.hcl @@ -63,6 +63,14 @@ job "tool-fl-${JOB_UUID}" { weight = -100 # anti-affinity for ai4eosc clients } + # Sometimes the user wants the deployment to land in a particular datacenter to comply + # with ISO or privacy requirements (i.e. data must not leave the datacenter) + constraint { + attribute = "${node.datacenter}" + operator = "=" + value = "${DATACENTER}" + } + # CPU-only jobs should deploy *preferably* on CPU clients (affinity) to avoid # overloading GPU clients with CPU-only jobs. affinity { diff --git a/etc/tools/ai4os-federated-server/user.yaml b/etc/tools/ai4os-federated-server/user.yaml index cc1fa045..b33c0719 100644 --- a/etc/tools/ai4os-federated-server/user.yaml +++ b/etc/tools/ai4os-federated-server/user.yaml @@ -47,6 +47,12 @@ general: value: '' description: Select a password for your IDE (JupyterLab or VS Code). It should have at least 9 characters. + datacenter: + name: Datacenter + value: '' + options: [''] + description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter. + hardware: cpu_num: name: Number of CPUs diff --git a/etc/tools/ai4os-llm/nomad.hcl b/etc/tools/ai4os-llm/nomad.hcl index 898b9205..ea569cc1 100644 --- a/etc/tools/ai4os-llm/nomad.hcl +++ b/etc/tools/ai4os-llm/nomad.hcl @@ -53,6 +53,14 @@ job "tool-llm-${JOB_UUID}" { value = "${NAMESPACE}" } + # Sometimes the user wants the deployment to land in a particular datacenter to comply + # with ISO or privacy requirements (i.e. data must not leave the datacenter) + constraint { + attribute = "${node.datacenter}" + operator = "=" + value = "${DATACENTER}" + } + # Try to deploy iMagine jobs on nodes that are iMagine-exclusive # In this way, we leave AI4EOSC nodes for AI4EOSC users and for iMagine users only # when iMagine nodes are fully booked. diff --git a/etc/tools/ai4os-llm/user.yaml b/etc/tools/ai4os-llm/user.yaml index b5253742..709d1bf4 100644 --- a/etc/tools/ai4os-llm/user.yaml +++ b/etc/tools/ai4os-llm/user.yaml @@ -20,6 +20,12 @@ general: value: '' description: Provide some additional extended information about this deployment. + datacenter: + name: Datacenter + value: '' + options: [''] + description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter. + llm: type: name: Deployment type diff --git a/etc/tools/ai4os-nvflare/nomad.hcl b/etc/tools/ai4os-nvflare/nomad.hcl index b1032564..5645f677 100644 --- a/etc/tools/ai4os-nvflare/nomad.hcl +++ b/etc/tools/ai4os-nvflare/nomad.hcl @@ -54,6 +54,14 @@ job "tool-nvflare-${JOB_UUID}" { value = "${NAMESPACE}" } + # Sometimes the user wants the deployment to land in a particular datacenter to comply + # with ISO or privacy requirements (i.e. data must not leave the datacenter) + constraint { + attribute = "${node.datacenter}" + operator = "=" + value = "${DATACENTER}" + } + # Try to deploy iMagine jobs on nodes that are iMagine-exclusive # In this way, we leave AI4EOSC nodes for AI4EOSC users and for iMagine users only # when iMagine nodes are fully booked. diff --git a/etc/tools/ai4os-nvflare/user.yaml b/etc/tools/ai4os-nvflare/user.yaml index 4fe5c28c..764e99ad 100644 --- a/etc/tools/ai4os-nvflare/user.yaml +++ b/etc/tools/ai4os-nvflare/user.yaml @@ -20,6 +20,12 @@ general: value: '' description: Provide some additional extended information about this deployment. + datacenter: + name: Datacenter + value: '' + options: [''] + description: Sometimes, due to ISO or privacy compliance, you are required to deploy on a particular datacenter. This option allows to force your deployment to land on a specific datacenter. + nvflare: username: name: NVFlare Dashboard username