diff --git a/charts/llm-d-kserve/.helmignore b/charts/llm-d-kserve/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/llm-d-kserve/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/llm-d-kserve/Chart.yaml b/charts/llm-d-kserve/Chart.yaml new file mode 100644 index 0000000..2c6e0f7 --- /dev/null +++ b/charts/llm-d-kserve/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +type: application + +name: llm-d-kserve +description: A Helm chart for deploying LLM-D with KServe on OpenShift AI + +version: 0.1.0 + +# OpenShift AI version +appVersion: "3.3.0" + +icon: https://avatars.githubusercontent.com/u/211385051?s=48&v=4 +sources: + - https://github.com/redhat-ai-services/helm-charts/tree/main/charts/llm-d-kserve diff --git a/charts/llm-d-kserve/README.md b/charts/llm-d-kserve/README.md new file mode 100644 index 0000000..d55e92c --- /dev/null +++ b/charts/llm-d-kserve/README.md @@ -0,0 +1,78 @@ +# llm-d-kserve + +A Helm chart for deploying LLM-D with KServe on OpenShift AI + +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.3.0](https://img.shields.io/badge/AppVersion-3.3.0-informational?style=flat-square) + +## Installing the Chart + +To access charts from this from the cli repository add it: + +```sh +helm repo add redhat-ai-services https://redhat-ai-services.github.io/helm-charts/ +helm repo update redhat-ai-services +helm upgrade -i [release-name] redhat-ai-services/llm-d-kserve +``` + +To include a chart from this repository in an umbrella chart, include it in your dependencies in your `Chart.yaml` file. + +```yaml +apiVersion: v2 +name: example-chart +description: A Helm chart for Kubernetes +type: application + +version: 0.1.0 + +appVersion: "1.16.0" + +dependencies: + - name: "llm-d-kserve" + version: "0.1.0" + repository: "https://redhat-ai-services.github.io/helm-charts/" +``` + +## Usage + +### Quick Start + +TODO + +## Configuration + +For a complete list of all configuration options, see the [Values](#values) section below. + +## Source Code + +* + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| fullnameOverride | string | `""` | String to fully override fullname template | +| hardwareProfile.create | bool | `false` | Creates a hardware profile and adds it to the release namespace. | +| hardwareProfile.identifiers | list | `[]` | The identifiers to add to the hardware profile. This value will only be used if create is set to true. | +| hardwareProfile.name | string | `"nvidia-gpu"` | The name of the hardware profile to use. If create is set to true, this value will be ignored and the hardware profile will be created with the release name. | +| hardwareProfile.namespace | string | `"redhat-ods-applications"` | The namespace of the hardware profile to use. If create is set to true, this value will be ignored and the hardware profile will be created in the release namespace. | +| hardwareProfile.tolerations | list | `[]` | The tolerations to apply to the hardware profile. This value will only be used if create is set to true. | +| imagePullSecrets | list | `[]` | The image pull secrets to use for the model server. | +| model.name | string | `"meta-llama/llama-3.2-1b-instruct"` | The name of the model to be advertised in the /v1/models endpoint | +| model.uri | string | `"oci://quay.io/redhat-ai-services/modelcar-catalog:llama-3.2-1b-instruct"` | The URI of the model to be used by the model server | +| nameOverride | string | `""` | String to partially override fullname template (will maintain the release name) | +| scheduler.endpointPickerConfig | string | `""` | | +| serviceAccount.annotations | object | `{}` | Annotations to add to the service account | +| serviceAccount.automount | bool | `true` | Automatically mount a ServiceAccount's API credentials? | +| serviceAccount.create | bool | `true` | Specifies whether a service account should be created | +| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | +| vllm.additionalArguments | string | `""` | The additional arguments to pass to the model server. | +| vllm.image.pullPolicy | string | `"IfNotPresent"` | The pull policy of the image to use for the model server | +| vllm.image.repository | string | `""` | The repository of the image to use for the model server If not set, the default image will be used. | +| vllm.image.tag | string | `""` | The tag of the image to use for the model server | +| vllm.replicaCount | int | `1` | The number of replicas to run for the model server | +| vllm.resources | object | `{}` | The resources to apply to the model server pod. If not set, the hardware profile will add the default resources. | +| vllm.tolerations | list | `[]` | The tolerations to apply to the model server pod. If not set, the hardware profile will add the default tolerations. | +| vllm.topologySpreadConstraints | list | `[{"maxSkew":1,"topologyKey":"kubernetes.io/hostname","whenUnsatisfiable":"ScheduleAnyway"}]` | Pod topology spread constraints for the model server. If labelSelector is omitted on an item, the chart sets matchLabels app.kubernetes.io/name to llm-d-kserve.fullname. | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/charts/llm-d-kserve/README.md.gotmpl b/charts/llm-d-kserve/README.md.gotmpl new file mode 100644 index 0000000..28ca528 --- /dev/null +++ b/charts/llm-d-kserve/README.md.gotmpl @@ -0,0 +1,50 @@ +{{ template "chart.header" . }} +{{ template "chart.description" . }} + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +## Installing the Chart + +To access charts from this from the cli repository add it: + +```sh +helm repo add redhat-ai-services https://redhat-ai-services.github.io/helm-charts/ +helm repo update redhat-ai-services +helm upgrade -i [release-name] redhat-ai-services/{{ template "chart.name" . }} +``` + +To include a chart from this repository in an umbrella chart, include it in your dependencies in your `Chart.yaml` file. + +```yaml +apiVersion: v2 +name: example-chart +description: A Helm chart for Kubernetes +type: application + +version: 0.1.0 + +appVersion: "1.16.0" + +dependencies: + - name: "{{ template "chart.name" . }}" + version: "{{ template "chart.version" . }}" + repository: "https://redhat-ai-services.github.io/helm-charts/" +``` + +## Usage + +### Quick Start + +TODO + +## Configuration + +For a complete list of all configuration options, see the [Values](#values) section below. + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} + +{{ template "helm-docs.versionFooter" . }} diff --git a/charts/llm-d-kserve/templates/NOTES.txt b/charts/llm-d-kserve/templates/NOTES.txt new file mode 100644 index 0000000..3a72982 --- /dev/null +++ b/charts/llm-d-kserve/templates/NOTES.txt @@ -0,0 +1,65 @@ +Successfully deployed LLM-D InferenceService! + +DEPLOYMENT INFORMATION: + Name: {{ include "llm-d-kserve.fullname" . }} + Namespace: {{ .Release.Namespace }} + Model: {{ .Values.model.name }} + Model URI: {{ .Values.model.uri }} + Replicas: {{ .Values.vllm.replicaCount }} +{{- if .Values.hardwareProfile.create }} + + A hardware profile "{{ include "llm-d-kserve.fullname" . }}" has been created in this namespace. +{{- else }} + + Hardware Profile: {{ .Values.hardwareProfile.name }} (namespace: {{ .Values.hardwareProfile.namespace }}) +{{- end }} + +CHECK STATUS: + oc get llminferenceservices {{ include "llm-d-kserve.fullname" . }} -n {{ .Release.Namespace }} + oc get pods -l serving.kserve.io/llminferenceservice={{ include "llm-d-kserve.fullname" . }} -n {{ .Release.Namespace }} + +INTERNAL ACCESS: + Your model is configured for internal cluster access. + + To get the service URL (if available): + oc get llminferenceservices {{ include "llm-d-kserve.fullname" . }} -n {{ .Release.Namespace }} -o jsonpath='{.status.url}' + + For local access, use port forwarding: + oc port-forward svc/{{ include "llm-d-kserve.fullname" . }}-predictor -n {{ .Release.Namespace }} 8080:80 + + Then test with: + curl -X POST http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "{{ .Values.model.name }}", + "messages": [{"role": "user", "content": "Hello, how are you?"}], + "max_tokens": 50 + }' + + Or for completions: + curl -X POST http://localhost:8080/v1/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "{{ .Values.model.name }}", + "prompt": "Hello, how are you?", + "max_tokens": 50 + }' + +USEFUL COMMANDS: + # Check pod logs + oc logs -l serving.kserve.io/llminferenceservice={{ include "llm-d-kserve.fullname" . }} -n {{ .Release.Namespace }} -f + + # List available models + curl http://localhost:8080/v1/models + + # Health check + curl http://localhost:8080/health + +API DOCUMENTATION: + LLM-D exposes an OpenAI-compatible API. Available endpoints: + - GET /v1/models - List available models + - POST /v1/chat/completions - Chat completion + - POST /v1/completions - Text completion + - GET /health - Health check + + Use port-forward (see above) before calling these endpoints from your machine. diff --git a/charts/llm-d-kserve/templates/_helpers.tpl b/charts/llm-d-kserve/templates/_helpers.tpl new file mode 100644 index 0000000..b0d4d1e --- /dev/null +++ b/charts/llm-d-kserve/templates/_helpers.tpl @@ -0,0 +1,83 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "llm-d-kserve.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "llm-d-kserve.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "llm-d-kserve.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "llm-d-kserve.labels" -}} +helm.sh/chart: {{ include "llm-d-kserve.chart" . }} +{{ include "llm-d-kserve.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "llm-d-kserve.selectorLabels" -}} +app.kubernetes.io/name: {{ include "llm-d-kserve.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "llm-d-kserve.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "llm-d-kserve.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Render a map as a compact YAML list item ("- key: …" instead of "-\n key: …"). +*/}} +{{- define "llm-d-kserve.toYamlListItem" -}} +{{- $lines := splitList "\n" (toYaml .) -}} +- {{ first $lines }} +{{- range rest $lines }} +{{- if . }} + {{ . }} +{{- end }} +{{- end }} +{{- end -}} + +{{- define "llm-d-kserve.image" -}} +{{- if hasPrefix "sha256:" (toString .Values.vllm.image.tag) }} +{{- printf "%s@%s" .Values.vllm.image.repository .Values.vllm.image.tag }} +{{- else }} +{{- printf "%s:%s" .Values.vllm.image.repository .Values.vllm.image.tag }} +{{- end }} +{{- end }} diff --git a/charts/llm-d-kserve/templates/auth-rbac.yaml b/charts/llm-d-kserve/templates/auth-rbac.yaml new file mode 100644 index 0000000..d6a00eb --- /dev/null +++ b/charts/llm-d-kserve/templates/auth-rbac.yaml @@ -0,0 +1,44 @@ +{{- if .Values.auth.enabled }} +{{- range $serviceAccount := .Values.auth.serviceAccounts }} +{{- if list nil true | has $serviceAccount.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $serviceAccount.name }} + namespace: {{ $serviceAccount.namespace | default $.Release.Namespace }} + labels: + {{- include "llm-d-kserve.labels" $ | nindent 4 }} +{{ if list nil true | has $serviceAccount.createLegacyToken }} +secrets: + - name: {{ $serviceAccount.name }} +{{- end }} +automountServiceAccountToken: true +{{ if list nil true | has $serviceAccount.createLegacyToken }} +--- +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ $serviceAccount.name }} + annotations: + kubernetes.io/service-account.name: {{ $serviceAccount.name }} +{{- end }} +{{- end }} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "llm-d-kserve.fullname" $ }}-{{ $serviceAccount.name }}-view + labels: + {{- include "llm-d-kserve.labels" $ | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccount.name }} + namespace: {{ $serviceAccount.namespace | default $.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "llm-d-kserve.fullname" $ }}-view +{{- end }} +{{- end }} diff --git a/charts/llm-d-kserve/templates/auth-role.yaml b/charts/llm-d-kserve/templates/auth-role.yaml new file mode 100644 index 0000000..5d94d7b --- /dev/null +++ b/charts/llm-d-kserve/templates/auth-role.yaml @@ -0,0 +1,17 @@ +{{- if .Values.auth.enabled }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "llm-d-kserve.fullname" . }}-view + labels: + {{- include "llm-d-kserve.labels" $ | nindent 4 }} +rules: + - verbs: + - get + apiGroups: + - serving.kserve.io + resources: + - inferenceservices + resourceNames: + - {{ include "llm-d-kserve.inferenceServiceName" . }} +{{- end }} diff --git a/charts/llm-d-kserve/templates/hardwareprofile.yaml b/charts/llm-d-kserve/templates/hardwareprofile.yaml new file mode 100644 index 0000000..7f99892 --- /dev/null +++ b/charts/llm-d-kserve/templates/hardwareprofile.yaml @@ -0,0 +1,25 @@ +{{- if .Values.hardwareProfile.create }} +apiVersion: infrastructure.opendatahub.io/v1 +kind: HardwareProfile +metadata: + annotations: + opendatahub.io/dashboard-feature-visibility: '["model-serving"]' + opendatahub.io/description: "" + opendatahub.io/disabled: "false" + opendatahub.io/display-name: {{ include "llm-d-kserve.fullname" . | quote }} + labels: + {{- include "llm-d-kserve.labels" . | nindent 4 }} + name: {{ include "llm-d-kserve.fullname" . }} +spec: + identifiers: + {{- toYaml .Values.hardwareProfile.identifiers | nindent 4 }} + scheduling: + {{- if .Values.hardwareProfile.tolerations }} + node: + tolerations: + {{- toYaml .Values.hardwareProfile.tolerations | nindent 8 }} + {{- else }} + node: {} + {{- end }} + type: Node +{{- end }} diff --git a/charts/llm-d-kserve/templates/llminferenceservice.yaml b/charts/llm-d-kserve/templates/llminferenceservice.yaml new file mode 100644 index 0000000..3fcaf8f --- /dev/null +++ b/charts/llm-d-kserve/templates/llminferenceservice.yaml @@ -0,0 +1,65 @@ +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + annotations: + {{- if .Values.hardwareProfile.create }} + opendatahub.io/hardware-profile-name: {{ include "llm-d-kserve.fullname" . | quote }} + opendatahub.io/hardware-profile-namespace: {{ .Release.Namespace | quote }} + {{- else }} + opendatahub.io/hardware-profile-name: {{ .Values.hardwareProfile.name | quote }} + opendatahub.io/hardware-profile-namespace: {{ .Values.hardwareProfile.namespace | quote }} + {{- end }} + opendatahub.io/model-type: generative + openshift.io/display-name: {{ .Values.model.name | quote }} + security.opendatahub.io/enable-auth: {{ .Values.auth.enabled | quote }} + labels: + {{- include "llm-d-kserve.labels" . | nindent 4 }} + opendatahub.io/dashboard: "true" + name: {{ include "llm-d-kserve.fullname" . }} +spec: + model: + name: {{ .Values.model.name | quote }} + uri: {{ .Values.model.uri | quote }} + router: + gateway: {} + route: {} + scheduler: {} + replicas: {{ .Values.vllm.replicaCount }} + template: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 6 }} + {{- end }} + {{- if .Values.vllm.topologySpreadConstraints }} + topologySpreadConstraints: + {{- range .Values.vllm.topologySpreadConstraints }} + {{- if hasKey . "labelSelector" }} + {{- /* Caller supplied labelSelector — render unchanged. */}} + {{- include "llm-d-kserve.toYamlListItem" . | nindent 6 }} + {{- else }} + {{- /* No labelSelector — default to matching this chart's pods. */}} + {{- $c := deepCopy . }} + {{- $defaultMatchLabels := dict "app.kubernetes.io/name" (include "llm-d-kserve.fullname" $) }} + {{- $defaultLabelSelector := dict "matchLabels" $defaultMatchLabels }} + {{- $_ := set $c "labelSelector" $defaultLabelSelector }} + {{- include "llm-d-kserve.toYamlListItem" $c | nindent 6 }} + {{- end }} + {{- end }} + {{- end }} + containers: + - name: main + env: + - name: VLLM_ADDITIONAL_ARGS + value: {{ join " " .Values.vllm.additionalArguments | quote }} + {{- if .Values.vllm.image.repository }} + image: {{ include "llm-d-kserve.image" . }} + {{- end }} + imagePullPolicy: {{ .Values.vllm.image.pullPolicy | quote }} + {{- if .Values.vllm.resources }} + resources: + {{- toYaml .Values.vllm.resources | nindent 10 }} + {{- end }} + {{- if .Values.vllm.tolerations }} + tolerations: + {{- toYaml .Values.vllm.tolerations | nindent 8 }} + {{- end }} diff --git a/charts/llm-d-kserve/templates/serviceaccount.yaml b/charts/llm-d-kserve/templates/serviceaccount.yaml new file mode 100644 index 0000000..39574b5 --- /dev/null +++ b/charts/llm-d-kserve/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "llm-d-kserve.serviceAccountName" . }} + labels: + {{- include "llm-d-kserve.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/charts/llm-d-kserve/values.yaml b/charts/llm-d-kserve/values.yaml new file mode 100644 index 0000000..e5275ed --- /dev/null +++ b/charts/llm-d-kserve/values.yaml @@ -0,0 +1,144 @@ +# -- String to partially override fullname template (will maintain the release name) +nameOverride: "" +# -- String to fully override fullname template +fullnameOverride: "" + +model: + # -- The name of the model to be advertised in the /v1/models endpoint + name: "meta-llama/llama-3.2-1b-instruct" + # -- The URI of the model to be used by the model server + uri: oci://quay.io/redhat-ai-services/modelcar-catalog:llama-3.2-1b-instruct + +auth: + # -- Secures the model endpoint and creates a role to grant permissions to service accounts + enabled: false + # -- Creates service accounts with permissions to access the secured endpoint + # If create not set, assume true; if create is set to false, the service account must already exist. + # If createLegacyToken not set, assume true; if createLegacyToken is true, a static token will be created for the service account. + # If namespace not set, assume the release namespace. + serviceAccounts: [] + # - name: my-service-account + # - name: existing-service-account + # create: false + # - name: existing-in-another-namespace + # namespace: my-other-namespace + # create: false + # - name: no-token + # createLegacyToken: false + +hardwareProfile: + # -- Creates a hardware profile and adds it to the release namespace. + create: false + # -- The name of the hardware profile to use. + # If create is set to true, this value will be ignored and the hardware profile will be created with the release name. + name: nvidia-gpu + # -- The namespace of the hardware profile to use. + # If create is set to true, this value will be ignored and the hardware profile will be created in the release namespace. + namespace: redhat-ods-applications + # -- The identifiers to add to the hardware profile. + # This value will only be used if create is set to true. + identifiers: [] + # - identifier: cpu + # displayName: CPU + # resourceType: CPU + # defaultCount: "2" + # minCount: "1" + # maxCount: "4" + # - identifier: memory + # displayName: Memory + # resourceType: Memory + # defaultCount: 4Gi + # minCount: 2Gi + # maxCount: 8Gi + # - identifier: nvidia.com/gpu + # displayName: Nvidia Gpu + # defaultCount: 1 + # minCount: 1 + # maxCount: 2 + # resourceType: Accelerator + # -- The tolerations to apply to the hardware profile. + # This value will only be used if create is set to true. + tolerations: [] + # - key: nvidia.com/gpu + # operator: Exists + # effect: NoSchedule + +vllm: + # -- The number of replicas to run for the model server + replicaCount: 1 + # -- The tolerations to apply to the model server pod. + # If not set, the hardware profile will add the default tolerations. + tolerations: [] + # - key: "nvidia.com/gpu" + # operator: "Exists" + # effect: "NoSchedule" + # -- The resources to apply to the model server pod. + # If not set, the hardware profile will add the default resources, + # however it is highly recommended to set the resources explicitly. + resources: {} + # requests: + # cpu: 1 + # memory: 4Gi + # nvidia.com/gpu: 1 + # limits: + # cpu: 2 + # memory: 8Gi + # nvidia.com/gpu: 1\ + + # -- The additional arguments to pass to the model server. + additionalArguments: [] + # - "--max-model-len=8192" + # - "--gpu-memory-utilization=0.90" + + image: + # -- The repository of the image to use for the model server + # If not set, the default image will be used. + repository: "" + # -- The tag of the image to use for the model server + tag: "" + # -- The pull policy of the image to use for the model server + pullPolicy: IfNotPresent + # -- Pod topology spread constraints for the model server. + # If labelSelector is omitted on an item, the chart sets matchLabels app.kubernetes.io/name to llm-d-kserve.fullname. + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + +scheduler: + endpointPickerConfig: "" #|4 + # apiVersion: inference.networking.x-k8s.io/v1alpha1 + # kind: EndpointPickerConfig + # plugins: + # - type: pd-profile-handler + # parameters: + # threshold: 30000 + # - type: prefill-header-handler + # - type: prefill-filter + # - type: decode-filter + # - type: random-picker + # parameters: + # maxNumOfEndpoints: 1 + # schedulingProfiles: + # - name: prefill + # plugins: + # - pluginRef: prefill-filter + # - pluginRef: random-picker + # - name: decode + # plugins: + # - pluginRef: decode-filter + # - pluginRef: random-picker + +# -- The image pull secrets to use for the model server. +imagePullSecrets: [] + +serviceAccount: + # -- Specifies whether a service account should be created + create: true + # -- Automatically mount a ServiceAccount's API credentials? + automount: true + # -- Annotations to add to the service account + annotations: {} + # -- The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: ""