llm-d wip

charts/llm-d-kserve/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/llm-d-kserve/Chart.yaml

@@ -0,0 +1,14 @@
+apiVersion: v2
+type: application
+
+name: llm-d-kserve
+description: A Helm chart for deploying LLM-D with KServe on OpenShift AI
+
+version: 0.1.0
+
+# OpenShift AI version
+appVersion: "3.3.0"
+
+icon: https://avatars.githubusercontent.com/u/211385051?s=48&v=4
+sources:
+  - https://github.com/redhat-ai-services/helm-charts/tree/main/charts/llm-d-kserve

charts/llm-d-kserve/README.md

@@ -0,0 +1,78 @@
+# llm-d-kserve
+
+A Helm chart for deploying LLM-D with KServe on OpenShift AI
+
+![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.3.0](https://img.shields.io/badge/AppVersion-3.3.0-informational?style=flat-square)
+
+## Installing the Chart
+
+To access charts from this from the cli repository add it:
+
+```sh
+helm repo add redhat-ai-services https://redhat-ai-services.github.io/helm-charts/
+helm repo update redhat-ai-services
+helm upgrade -i [release-name] redhat-ai-services/llm-d-kserve
+```
+
+To include a chart from this repository in an umbrella chart, include it in your dependencies in your `Chart.yaml` file.
+
+```yaml
+apiVersion: v2
+name: example-chart
+description: A Helm chart for Kubernetes
+type: application
+
+version: 0.1.0
+
+appVersion: "1.16.0"
+
+dependencies:
+  - name: "llm-d-kserve"
+    version: "0.1.0"
+    repository: "https://redhat-ai-services.github.io/helm-charts/"
+```
+
+## Usage
+
+### Quick Start
+
+TODO
+
+## Configuration
+
+For a complete list of all configuration options, see the [Values](#values) section below.
+
+## Source Code
+
+* <https://github.com/redhat-ai-services/helm-charts/tree/main/charts/llm-d-kserve>
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| fullnameOverride | string | `""` | String to fully override fullname template |
+| hardwareProfile.create | bool | `false` | Creates a hardware profile and adds it to the release namespace. |
+| hardwareProfile.identifiers | list | `[]` | The identifiers to add to the hardware profile. This value will only be used if create is set to true. |
+| hardwareProfile.name | string | `"nvidia-gpu"` | The name of the hardware profile to use. If create is set to true, this value will be ignored and the hardware profile will be created with the release name. |
+| hardwareProfile.namespace | string | `"redhat-ods-applications"` | The namespace of the hardware profile to use. If create is set to true, this value will be ignored and the hardware profile will be created in the release namespace. |
+| hardwareProfile.tolerations | list | `[]` | The tolerations to apply to the hardware profile. This value will only be used if create is set to true. |
+| imagePullSecrets | list | `[]` | The image pull secrets to use for the model server. |
+| model.name | string | `"meta-llama/llama-3.2-1b-instruct"` | The name of the model to be advertised  in the /v1/models endpoint |
+| model.uri | string | `"oci://quay.io/redhat-ai-services/modelcar-catalog:llama-3.2-1b-instruct"` | The URI of the model to be used by the model server |
+| nameOverride | string | `""` | String to partially override fullname template (will maintain the release name) |
+| scheduler.endpointPickerConfig | string | `""` |  |
+| serviceAccount.annotations | object | `{}` | Annotations to add to the service account |
+| serviceAccount.automount | bool | `true` | Automatically mount a ServiceAccount's API credentials? |
+| serviceAccount.create | bool | `true` | Specifies whether a service account should be created |
+| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template |
+| vllm.additionalArguments | string | `""` | The additional arguments to pass to the model server. |
+| vllm.image.pullPolicy | string | `"IfNotPresent"` | The pull policy of the image to use for the model server |
+| vllm.image.repository | string | `""` | The repository of the image to use for the model server If not set, the default image will be used. |
+| vllm.image.tag | string | `""` | The tag of the image to use for the model server |
+| vllm.replicaCount | int | `1` | The number of replicas to run for the model server |
+| vllm.resources | object | `{}` | The resources to apply to the model server pod. If not set, the hardware profile will add the default resources. |
+| vllm.tolerations | list | `[]` | The tolerations to apply to the model server pod. If not set, the hardware profile will add the default tolerations. |
+| vllm.topologySpreadConstraints | list | `[{"maxSkew":1,"topologyKey":"kubernetes.io/hostname","whenUnsatisfiable":"ScheduleAnyway"}]` | Pod topology spread constraints for the model server. If labelSelector is omitted on an item, the chart sets matchLabels app.kubernetes.io/name to llm-d-kserve.fullname. |
+
+----------------------------------------------
+Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2)

charts/llm-d-kserve/README.md.gotmpl

@@ -0,0 +1,50 @@
+{{ template "chart.header" . }}
+{{ template "chart.description" . }}
+
+{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }}
+
+## Installing the Chart
+
+To access charts from this from the cli repository add it:
+
+```sh
+helm repo add redhat-ai-services https://redhat-ai-services.github.io/helm-charts/
+helm repo update redhat-ai-services
+helm upgrade -i [release-name] redhat-ai-services/{{ template "chart.name" . }}
+```
+
+To include a chart from this repository in an umbrella chart, include it in your dependencies in your `Chart.yaml` file.
+
+```yaml
+apiVersion: v2
+name: example-chart
+description: A Helm chart for Kubernetes
+type: application
+
+version: 0.1.0
+
+appVersion: "1.16.0"
+
+dependencies:
+  - name: "{{ template "chart.name" . }}"
+    version: "{{ template "chart.version" . }}"
+    repository: "https://redhat-ai-services.github.io/helm-charts/"
+```
+
+## Usage
+
+### Quick Start
+
+TODO
+
+## Configuration
+
+For a complete list of all configuration options, see the [Values](#values) section below.
+
+{{ template "chart.sourcesSection" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+{{ template "chart.valuesSection" . }}
+
+{{ template "helm-docs.versionFooter" . }}

charts/llm-d-kserve/templates/NOTES.txt

@@ -0,0 +1,65 @@
+Successfully deployed LLM-D InferenceService!
+
+DEPLOYMENT INFORMATION:
+  Name: {{ include "llm-d-kserve.fullname" . }}
+  Namespace: {{ .Release.Namespace }}
+  Model: {{ .Values.model.name }}
+  Model URI: {{ .Values.model.uri }}
+  Replicas: {{ .Values.vllm.replicaCount }}
+{{- if .Values.hardwareProfile.create }}
+
+  A hardware profile "{{ include "llm-d-kserve.fullname" . }}" has been created in this namespace.
+{{- else }}
+
+  Hardware Profile: {{ .Values.hardwareProfile.name }} (namespace: {{ .Values.hardwareProfile.namespace }})
+{{- end }}
+
+CHECK STATUS:
+  oc get llminferenceservices {{ include "llm-d-kserve.fullname" . }} -n {{ .Release.Namespace }}
+  oc get pods -l serving.kserve.io/llminferenceservice={{ include "llm-d-kserve.fullname" . }} -n {{ .Release.Namespace }}
+
+INTERNAL ACCESS:
+  Your model is configured for internal cluster access.
+
+  To get the service URL (if available):
+  oc get llminferenceservices {{ include "llm-d-kserve.fullname" . }} -n {{ .Release.Namespace }} -o jsonpath='{.status.url}'
+
+  For local access, use port forwarding:
+  oc port-forward svc/{{ include "llm-d-kserve.fullname" . }}-predictor -n {{ .Release.Namespace }} 8080:80
+
+  Then test with:
+  curl -X POST http://localhost:8080/v1/chat/completions \
+    -H "Content-Type: application/json" \
+    -d '{
+      "model": "{{ .Values.model.name }}",
+      "messages": [{"role": "user", "content": "Hello, how are you?"}],
+      "max_tokens": 50
+    }'
+
+  Or for completions:
+  curl -X POST http://localhost:8080/v1/completions \
+    -H "Content-Type: application/json" \
+    -d '{
+      "model": "{{ .Values.model.name }}",
+      "prompt": "Hello, how are you?",
+      "max_tokens": 50
+    }'
+
+USEFUL COMMANDS:
+  # Check pod logs
+  oc logs -l serving.kserve.io/llminferenceservice={{ include "llm-d-kserve.fullname" . }} -n {{ .Release.Namespace }} -f
+
+  # List available models
+  curl http://localhost:8080/v1/models
+
+  # Health check
+  curl http://localhost:8080/health
+
+API DOCUMENTATION:
+  LLM-D exposes an OpenAI-compatible API. Available endpoints:
+  - GET  /v1/models           - List available models
+  - POST /v1/chat/completions - Chat completion
+  - POST /v1/completions      - Text completion
+  - GET  /health              - Health check
+
+  Use port-forward (see above) before calling these endpoints from your machine.

charts/llm-d-kserve/templates/_helpers.tpl

@@ -0,0 +1,83 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "llm-d-kserve.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "llm-d-kserve.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "llm-d-kserve.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "llm-d-kserve.labels" -}}
+helm.sh/chart: {{ include "llm-d-kserve.chart" . }}
+{{ include "llm-d-kserve.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "llm-d-kserve.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "llm-d-kserve.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "llm-d-kserve.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "llm-d-kserve.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Render a map as a compact YAML list item ("- key: …" instead of "-\n  key: …").
+*/}}
+{{- define "llm-d-kserve.toYamlListItem" -}}
+{{- $lines := splitList "\n" (toYaml .) -}}
+- {{ first $lines }}
+{{- range rest $lines }}
+{{- if . }}
+  {{ . }}
+{{- end }}
+{{- end }}
+{{- end -}}
+
+{{- define "llm-d-kserve.image" -}}
+{{- if hasPrefix "sha256:" (toString .Values.vllm.image.tag) }}
+{{- printf "%s@%s" .Values.vllm.image.repository .Values.vllm.image.tag }}
+{{- else }}
+{{- printf "%s:%s" .Values.vllm.image.repository .Values.vllm.image.tag }}
+{{- end }}
+{{- end }}

charts/llm-d-kserve/templates/auth-rbac.yaml

@@ -0,0 +1,44 @@
+{{- if .Values.auth.enabled }}
+{{- range $serviceAccount := .Values.auth.serviceAccounts }}
+{{- if list nil true | has $serviceAccount.create }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ $serviceAccount.name }}
+  namespace: {{ $serviceAccount.namespace | default $.Release.Namespace }}
+  labels:
+    {{- include "llm-d-kserve.labels" $ | nindent 4 }}
+{{ if list nil true | has $serviceAccount.createLegacyToken }}
+secrets:
+  - name: {{ $serviceAccount.name }}
+{{- end }}
+automountServiceAccountToken: true
+{{ if list nil true | has $serviceAccount.createLegacyToken }}
+---
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/service-account-token
+metadata:
+  name: {{ $serviceAccount.name }}
+  annotations:
+    kubernetes.io/service-account.name: {{ $serviceAccount.name }}
+{{- end }}
+{{- end }}
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "llm-d-kserve.fullname" $ }}-{{ $serviceAccount.name }}-view
+  labels:
+    {{- include "llm-d-kserve.labels" $ | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccount.name }}
+    namespace: {{ $serviceAccount.namespace | default $.Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "llm-d-kserve.fullname" $ }}-view
+{{- end }}
+{{- end }}

charts/llm-d-kserve/templates/auth-role.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.auth.enabled }}
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "llm-d-kserve.fullname" . }}-view
+  labels:
+    {{- include "llm-d-kserve.labels" $ | nindent 4 }}
+rules:
+  - verbs:
+      - get
+    apiGroups:
+      - serving.kserve.io
+    resources:
+      - inferenceservices
+    resourceNames:
+      - {{ include "llm-d-kserve.inferenceServiceName" . }}
+{{- end }}