diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 8034386..507da99 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -56,25 +56,13 @@ jobs: --state-values-file ${{ github.workspace }}/helm/example.values.cpu.yaml \ --state-values-file ${{ github.workspace }}/test/e2e/values.yaml - - name: Wait for gateway + - name: Wait for LLMInferenceService run: | - kubectl wait gateway/inference-gateway \ - --namespace thalamus \ - --for=condition=Programmed=True \ - --timeout=120s - - - name: Wait for model pod - run: | - kubectl rollout status deployment/vllm-tiny-llm \ + kubectl wait LLMInferenceService/arnir0-tiny-llm \ --namespace thalamus \ + --for=condition=Ready \ --timeout=600s - - name: Wait for EPP - run: | - kubectl rollout status deployment/vllm-tiny-llm-epp \ - --namespace thalamus \ - --timeout=120s - - name: Port-forward gateway run: | kubectl port-forward svc/inference-gateway \ @@ -104,3 +92,4 @@ jobs: -l app.kubernetes.io/name=agentgateway || true echo "=== Gateway / HTTPRoutes / InferencePools ===" kubectl get gateway,httproutes,inferencepools -n thalamus -o yaml + kubectl get llminferenceservices -n thalamus -o yaml diff --git a/helm/Makefile b/helm/Makefile index d7d7b0f..a6993d5 100644 --- a/helm/Makefile +++ b/helm/Makefile @@ -1,7 +1,7 @@ # renovate: datasource=github-releases depName=kubernetes-sigs/gateway-api GATEWAY_API_VERSION ?= v1.5.1 # renovate: datasource=github-releases depName=kubernetes-sigs/gateway-api-inference-extension -GATEWAY_API_INFERENCE_EXTENSION_VERSION ?= v1.5.0 +GATEWAY_API_INFERENCE_EXTENSION_VERSION ?= v1.4.0 .PHONY: vendor-crds vendor-crds: ## Re-download vendored upstream CRDs from their pinned release tags. diff --git a/helm/example.values.cpu.yaml b/helm/example.values.cpu.yaml index 98c42e7..1e8783e 100644 --- a/helm/example.values.cpu.yaml +++ b/helm/example.values.cpu.yaml @@ -8,11 +8,10 @@ gpu-operator: thalamus: models: - - slug: tiny-llm - model: arnir0/Tiny-LLM - accelerator: cpu + - name: arnir0/Tiny-LLM + config: cpu replicas: 1 - extraArgs: + args: - "--max-model-len=512" - "--dtype=float32" - "--gpu-memory-utilization=0.50" @@ -23,12 +22,3 @@ thalamus: limits: cpu: "4" memory: "6Gi" - - epp: - resources: - requests: - cpu: "100m" - memory: "256Mi" - limits: - cpu: "200m" - memory: "512Mi" diff --git a/helm/example.values.yaml b/helm/example.values.yaml index 04c471e..575ad1b 100644 --- a/helm/example.values.yaml +++ b/helm/example.values.yaml @@ -3,10 +3,10 @@ thalamus: models: - - slug: qwen3-6-27b - model: Qwen/Qwen3.6-27B - accelerator: nvidia - extraArgs: + - name: Qwen/Qwen3.6-27B + config: nvidia + replicas: 1 + args: - "--tensor-parallel-size=2" - "--reasoning-parser=qwen3" resources: diff --git a/helm/gateway-api-inference-extension/Chart.yaml b/helm/gateway-api-inference-extension/Chart.yaml index da51797..d9ab11f 100644 --- a/helm/gateway-api-inference-extension/Chart.yaml +++ b/helm/gateway-api-inference-extension/Chart.yaml @@ -2,4 +2,4 @@ apiVersion: v2 name: gateway-api-inference-extension description: Vendored Gateway API Inference Extension CRDs. type: application -version: 1.5.0 +version: 1.4.0 diff --git a/helm/gateway-api-inference-extension/templates/crds.yaml b/helm/gateway-api-inference-extension/templates/crds.yaml index cb7baf1..30079b4 100644 --- a/helm/gateway-api-inference-extension/templates/crds.yaml +++ b/helm/gateway-api-inference-extension/templates/crds.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - inference.networking.k8s.io/bundle-version: v1.5.0-rc.2 + inference.networking.k8s.io/bundle-version: v1.4.0 name: inferencemodelrewrites.inference.networking.x-k8s.io spec: group: inference.networking.x-k8s.io @@ -238,7 +238,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - inference.networking.k8s.io/bundle-version: v1.5.0-rc.2 + inference.networking.k8s.io/bundle-version: v1.4.0 name: inferenceobjectives.inference.networking.x-k8s.io spec: group: inference.networking.x-k8s.io @@ -331,7 +331,7 @@ spec: Fairness is only enforced and tracked between requests of the same priority. Example: requests with Priority 10 will always be served before - requests with Priority of 0 (the value used if Priority is unset or no InferenceObjective is specified). + requests with Priority of 0 (the value used if Priority is unset or no InfereneceObjective is specified). Similarly requests with a Priority of -10 will always be served after requests with Priority of 0. type: integer required: @@ -429,7 +429,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - inference.networking.k8s.io/bundle-version: v1.5.0-rc.2 + inference.networking.k8s.io/bundle-version: v1.4.0 name: inferencepoolimports.inference.networking.x-k8s.io spec: group: inference.networking.x-k8s.io @@ -761,7 +761,7 @@ kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api-inference-extension/pull/1173 - inference.networking.k8s.io/bundle-version: v1.5.0-rc.2 + inference.networking.k8s.io/bundle-version: v1.4.0 name: inferencepools.inference.networking.k8s.io spec: group: inference.networking.k8s.io @@ -1129,3 +1129,296 @@ status: plural: "" conditions: null storedVersions: null +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: unapproved, experimental-only + inference.networking.k8s.io/bundle-version: v1.4.0 + name: inferencepools.inference.networking.x-k8s.io +spec: + group: inference.networking.x-k8s.io + names: + kind: InferencePool + listKind: InferencePoolList + plural: inferencepools + shortNames: + - xinfpool + singular: inferencepool + scope: Namespaced + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: InferencePool is the Schema for the InferencePools API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: InferencePoolSpec defines the desired state of InferencePool + properties: + extensionRef: + description: Extension configures an endpoint picker as an extension + service. + properties: + failureMode: + default: FailClose + description: |- + Configures how the gateway handles the case when the extension is not responsive. + Defaults to failClose. + enum: + - FailOpen + - FailClose + type: string + group: + default: "" + description: |- + Group is the group of the referent. + The default value is "", representing the Core API group. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: |- + Kind is the Kubernetes resource kind of the referent. + + Defaults to "Service" when not specified. + + ExternalName services can refer to CNAME DNS records that may live + outside of the cluster and as such are difficult to reason about in + terms of conformance. They also may not be safe to forward to (see + CVE-2021-25740 for more information). Implementations MUST NOT + support ExternalName Services. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + portNumber: + description: |- + The port number on the service running the extension. When unspecified, + implementations SHOULD infer a default value of 9002 when the Kind is + Service. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + selector: + additionalProperties: + description: |- + LabelValue is the value of a label. This is used for validation + of maps. This matches the Kubernetes label validation rules: + * must be 63 characters or less (can be empty), + * unless empty, must begin and end with an alphanumeric character ([a-z0-9A-Z]), + * could contain dashes (-), underscores (_), dots (.), and alphanumerics between. + + Valid values include: + + * MyValue + * my.name + * 123-my-value + maxLength: 63 + minLength: 0 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string + description: |- + Selector defines a map of labels to watch model server Pods + that should be included in the InferencePool. + In some cases, implementations may translate this field to a Service selector, so this matches the simple + map used for Service selectors instead of the full Kubernetes LabelSelector type. + If specified, it will be applied to match the model server pods in the same namespace as the InferencePool. + Cross namesoace selector is not supported. + type: object + targetPortNumber: + description: |- + TargetPortNumber defines the port number to access the selected model server Pods. + The number must be in the range 1 to 65535. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - extensionRef + - selector + - targetPortNumber + type: object + status: + default: + parent: + - conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Accepted + parentRef: + kind: Status + name: default + description: Status defines the observed state of InferencePool. + properties: + parent: + description: |- + Parents is a list of parent resources (usually Gateways) that are + associated with the InferencePool, and the status of the InferencePool with respect to + each parent. + + A maximum of 32 Gateways will be represented in this list. When the list contains + `kind: Status, name: default`, it indicates that the InferencePool is not + associated with any Gateway and a controller must perform the following: + + - Remove the parent when setting the "Accepted" condition. + - Add the parent when the controller will no longer manage the InferencePool + and no other parents exist. + items: + description: PoolStatus defines the observed state of InferencePool + from a Gateway. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Accepted + description: |- + Conditions track the state of the InferencePool. + + Known condition types are: + + * "Accepted" + * "ResolvedRefs" + items: + description: Condition contains details for one aspect of + the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + parentRef: + description: GatewayRef indicates the gateway that observed + state of InferencePool. + properties: + group: + default: gateway.networking.k8s.io + description: Group is the group of the referent. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: Kind is kind of the referent. For example "Gateway". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. If not present, + the namespace of the referent is assumed to be the same as + the namespace of the referring object. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + required: + - parentRef + type: object + maxItems: 32 + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null diff --git a/helm/helmfile.yaml.gotmpl b/helm/helmfile.yaml.gotmpl index 63f331f..618c759 100644 --- a/helm/helmfile.yaml.gotmpl +++ b/helm/helmfile.yaml.gotmpl @@ -71,6 +71,58 @@ releases: enabled: true - {{ dig "agentgateway" dict .Values | toYaml | nindent 10 }} + - name: cert-manager + namespace: cert-manager + createNamespace: true + chart: oci://quay.io/jetstack/charts/cert-manager + version: "v1.20.3" + installed: {{ dig "cert-manager" "enabled" true .Values }} + values: + - crds: + enabled: true + + - name: kserve-llmisvc-crd + namespace: kserve + createNamespace: true + chart: oci://ghcr.io/kserve/charts/kserve-llmisvc-crd + version: "v0.19.0" + installed: {{ dig "kserve-llmisvc-crd" "enabled" true .Values }} + + - name: kserve-llmisvc-resources + namespace: kserve + chart: oci://ghcr.io/kserve/charts/kserve-llmisvc-resources + version: "v0.19.0" + installed: {{ dig "kserve-llmisvc-resources" "enabled" true .Values }} + needs: + - agentgateway-system/agentgateway + - cert-manager/cert-manager + - kserve/kserve-llmisvc-crd + - thalamus/thalamus # install controller after thalamus, to avoid webhook timeouts + values: + - kserve: + llmisvc: + createGIECRDs: false # https://github.com/kserve/kserve/issues/5543#issuecomment-4875485608 + storage: + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "4" + memory: 16Gi + + - name: kserve-runtime-configs + namespace: kserve + chart: oci://ghcr.io/kserve/charts/kserve-runtime-configs + version: "v0.19.0" + installed: {{ dig "kserve-runtime-configs" "enabled" true .Values }} + needs: + - kserve/kserve-llmisvc-crd + values: + - kserve: + llmisvcConfigs: + enabled: true + - name: open-webui namespace: open-webui createNamespace: true @@ -113,5 +165,6 @@ releases: needs: - thalamus/thalamus-crds - agentgateway-system/agentgateway + - kserve/kserve-llmisvc-crd values: - {{ dig "thalamus" dict .Values | toYaml | nindent 10 }} diff --git a/helm/thalamus/ci/test-values.yaml b/helm/thalamus/ci/test-values.yaml deleted file mode 100644 index b31157c..0000000 --- a/helm/thalamus/ci/test-values.yaml +++ /dev/null @@ -1,9 +0,0 @@ -models: - - slug: test-model - model: org/test-model - accelerator: cpu - resources: - requests: - cpu: "1" - limits: - cpu: "1" diff --git a/helm/thalamus/templates/configs.yaml b/helm/thalamus/templates/configs.yaml new file mode 100644 index 0000000..b6eea07 --- /dev/null +++ b/helm/thalamus/templates/configs.yaml @@ -0,0 +1,9 @@ +{{- range $key, $config := .Values.configs }} +--- +apiVersion: serving.kserve.io/v1alpha2 +kind: LLMInferenceServiceConfig +metadata: + name: thalamus-{{ $key }} + namespace: {{ $.Release.Namespace }} +spec: {{ toYaml $config.spec | nindent 2 }} +{{- end }} diff --git a/helm/thalamus/templates/epp.yaml b/helm/thalamus/templates/epp.yaml deleted file mode 100644 index 5024210..0000000 --- a/helm/thalamus/templates/epp.yaml +++ /dev/null @@ -1,188 +0,0 @@ -# EPP (Endpoint Picker) resources are defined here directly rather than using the upstream -# inferencepool chart because that chart has a local file:// dependency on epplib which -# makes it unusable as a remote subchart. Additionally, the upstream chart creates one Helm -# release per model, which requires helmfile-style dynamic releases — not possible in a -# static Helm chart. These templates are adapted from: -# https://github.com/kubernetes-sigs/gateway-api-inference-extension/tree/main/config/charts/epplib -# When upgrading the EPP image version, diff _deployment.yaml and _config.yaml against this file. -{{- $accelMap := .Values.accelerators }} -{{- range .Values.models }} -{{- $m := . }} -{{- $slug := $m.slug }} -{{- $epp := printf "vllm-%s-epp" $slug }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ $epp }} - namespace: {{ $.Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ $epp }} - namespace: {{ $.Release.Namespace }} -rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "watch", "list"] - - apiGroups: ["inference.networking.k8s.io"] - resources: ["inferencepools"] - verbs: ["get", "watch", "list"] - - apiGroups: ["inference.networking.x-k8s.io"] - resources: ["inferencemodelrewrites", "inferenceobjectives"] - verbs: ["get", "watch", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ $epp }} - namespace: {{ $.Release.Namespace }} -subjects: - - kind: ServiceAccount - name: {{ $epp }} - namespace: {{ $.Release.Namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ $epp }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ $epp }} - namespace: {{ $.Release.Namespace }} -data: - default-plugins.yaml: | - apiVersion: inference.networking.x-k8s.io/v1alpha1 - kind: EndpointPickerConfig - {{- if $.Values.epp.flowControl.enabled }} - featureGates: - - flowControl - {{- end }} - plugins: - - type: queue-scorer - - type: kv-cache-utilization-scorer - - type: prefix-cache-scorer - - type: metrics-data-source - parameters: - scheme: "http" - path: "/metrics" - insecureSkipVerify: true - {{- if $.Values.epp.flowControl.enabled }} - {{- with $.Values.epp.flowControl.saturationDetectorPlugin }} - - {{ toYaml . | nindent 6 | trim }} - {{- end }} - {{- end }} - schedulingProfiles: - - name: default - plugins: - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - pluginRef: prefix-cache-scorer - weight: 3 - {{- if $.Values.epp.flowControl.enabled }} - {{- $fc := omit $.Values.epp.flowControl "enabled" "saturationDetectorPlugin" }} - {{- with $.Values.epp.flowControl.saturationDetectorPlugin }} - {{- $fc = merge $fc (dict "saturationDetector" (dict "pluginRef" .type)) }} - {{- end }} - flowControl: -{{- $fc | toYaml | nindent 6 }} - {{- end }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ $epp }} - namespace: {{ $.Release.Namespace }} - labels: - inferencepool: {{ $epp }} - inference.networking.k8s.io/igw-mode: inferencepool -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - inferencepool: {{ $epp }} - template: - metadata: - labels: - inferencepool: {{ $epp }} - inference.networking.k8s.io/igw-mode: inferencepool - spec: - serviceAccountName: {{ $epp }} - terminationGracePeriodSeconds: 130 - containers: - - name: epp - image: {{ include "thalamus.image" $.Values.epp.image }} - imagePullPolicy: {{ $.Values.epp.image.pullPolicy }} - args: - - --pool-name - - vllm-{{ $slug }} - - --pool-namespace - - {{ $.Release.Namespace }} - - --pool-group - - inference.networking.k8s.io - - --zap-encoder - - json - - --config-file - - /config/default-plugins.yaml - - --metrics-endpoint-auth=false - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - resources: {{ $.Values.epp.resources | toYaml | nindent 12 }} - ports: - - name: grpc - containerPort: 9002 - - name: grpc-health - containerPort: 9003 - - name: metrics - containerPort: 9090 - livenessProbe: - grpc: - port: 9003 - service: inference-extension - initialDelaySeconds: 5 - periodSeconds: 10 - readinessProbe: - grpc: - port: 9003 - service: inference-extension - periodSeconds: 2 - volumeMounts: - - name: plugins-config-volume - mountPath: /config - volumes: - - name: plugins-config-volume - configMap: - name: {{ $epp }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ $epp }} - namespace: {{ $.Release.Namespace }} - labels: - inferencepool: {{ $epp }} -spec: - selector: - inferencepool: {{ $epp }} - ports: - - name: grpc-ext-proc - protocol: TCP - port: 9002 - - name: http-metrics - protocol: TCP - port: 9090 - type: ClusterIP -{{- end }} diff --git a/helm/thalamus/templates/gateway.yaml b/helm/thalamus/templates/gateway.yaml index a93f7d1..4485036 100644 --- a/helm/thalamus/templates/gateway.yaml +++ b/helm/thalamus/templates/gateway.yaml @@ -37,6 +37,6 @@ spec: kind: Gateway name: {{ $.Values.gateway.name }} {{- end }} - {{- toYaml (omit $policy "enabled" "targetRefs") | nindent 2 }} + {{- tpl (toYaml (omit $policy "enabled" "targetRefs")) $ | nindent 2 }} {{- end }} {{- end }} diff --git a/helm/thalamus/templates/httproute.yaml b/helm/thalamus/templates/httproute.yaml index ab3f2d7..6009974 100644 --- a/helm/thalamus/templates/httproute.yaml +++ b/helm/thalamus/templates/httproute.yaml @@ -19,25 +19,6 @@ spec: - path: type: Exact value: /v1/models -{{- if .Values.gateway.apiKeyAuth }} ---- -apiVersion: agentgateway.dev/v1alpha1 -kind: AgentgatewayPolicy -metadata: - name: apikey-auth-aggregated-models - namespace: {{ .Release.Namespace }} -spec: - targetRefs: - - group: gateway.networking.k8s.io - kind: HTTPRoute - name: aggregated-models - traffic: - apiKeyAuthentication: - mode: Strict - secretSelector: - matchLabels: - thalamus-apikey: "true" -{{- end }} --- apiVersion: agentgateway.dev/v1alpha1 kind: AgentgatewayPolicy @@ -58,9 +39,9 @@ spec: "data": [ {{- range $i, $m := .Values.models }} { - "id": {{ $m.model | quote }}, + "id": {{ $m.name | quote }}, "object": "model", - "owned_by": {{ $m.model | splitList "/" | first | quote }} + "owned_by": {{ $m.name | splitList "/" | first | quote }} }{{ if lt (add1 $i) (len $.Values.models) }},{{ end }} {{- end }} ] diff --git a/helm/thalamus/templates/llmisvc.yaml b/helm/thalamus/templates/llmisvc.yaml new file mode 100644 index 0000000..edad8b8 --- /dev/null +++ b/helm/thalamus/templates/llmisvc.yaml @@ -0,0 +1,41 @@ +{{- range .Values.models }} +{{- $m := . }} +--- +apiVersion: serving.kserve.io/v1alpha2 +kind: LLMInferenceService +metadata: + name: {{ trimSuffix "-" (trunc 63 (regexReplaceAll "[^a-z0-9-]+" (lower $m.name) "-")) }} + namespace: {{ $.Release.Namespace }} +spec: + baseRefs: + - name: kserve-config-llm-template + - name: kserve-config-llm-scheduler + - name: kserve-config-llm-router-route + {{- if hasKey $.Values.configs "image-overrides" }} + - name: thalamus-image-overrides + {{- end }} + {{- if hasKey $.Values.configs "non-root" }} + - name: thalamus-non-root + {{- end }} + - name: thalamus-{{ required "model.config is required" $m.config }} + model: + uri: hf://{{ $m.name }} + name: {{ $m.name | quote }} + {{- if ne $m.replicas nil }} + replicas: {{ $m.replicas }} + {{- end }} + router: + gateway: + refs: + - name: {{ $.Values.gateway.name }} + namespace: {{ $.Release.Namespace }} + template: + containers: + - name: main + {{- with $m.args }} + args: {{ toYaml . | nindent 10 }} + {{- end }} + {{- with $m.resources }} + resources: {{ toYaml . | nindent 10 }} + {{- end }} +{{- end }} diff --git a/helm/thalamus/templates/models.yaml b/helm/thalamus/templates/models.yaml deleted file mode 100644 index 4146f66..0000000 --- a/helm/thalamus/templates/models.yaml +++ /dev/null @@ -1,175 +0,0 @@ -{{- $accelMap := .Values.accelerators }} -{{- range .Values.models }} -{{- $m := . }} -{{- $accel := index $accelMap $m.accelerator | default dict }} -{{- $replicas := $m.replicas | default 1 }} -{{- $args := concat ($accel.baseArgs | default list) ($m.extraArgs | default list) }} -{{- $env := concat ($accel.baseEnv | default list) ($m.extraEnv | default list) }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: vllm-{{ $m.slug }} - namespace: {{ $.Release.Namespace }} - labels: - app: vllm-{{ $m.slug }} - llm-d.ai/inference-serving: "true" -spec: - replicas: {{ $replicas }} - strategy: - type: {{ $m.updateStrategy | default "RollingUpdate" }} - selector: - matchLabels: - app: vllm-{{ $m.slug }} - llm-d.ai/inference-serving: "true" - template: - metadata: - labels: - app: vllm-{{ $m.slug }} - llm-d.ai/inference-serving: "true" - spec: - containers: - - name: vllm - image: {{ include "thalamus.image" $accel.image }} - imagePullPolicy: {{ $accel.image.pullPolicy }} - command: - - vllm - - serve - args: - - {{ $m.model | quote }} - - "--port=8000" - - "--served-model-name" - - {{ $m.model | quote }} - - "--disable-uvicorn-access-log" - {{- range $args }} - - {{ . | quote }} - {{- end }} - env: - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token - key: HF_TOKEN - {{- range $env }} - - {{ toYaml . | indent 14 | trim }} - {{- end }} - {{- with $m.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- with $accel.securityContext }} - securityContext: {{ toYaml . | nindent 12 }} - {{- end }} - ports: - - containerPort: 8000 - name: vllm - protocol: TCP - startupProbe: - httpGet: - path: /v1/models - port: vllm - initialDelaySeconds: 15 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 60 - livenessProbe: - httpGet: - path: /health - port: vllm - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /v1/models - port: vllm - periodSeconds: 5 - timeoutSeconds: 2 - failureThreshold: 3 - volumeMounts: - - name: vllm-cache - mountPath: /root/.cache - - name: dshm - mountPath: /dev/shm - volumes: - - name: vllm-cache - emptyDir: {} - - name: dshm - emptyDir: - medium: Memory ---- -apiVersion: v1 -kind: Service -metadata: - name: vllm-{{ $m.slug }} - namespace: {{ $.Release.Namespace }} - labels: - app: vllm-{{ $m.slug }} -spec: - selector: - app: vllm-{{ $m.slug }} - ports: - - port: 8000 - targetPort: vllm - name: vllm ---- -apiVersion: inference.networking.k8s.io/v1 -kind: InferencePool -metadata: - name: vllm-{{ $m.slug }} - namespace: {{ $.Release.Namespace }} -spec: - targetPorts: - - number: 8000 - selector: - matchLabels: - app: vllm-{{ $m.slug }} - endpointPickerRef: - name: vllm-{{ $m.slug }}-epp - port: - number: 9002 ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: vllm-{{ $m.slug }} - namespace: {{ $.Release.Namespace }} -spec: - parentRefs: - {{- if $.Values.defaultRoute.parentRefs }} - {{- toYaml $.Values.defaultRoute.parentRefs | nindent 4 }} - {{- else }} - - name: {{ $.Values.gateway.name }} - {{- end }} - {{- with $.Values.defaultRoute.hostnames }} - hostnames: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - backendRefs: - - group: inference.networking.k8s.io - kind: InferencePool - name: vllm-{{ $m.slug }} - matches: - - headers: - - name: X-Gateway-Base-Model-Name - value: {{ $m.model }} -{{- if $.Values.gateway.apiKeyAuth }} ---- -apiVersion: agentgateway.dev/v1alpha1 -kind: AgentgatewayPolicy -metadata: - name: apikey-auth-vllm-{{ $m.slug }} - namespace: {{ $.Release.Namespace }} -spec: - targetRefs: - - group: gateway.networking.k8s.io - kind: HTTPRoute - name: vllm-{{ $m.slug }} - traffic: - apiKeyAuthentication: - mode: Strict - secretSelector: - matchLabels: - thalamus-apikey: "true" -{{- end }} -{{- end }} diff --git a/helm/thalamus/values.yaml b/helm/thalamus/values.yaml index 823eeb1..6ca0a99 100644 --- a/helm/thalamus/values.yaml +++ b/helm/thalamus/values.yaml @@ -66,63 +66,88 @@ gateway: # type: LoadBalancer # loadBalancerIP: "" - # Set to true to enable API key authentication. - # Secrets must be labelled `thalamus-apikey: "true"`. - apiKeyAuth: false - policies: bbr: - # enabled: false + enabled: true traffic: phase: PreRouting transformation: request: set: - - name: X-Gateway-Base-Model-Name - value: json(request.body).model + - name: X-Gateway-Model-Name + value: '"publishers/{{ .Release.Namespace }}/models/" + json(request.body).model' + + apikey-auth: + enabled: false + traffic: + apiKeyAuthentication: + mode: Strict + secretSelector: + matchLabels: + thalamus-apikey: "true" + +configs: + image-overrides: + spec: + router: + scheduler: + template: + containers: + - name: main + image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0 + - name: tokenizer + image: ghcr.io/llm-d/llm-d-uds-tokenizer:v0.9.0 + + non-root: + spec: + template: + containers: + - name: main + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + seccompProfile: + type: RuntimeDefault + capabilities: + drop: [ALL] -# Accelerator profiles -accelerators: - cpu: - image: - registry: docker.io - repository: vllm/vllm-openai-cpu - tag: v0.24.0 - digest: "" - pullPolicy: IfNotPresent - baseArgs: [] - baseEnv: [] - securityContext: - seccompProfile: - type: Unconfined - capabilities: - add: - - SYS_NICE nvidia: - image: - registry: docker.io - repository: vllm/vllm-openai - tag: v0.24.0 - digest: "" - pullPolicy: IfNotPresent - baseArgs: - - "--gpu-memory-utilization=0.95" - baseEnv: - - name: NCCL_P2P_LEVEL - value: "PIX" - - name: NCCL_IB_DISABLE - value: "1" - securityContext: {} + spec: + template: + containers: + - name: main + image: docker.io/vllm/vllm-openai:v0.24.0 + env: + - name: NCCL_IB_DISABLE + value: "1" + cpu: + spec: + template: + containers: + - name: main + image: docker.io/vllm/vllm-openai-cpu:v0.24.0 + securityContext: + seccompProfile: + type: Unconfined + capabilities: + add: + - SYS_NICE -# List of models to deploy. Each entry produces a model Deployment, Service, and InferencePool. +# List of models to deploy. Each entry produces one LLMInferenceService +# (which the KServe controller expands into a workload Deployment, EPP, +# InferencePool, and HTTPRoute using the llm-d presets composed with the +# selected config plus the always-on `image-overrides`). # Example: # models: -# - slug: my-model -# model: org/my-model -# accelerator: nvidia -# replicas: 1 -# extraArgs: [] -# extraEnv: [] +# - name: org/my-model # HuggingFace model id; slug is auto-derived +# config: nvidia # which config from `.Values.configs` to compose +# replicas: 1 # optional; defaults to 1 +# args: # optional; passed as container args ($@ in the preset) +# - "--tensor-parallel-size=2" # resources: # requests: # nvidia.com/gpu: "1" @@ -130,27 +155,6 @@ accelerators: # nvidia.com/gpu: "1" models: [] -# Endpoint Picker (EPP) -epp: - image: - registry: ghcr.io - repository: llm-d/llm-d-router-endpoint-picker - tag: v0.9.2 - digest: "" - pullPolicy: IfNotPresent - resources: - requests: - cpu: "1" - memory: 1Gi - limits: - memory: 2Gi - flowControl: - enabled: false - # saturationDetector: - # pluginRef: concurrency-detector # defaults to utilization-detector - # saturationDetectorPlugin: - # type: concurrency-detector - extraDeploy: [] # - apiVersion: v1 # kind: ConfigMap