feat(runtime): close out the SGLang kitchen-sink (#1060): minor flags, accept thresholds, typed LoRA adapters, LoRAAdapter CRD

api/v1alpha1/inferenceservice_types.go

@@ -1092,6 +1092,15 @@ type SGLangConfig struct {
 
 	// DataParallelSize sets the number of data-parallel replicas (SGLang-side controller).
 	// Maps to SGLang --dp flag. Not auto-derived; set explicitly.
+	//
+	// NOTE: at present this only sets the in-process SGLang `--dp` flag.
+	// Multi-replica rendezvous (SGLang's --dist-init-addr + a stable
+	// network identity per pod, e.g. headless service + StatefulSet) is
+	// not yet wired into the InferenceService controller and is tracked
+	// at https://github.com/defilantech/LLMKube/issues/1102. Setting
+	// this on an InferenceService with replicas > 1 will leave each
+	// replica starting as its own DP-1 group; operators wanting true
+	// DP coordination should hold off on this flag until #1102 lands.
 	// +kubebuilder:validation:Minimum=1
 	// +optional
 	DataParallelSize *int32 `json:"dataParallelSize,omitempty"`
@@ -1183,8 +1192,12 @@ type SGLangConfig struct {
 	Speculative *SGLangSpeculativeConfig `json:"speculative,omitempty"`
 
 	// LoRA (basic)
-	// LoraModules is a JSON array of LoRA specs (model_id -> path). Maps to
-	// SGLang --lora-modules flag.
+	// LoraModules is the legacy form of --lora-paths entries. Each
+	// element is either `name=path` shorthand or a JSON object
+	// {"name":"x","path":"/p"}. New callers should prefer the typed
+	// LoraAdapters field; the controller merges both, with
+	// LoraAdapters winning on name collision. Deprecated: use
+	// LoraAdapters instead.
 	// +optional
 	LoraModules []string `json:"loraModules,omitempty"`
 
@@ -1199,12 +1212,60 @@ type SGLangConfig struct {
 	// +optional
 	LoraTargetModules []string `json:"loraTargetModules,omitempty"`
 
+	// LoraAdapters is a typed replacement for LoraModules. Each adapter has
+	// a stable Name (SGLang-side handle) and Path (file mount). When both
+	// LoraAdapters and LoraModules are set, LoraAdapters wins on name
+	// collision. Maps to SGLang --lora-paths flag (singular `lora_paths`,
+	// not vLLM's --lora-modules — see
+	// https://github.com/sgl-project/sglang/blob/v0.5.15/python/sglang/srt/server_args.py).
+	// +optional
+	LoraAdapters []SGLangLoRAAdapter `json:"loraAdapters,omitempty"`
+
+	// LogLevel sets the SGLang server log level. SGLang accepts
+	// "debug"/"info"/"warning"/"error". Maps to SGLang --log-level flag.
+	// +kubebuilder:validation:Enum=debug;info;warning;error
+	// +optional
+	LogLevel string `json:"logLevel,omitempty"`
+
+	// TrustRemoteCode allows loading remote code from the HuggingFace Hub
+	// model repo. Mirrors the flag on other runtimes. Maps to SGLang
+	// --trust-remote-code flag. Omit to leave SGLang's default.
+	// +optional
+	TrustRemoteCode *bool `json:"trustRemoteCode,omitempty"`
+
+	// SkipTokenizerInit skips tokenizer initialization at startup. Useful
+	// for prefill-only disaggregation deployments. Maps to SGLang
+	// --skip-tokenizer-init flag. Omit to leave SGLang's default.
+	// +optional
+	SkipTokenizerInit *bool `json:"skipTokenizerInit,omitempty"`
+
 	// HFTokenSecretRef references a Secret containing the HuggingFace token.
 	// Injected as HF_TOKEN env var.
 	// +optional
 	HFTokenSecretRef *corev1.SecretKeySelector `json:"hfTokenSecretRef,omitempty"`
 }
 
+// SGLangLoRAAdapter names a single LoRA adapter for SGLang's --lora-paths
+// flag (NOT vLLM's --lora-modules — see
+// https://github.com/sgl-project/sglang/blob/v0.5.15/python/sglang/srt/server_args.py).
+// Name is the SGLang-side adapter handle; Path is the file mount where
+// adapter weights live (typically backed by a PVC created via
+// LoRAAdapter resources). Prefer this typed shape over the legacy
+// LoraModules []string; both are merged with the typed form winning on name
+// collision.
+type SGLangLoRAAdapter struct {
+	// Name is the SGLang-side adapter handle used in inference requests.
+	// +kubebuilder:validation:MinLength=1
+	// +required
+	Name string `json:"name"`
+
+	// Path is the path on disk inside the SGLang container where the
+	// adapter weights are mounted.
+	// +kubebuilder:validation:MinLength=1
+	// +required
+	Path string `json:"path"`
+}
+
 // SGLangSpeculativeConfig configures speculative decoding for SGLang.
 type SGLangSpeculativeConfig struct {
 	// Enabled toggles speculative decoding on. When false or nil, no flags
@@ -1241,6 +1302,26 @@ type SGLangSpeculativeConfig struct {
 	// +kubebuilder:default=4
 	// +optional
 	NumDraftTokens *int32 `json:"numDraftTokens,omitempty"`
+
+	// AcceptThresholdSingle sets the acceptance threshold for non-matched
+	// tokens in single-sequence decoding (a draft token is accepted when
+	// its probability exceeds p * accept_threshold_single). Valid only
+	// when Enabled is true; surface a status condition when set otherwise.
+	// Maps to SGLang --speculative-accept-threshold-single flag.
+	// +kubebuilder:validation:Minimum=0.0
+	// +kubebuilder:validation:Maximum=1.0
+	// +optional
+	AcceptThresholdSingle *float64 `json:"acceptThresholdSingle,omitempty"`
+
+	// AcceptThresholdAcc sets the acceptance threshold for the bonus token
+	// in accepted-token-sequence verification (an accepted draft token's
+	// probability must exceed p * accept_threshold_acc). Valid only when
+	// Enabled is true; surface a status condition when set otherwise.
+	// Maps to SGLang --speculative-accept-threshold-acc flag.
+	// +kubebuilder:validation:Minimum=0.0
+	// +kubebuilder:validation:Maximum=1.0
+	// +optional
+	AcceptThresholdAcc *float64 `json:"acceptThresholdAcc,omitempty"`
 }
 
 func init() {

api/v1alpha1/loraadapter_types.go

@@ -0,0 +1,145 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// LoRAAdapterSpec defines a single SGLang LoRA adapter that should be
+// loaded into a target InferenceService without restarting its pod. The
+// controller translates this resource into a POST against SGLang's
+// /load_lora_adapter HTTP endpoint (singular `lora_adapter`, no /v1
+// prefix, as of SGLang v0.5.15) and a finalizer-driven
+// /unload_lora_adapter on delete. See
+// https://github.com/sgl-project/sglang/blob/v0.5.15/python/sglang/srt/entrypoints/http_server.py
+// for the wire format.
+type LoRAAdapterSpec struct {
+	// InferenceServiceRef names the InferenceService this adapter should
+	// be loaded into. The reconciler resolves it; if it points at a
+	// non-sglang runtime, Available=False with reason RuntimeMismatch.
+	// +kubebuilder:validation:Required
+	InferenceServiceRef LocalInferenceServiceReference `json:"inferenceServiceRef"`
+
+	// Name is the SGLang-side adapter handle. Operators quote this in
+	// inference requests to pick the adapter. Must be unique within the
+	// target InferenceService's loaded set; the SGLang HTTP API rejects
+	// duplicates with 409.
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=63
+	// +kubebuilder:validation:Pattern=`^[A-Za-z0-9][A-Za-z0-9._-]*$`
+	// +required
+	Name string `json:"name"`
+
+	// Path is the path on disk inside the SGLang container where the
+	// adapter weights are mounted. SGLang reads from this path when
+	// /load_lora_adapter is invoked, so it must be reachable from the
+	// inference pod (typically via a PVC exposed in
+	// InferenceService.spec.extraVolumes) at the path the operator
+	// chose. The controller does not auto-mount.
+	// +kubebuilder:validation:MinLength=1
+	// +required
+	Path string `json:"path"`
+}
+
+// LocalInferenceServiceReference is a namespaced pointer at an
+// InferenceService. Kept narrow so it can never accidentally refer to a
+// cluster-scoped resource or hang a cross-cluster reference.
+type LocalInferenceServiceReference struct {
+	// Name is the name of the target InferenceService. Must exist in
+	// Namespace (or be omitted to default to this resource's namespace).
+	// +kubebuilder:validation:Required
+	// +required
+	Name string `json:"name"`
+
+	// Namespace is the namespace of the target InferenceService. When
+	// omitted, defaults to the LoRAAdapter's namespace. Cross-namespace
+	// references are allowed; the controller only requires that the
+	// operator has RBAC to GET the referenced InferenceService.
+	// +optional
+	Namespace *string `json:"namespace,omitempty"`
+}
+
+// LoRAAdapterStatus reports observed load state for an adapter.
+type LoRAAdapterStatus struct {
+	// LoadedPath is the path SGLang has accepted for this adapter. Set
+	// after a successful POST /load_lora_adapter response. Empty before
+	// the first successful load.
+	// +optional
+	LoadedPath string `json:"loadedPath,omitempty"`
+
+	// LastLoadedAt is when the adapter was last successfully loaded
+	// into SGLang. Re-loads (after a SGLang pod restart, for example)
+	// update this field.
+	// +optional
+	LastLoadedAt *metav1.Time `json:"lastLoadedAt,omitempty"`
+
+	// conditions represent the current state of the LoRAAdapter
+	// resource.
+	//
+	// Standard condition types include:
+	// - "Available": the spec is well-formed and target InferenceService
+	//   exists with runtime == sglang.
+	// - "Loaded": the adapter has been POSTed to SGLang and SGLang
+	//   reported success (or, on teardown, acknowledged the unload).
+	// - "Error": a reconcile step failed; the message carries the
+	//   underlying reason (runtime mismatch, HTTP failure, etc.).
+	//
+	// The status of each condition is one of True, False, or Unknown.
+	// +listType=map
+	// +listMapKey=type
+	// +optional
+	Conditions []metav1.Condition `json:"conditions,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Service",type=string,JSONPath=`.spec.inferenceServiceRef.name`
+// +kubebuilder:printcolumn:name="Adapter",type=string,JSONPath=`.spec.name`
+// +kubebuilder:printcolumn:name="Loaded",type=string,JSONPath=`.status.conditions[?(@.type=="Loaded")].status`
+// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
+// +kubebuilder:resource:shortName=lora
+
+// LoRAAdapter is the Schema for the loraadapters API.
+type LoRAAdapter struct {
+	metav1.TypeMeta `json:",inline"`
+
+	// metadata is a standard object metadata
+	// +optional
+	metav1.ObjectMeta `json:"metadata,omitempty,omitzero"`
+
+	// spec defines the desired state of LoRAAdapter
+	// +required
+	Spec LoRAAdapterSpec `json:"spec"`
+
+	// status defines the observed state of LoRAAdapter
+	// +optional
+	Status LoRAAdapterStatus `json:"status,omitempty,omitzero"`
+}
+
+// +kubebuilder:object:root=true
+
+// LoRAAdapterList contains a list of LoRAAdapter.
+type LoRAAdapterList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []LoRAAdapter `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&LoRAAdapter{}, &LoRAAdapterList{})
+}