From 84b2f16cf2dc562acb3c665831f062c7d9a4d998 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:03:55 +0000 Subject: [PATCH 1/2] feat(config): add context compression policy Define the opt-in model configuration contract before the chat middleware consumes it. Document each policy field so later request handling does not invent a second schema.\n\nRefs #9534\n\nAssisted-by: Codex:gpt-5 --- core/config/model_config.go | 13 ++++++++ core/config/model_config_test.go | 24 ++++++++++++++ docs/content/features/context-compression.md | 35 ++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 docs/content/features/context-compression.md diff --git a/core/config/model_config.go b/core/config/model_config.go index 9a43bd539791..10683f121477 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -80,6 +80,7 @@ type ModelConfig struct { FunctionsConfig functions.FunctionsConfig `yaml:"function,omitempty" json:"function,omitempty"` ReasoningConfig reasoning.Config `yaml:"reasoning,omitempty" json:"reasoning,omitempty"` + Compression CompressionConfig `yaml:"compression,omitempty" json:"compression,omitempty"` // ReasoningEffort is the default reasoning effort (none|minimal|low|medium|high) // for this model. A per-request reasoning_effort overrides it. It is forwarded @@ -148,6 +149,18 @@ type ModelConfig struct { Limits LimitsConfig `yaml:"limits,omitempty" json:"limits,omitempty"` } +// CompressionConfig controls opt-in compression of chat history before inference. +// The request middleware consumes this configuration; keeping it on ModelConfig +// lets operators select a policy per context window and model workload. +type CompressionConfig struct { + Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + TriggerAtRatio float64 `yaml:"trigger_at_ratio,omitempty" json:"trigger_at_ratio,omitempty"` + KeepTailTokens int `yaml:"keep_tail_tokens,omitempty" json:"keep_tail_tokens,omitempty"` + MaxSummaryTokens int `yaml:"max_summary_tokens,omitempty" json:"max_summary_tokens,omitempty"` + CompressorModel string `yaml:"compressor_model,omitempty" json:"compressor_model,omitempty"` + OnPostCompressionOverflow string `yaml:"on_post_compression_overflow,omitempty" json:"on_post_compression_overflow,omitempty"` +} + // @Description Admission-control limits applied per request. The // admission middleware enforces these before invoking the handler; // requests that exceed a limit get 503 with a Retry-After hint so diff --git a/core/config/model_config_test.go b/core/config/model_config_test.go index 21741a061ce9..6e920ab55fb2 100644 --- a/core/config/model_config_test.go +++ b/core/config/model_config_test.go @@ -52,6 +52,30 @@ parameters: Expect(valid).To(BeTrue()) }) + It("round-trips context compression settings", func() { + raw := []byte(` +name: compressed-chat +backend: llama-cpp +parameters: + model: chat.gguf +compression: + enabled: true + trigger_at_ratio: 0.75 + keep_tail_tokens: 8000 + max_summary_tokens: 2048 + compressor_model: fast-summarizer + on_post_compression_overflow: error +`) + var cfg ModelConfig + Expect(yaml.Unmarshal(raw, &cfg)).To(Succeed()) + Expect(cfg.Compression.Enabled).To(BeTrue()) + Expect(cfg.Compression.TriggerAtRatio).To(Equal(0.75)) + Expect(cfg.Compression.KeepTailTokens).To(Equal(8000)) + Expect(cfg.Compression.MaxSummaryTokens).To(Equal(2048)) + Expect(cfg.Compression.CompressorModel).To(Equal("fast-summarizer")) + Expect(cfg.Compression.OnPostCompressionOverflow).To(Equal("error")) + }) + It("derives a managed snapshot filename without replacing the logical model", func() { const cacheKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" cfg := ModelConfig{ diff --git a/docs/content/features/context-compression.md b/docs/content/features/context-compression.md new file mode 100644 index 000000000000..853468ff94f4 --- /dev/null +++ b/docs/content/features/context-compression.md @@ -0,0 +1,35 @@ +--- +title: "Context compression" +description: "Configure automatic compression for long chat histories" +--- + +Context compression is an opt-in, per-model policy for chat requests that approach +the model context limit. The configuration is disabled by default and does not +change existing requests unless `enabled` is true. + +```yaml +name: long-context-chat +backend: llama-cpp +parameters: + model: chat-model.gguf +compression: + enabled: true + trigger_at_ratio: 0.75 + keep_tail_tokens: 8000 + max_summary_tokens: 2048 + compressor_model: fast-summarizer + on_post_compression_overflow: drop_oldest_summary +``` + +The fields reserve the model-level contract used by the chat request compressor: + +- `trigger_at_ratio` selects the fraction of `context_size` that starts compression. +- `keep_tail_tokens` protects the newest part of the conversation from compression. +- `max_summary_tokens` limits the generated summary. +- `compressor_model` selects a secondary model. An empty value selects the primary model. +- `on_post_compression_overflow` selects `drop_oldest_summary` or `error` when the compressed request still exceeds the context limit. + +{{% notice warning %}} +The configuration schema is available, but the request-compression middleware is +still under development. Enabling it does not transform requests yet. +{{% /notice %}} From f57b1372a07f02a19c312d26f9f0f3d57d06b9aa Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Mon, 17 Aug 2026 00:07:17 +0000 Subject: [PATCH 2/2] fix(config): register compression fields The model editor metadata gate rejects new config fields without descriptions and suitable controls. Register the compression policy so operators can edit its six fields safely. Assisted-by: Codex:gpt-5 [monitoring-prs] --- core/config/meta/registry.go | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/core/config/meta/registry.go b/core/config/meta/registry.go index aa25e57ad892..a8131a1f0c2e 100644 --- a/core/config/meta/registry.go +++ b/core/config/meta/registry.go @@ -182,6 +182,64 @@ func DefaultRegistry() map[string]FieldMetaOverride { Advanced: true, Order: 22, }, + "compression.enabled": { + Section: "llm", + Label: "Context Compression", + Description: "Enable compression of chat history before it reaches the model context limit", + Component: "checkbox", + Advanced: true, + Order: 24, + }, + "compression.trigger_at_ratio": { + Section: "llm", + Label: "Compression Trigger Ratio", + Description: "Fraction of the model context window that starts compression", + Component: "slider", + Min: f64(0), + Max: f64(1), + Step: f64(0.05), + Advanced: true, + Order: 25, + }, + "compression.keep_tail_tokens": { + Section: "llm", + Label: "Compression Tail Tokens", + Description: "Number of newest conversation tokens to keep outside the summary", + Component: "number", + Min: f64(0), + Advanced: true, + Order: 26, + }, + "compression.max_summary_tokens": { + Section: "llm", + Label: "Maximum Summary Tokens", + Description: "Maximum number of tokens produced by context compression", + Component: "number", + Min: f64(0), + Advanced: true, + Order: 27, + }, + "compression.compressor_model": { + Section: "llm", + Label: "Compressor Model", + Description: "Chat model used to summarize context; empty uses this model", + Component: "model-select", + AutocompleteProvider: ProviderModelsChat, + Advanced: true, + Order: 28, + }, + "compression.on_post_compression_overflow": { + Section: "llm", + Label: "Post-compression Overflow", + Description: "Action to take when compressed context still exceeds the context limit", + Component: "select", + Options: []FieldOption{ + {Value: "drop_oldest_summary", Label: "Drop Oldest Summary"}, + {Value: "error", Label: "Return Error"}, + }, + Advanced: true, + Order: 29, + }, "cache_type_k": { Section: "llm", Label: "KV Cache Type (K)",