Skip to content

Commit c844b7a

Browse files
mudlermudler
authored
feat: disable force eviction (#7725)
* feat: allow to set forcing backends eviction while requests are in flight Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat: try to make the request sit and retry if eviction couldn't be done Otherwise calls that in order to pass would need to shutdown other backends would just fail. In this way instead we make the request sit and retry eviction until it succeeds. The thresholds can be configured by the user. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * add tests Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * expose settings to CLI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Update docs Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent bb459e6 commit c844b7a

18 files changed

Lines changed: 739 additions & 41 deletions

‎core/application/config_file_watcher.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,9 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
214214
envAutoloadGalleries := appConfig.AutoloadGalleries == startupAppConfig.AutoloadGalleries
215215
envAutoloadBackendGalleries := appConfig.AutoloadBackendGalleries == startupAppConfig.AutoloadBackendGalleries
216216
envAgentJobRetentionDays := appConfig.AgentJobRetentionDays == startupAppConfig.AgentJobRetentionDays
217+
envForceEvictionWhenBusy := appConfig.ForceEvictionWhenBusy == startupAppConfig.ForceEvictionWhenBusy
218+
envLRUEvictionMaxRetries := appConfig.LRUEvictionMaxRetries == startupAppConfig.LRUEvictionMaxRetries
219+
envLRUEvictionRetryInterval := appConfig.LRUEvictionRetryInterval == startupAppConfig.LRUEvictionRetryInterval
217220

218221
if len(fileContent) > 0 {
219222
var settings config.RuntimeSettings
@@ -277,6 +280,20 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
277280
if settings.MemoryReclaimerThreshold != nil && !envMemoryReclaimerThreshold {
278281
appConfig.MemoryReclaimerThreshold = *settings.MemoryReclaimerThreshold
279282
}
283+
if settings.ForceEvictionWhenBusy != nil && !envForceEvictionWhenBusy {
284+
appConfig.ForceEvictionWhenBusy = *settings.ForceEvictionWhenBusy
285+
}
286+
if settings.LRUEvictionMaxRetries != nil && !envLRUEvictionMaxRetries {
287+
appConfig.LRUEvictionMaxRetries = *settings.LRUEvictionMaxRetries
288+
}
289+
if settings.LRUEvictionRetryInterval != nil && !envLRUEvictionRetryInterval {
290+
dur, err := time.ParseDuration(*settings.LRUEvictionRetryInterval)
291+
if err == nil {
292+
appConfig.LRUEvictionRetryInterval = dur
293+
} else {
294+
xlog.Warn("invalid LRU eviction retry interval in runtime_settings.json", "error", err, "interval", *settings.LRUEvictionRetryInterval)
295+
}
296+
}
280297
if settings.Threads != nil && !envThreads {
281298
appConfig.Threads = *settings.Threads
282299
}

‎core/application/startup.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,9 +350,16 @@ func initializeWatchdog(application *Application, options *config.ApplicationCon
350350
model.WithIdleCheck(options.WatchDogIdle),
351351
model.WithLRULimit(lruLimit),
352352
model.WithMemoryReclaimer(options.MemoryReclaimerEnabled, options.MemoryReclaimerThreshold),
353+
model.WithForceEvictionWhenBusy(options.ForceEvictionWhenBusy),
353354
)
354355
application.ModelLoader().SetWatchDog(wd)
355356

357+
// Initialize ModelLoader LRU eviction retry settings
358+
application.ModelLoader().SetLRUEvictionRetrySettings(
359+
options.LRUEvictionMaxRetries,
360+
options.LRUEvictionRetryInterval,
361+
)
362+
356363
// Start watchdog goroutine if any periodic checks are enabled
357364
// LRU eviction doesn't need the Run() loop - it's triggered on model load
358365
// But memory reclaimer needs the Run() loop for periodic checking

‎core/application/watchdog.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ func (a *Application) startWatchdog() error {
3535
model.WithIdleCheck(appConfig.WatchDogIdle),
3636
model.WithLRULimit(lruLimit),
3737
model.WithMemoryReclaimer(appConfig.MemoryReclaimerEnabled, appConfig.MemoryReclaimerThreshold),
38+
model.WithForceEvictionWhenBusy(appConfig.ForceEvictionWhenBusy),
3839
)
3940
a.modelLoader.SetWatchDog(wd)
4041

‎core/cli/run.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,9 @@ type RunCMD struct {
7373
WatchdogBusyTimeout string `env:"LOCALAI_WATCHDOG_BUSY_TIMEOUT,WATCHDOG_BUSY_TIMEOUT" default:"5m" help:"Threshold beyond which a busy backend should be stopped" group:"backends"`
7474
EnableMemoryReclaimer bool `env:"LOCALAI_MEMORY_RECLAIMER,MEMORY_RECLAIMER,LOCALAI_GPU_RECLAIMER,GPU_RECLAIMER" default:"false" help:"Enable memory threshold monitoring to auto-evict backends when memory usage exceeds threshold (uses GPU VRAM if available, otherwise RAM)" group:"backends"`
7575
MemoryReclaimerThreshold float64 `env:"LOCALAI_MEMORY_RECLAIMER_THRESHOLD,MEMORY_RECLAIMER_THRESHOLD,LOCALAI_GPU_RECLAIMER_THRESHOLD,GPU_RECLAIMER_THRESHOLD" default:"0.95" help:"Memory usage threshold (0.0-1.0) that triggers backend eviction (default 0.95 = 95%%)" group:"backends"`
76+
ForceEvictionWhenBusy bool `env:"LOCALAI_FORCE_EVICTION_WHEN_BUSY,FORCE_EVICTION_WHEN_BUSY" default:"false" help:"Force eviction even when models have active API calls (default: false for safety)" group:"backends"`
77+
LRUEvictionMaxRetries int `env:"LOCALAI_LRU_EVICTION_MAX_RETRIES,LRU_EVICTION_MAX_RETRIES" default:"30" help:"Maximum number of retries when waiting for busy models to become idle before eviction (default: 30)" group:"backends"`
78+
LRUEvictionRetryInterval string `env:"LOCALAI_LRU_EVICTION_RETRY_INTERVAL,LRU_EVICTION_RETRY_INTERVAL" default:"1s" help:"Interval between retries when waiting for busy models to become idle (e.g., 1s, 2s) (default: 1s)" group:"backends"`
7679
Federated bool `env:"LOCALAI_FEDERATED,FEDERATED" help:"Enable federated instance" group:"federated"`
7780
DisableGalleryEndpoint bool `env:"LOCALAI_DISABLE_GALLERY_ENDPOINT,DISABLE_GALLERY_ENDPOINT" help:"Disable the gallery endpoints" group:"api"`
7881
MachineTag string `env:"LOCALAI_MACHINE_TAG,MACHINE_TAG" help:"Add Machine-Tag header to each response which is useful to track the machine in the P2P network" group:"api"`
@@ -220,6 +223,21 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
220223
opts = append(opts, config.EnableSingleBackend)
221224
}
222225

226+
// Handle LRU eviction settings
227+
if r.ForceEvictionWhenBusy {
228+
opts = append(opts, config.WithForceEvictionWhenBusy(true))
229+
}
230+
if r.LRUEvictionMaxRetries > 0 {
231+
opts = append(opts, config.WithLRUEvictionMaxRetries(r.LRUEvictionMaxRetries))
232+
}
233+
if r.LRUEvictionRetryInterval != "" {
234+
dur, err := time.ParseDuration(r.LRUEvictionRetryInterval)
235+
if err != nil {
236+
return fmt.Errorf("invalid LRU eviction retry interval: %w", err)
237+
}
238+
opts = append(opts, config.WithLRUEvictionRetryInterval(dur))
239+
}
240+
223241
// split ":" to get backend name and the uri
224242
for _, v := range r.ExternalGRPCBackends {
225243
backend := v[:strings.IndexByte(v, ':')]

‎core/config/application_config.go‎

Lines changed: 61 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,11 @@ type ApplicationConfig struct {
6464
MemoryReclaimerEnabled bool // Enable memory threshold monitoring
6565
MemoryReclaimerThreshold float64 // Threshold 0.0-1.0 (e.g., 0.95 = 95%)
6666

67+
// Eviction settings
68+
ForceEvictionWhenBusy bool // Force eviction even when models have active API calls (default: false for safety)
69+
LRUEvictionMaxRetries int // Maximum number of retries when waiting for busy models to become idle (default: 30)
70+
LRUEvictionRetryInterval time.Duration // Interval between retries when waiting for busy models (default: 1s)
71+
6772
ModelsURL []string
6873

6974
WatchDogBusyTimeout, WatchDogIdleTimeout time.Duration
@@ -86,10 +91,12 @@ type AppOption func(*ApplicationConfig)
8691

8792
func NewApplicationConfig(o ...AppOption) *ApplicationConfig {
8893
opt := &ApplicationConfig{
89-
Context: context.Background(),
90-
UploadLimitMB: 15,
91-
Debug: true,
92-
AgentJobRetentionDays: 30, // Default: 30 days
94+
Context: context.Background(),
95+
UploadLimitMB: 15,
96+
Debug: true,
97+
AgentJobRetentionDays: 30, // Default: 30 days
98+
LRUEvictionMaxRetries: 30, // Default: 30 retries
99+
LRUEvictionRetryInterval: 1 * time.Second, // Default: 1 second
93100
PathWithoutAuth: []string{
94101
"/static/",
95102
"/generated-audio/",
@@ -259,6 +266,31 @@ func (o *ApplicationConfig) GetEffectiveMaxActiveBackends() int {
259266
return 0
260267
}
261268

269+
// WithForceEvictionWhenBusy sets whether to force eviction even when models have active API calls
270+
func WithForceEvictionWhenBusy(enabled bool) AppOption {
271+
return func(o *ApplicationConfig) {
272+
o.ForceEvictionWhenBusy = enabled
273+
}
274+
}
275+
276+
// WithLRUEvictionMaxRetries sets the maximum number of retries when waiting for busy models to become idle
277+
func WithLRUEvictionMaxRetries(maxRetries int) AppOption {
278+
return func(o *ApplicationConfig) {
279+
if maxRetries > 0 {
280+
o.LRUEvictionMaxRetries = maxRetries
281+
}
282+
}
283+
}
284+
285+
// WithLRUEvictionRetryInterval sets the interval between retries when waiting for busy models
286+
func WithLRUEvictionRetryInterval(interval time.Duration) AppOption {
287+
return func(o *ApplicationConfig) {
288+
if interval > 0 {
289+
o.LRUEvictionRetryInterval = interval
290+
}
291+
}
292+
}
293+
262294
var EnableParallelBackendRequests = func(o *ApplicationConfig) {
263295
o.ParallelBackendRequests = true
264296
}
@@ -505,6 +537,8 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings {
505537
parallelBackendRequests := o.ParallelBackendRequests
506538
memoryReclaimerEnabled := o.MemoryReclaimerEnabled
507539
memoryReclaimerThreshold := o.MemoryReclaimerThreshold
540+
forceEvictionWhenBusy := o.ForceEvictionWhenBusy
541+
lruEvictionMaxRetries := o.LRUEvictionMaxRetries
508542
threads := o.Threads
509543
contextSize := o.ContextSize
510544
f16 := o.F16
@@ -539,6 +573,12 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings {
539573
} else {
540574
watchdogInterval = "2s" // default
541575
}
576+
var lruEvictionRetryInterval string
577+
if o.LRUEvictionRetryInterval > 0 {
578+
lruEvictionRetryInterval = o.LRUEvictionRetryInterval.String()
579+
} else {
580+
lruEvictionRetryInterval = "1s" // default
581+
}
542582

543583
return RuntimeSettings{
544584
WatchdogEnabled: &watchdogEnabled,
@@ -552,6 +592,9 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings {
552592
ParallelBackendRequests: &parallelBackendRequests,
553593
MemoryReclaimerEnabled: &memoryReclaimerEnabled,
554594
MemoryReclaimerThreshold: &memoryReclaimerThreshold,
595+
ForceEvictionWhenBusy: &forceEvictionWhenBusy,
596+
LRUEvictionMaxRetries: &lruEvictionMaxRetries,
597+
LRUEvictionRetryInterval: &lruEvictionRetryInterval,
555598
Threads: &threads,
556599
ContextSize: &contextSize,
557600
F16: &f16,
@@ -644,6 +687,20 @@ func (o *ApplicationConfig) ApplyRuntimeSettings(settings *RuntimeSettings) (req
644687
requireRestart = true
645688
}
646689
}
690+
if settings.ForceEvictionWhenBusy != nil {
691+
o.ForceEvictionWhenBusy = *settings.ForceEvictionWhenBusy
692+
// This setting doesn't require restart, can be updated dynamically
693+
}
694+
if settings.LRUEvictionMaxRetries != nil {
695+
o.LRUEvictionMaxRetries = *settings.LRUEvictionMaxRetries
696+
// This setting doesn't require restart, can be updated dynamically
697+
}
698+
if settings.LRUEvictionRetryInterval != nil {
699+
if dur, err := time.ParseDuration(*settings.LRUEvictionRetryInterval); err == nil {
700+
o.LRUEvictionRetryInterval = dur
701+
// This setting doesn't require restart, can be updated dynamically
702+
}
703+
}
647704
if settings.Threads != nil {
648705
o.Threads = *settings.Threads
649706
}

‎core/config/runtime_settings.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,11 @@ type RuntimeSettings struct {
2626
MemoryReclaimerEnabled *bool `json:"memory_reclaimer_enabled,omitempty"` // Enable memory threshold monitoring
2727
MemoryReclaimerThreshold *float64 `json:"memory_reclaimer_threshold,omitempty"` // Threshold 0.0-1.0 (e.g., 0.95 = 95%)
2828

29+
// Eviction settings
30+
ForceEvictionWhenBusy *bool `json:"force_eviction_when_busy,omitempty"` // Force eviction even when models have active API calls (default: false for safety)
31+
LRUEvictionMaxRetries *int `json:"lru_eviction_max_retries,omitempty"` // Maximum number of retries when waiting for busy models to become idle (default: 30)
32+
LRUEvictionRetryInterval *string `json:"lru_eviction_retry_interval,omitempty"` // Interval between retries when waiting for busy models (e.g., 1s, 2s) (default: 1s)
33+
2934
// Performance settings
3035
Threads *int `json:"threads,omitempty"`
3136
ContextSize *int `json:"context_size,omitempty"`

‎core/http/endpoints/localai/settings.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,14 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
7676
})
7777
}
7878
}
79+
if settings.LRUEvictionRetryInterval != nil {
80+
if _, err := time.ParseDuration(*settings.LRUEvictionRetryInterval); err != nil {
81+
return c.JSON(http.StatusBadRequest, schema.SettingsResponse{
82+
Success: false,
83+
Error: "Invalid lru_eviction_retry_interval format: " + err.Error(),
84+
})
85+
}
86+
}
7987

8088
// Save to file
8189
if appConfig.DynamicConfigsDir == "" {
@@ -111,6 +119,31 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
111119
appConfig.ApiKeys = append(envKeys, runtimeKeys...)
112120
}
113121

122+
// Update watchdog dynamically for settings that don't require restart
123+
if settings.ForceEvictionWhenBusy != nil {
124+
currentWD := app.ModelLoader().GetWatchDog()
125+
if currentWD != nil {
126+
currentWD.SetForceEvictionWhenBusy(*settings.ForceEvictionWhenBusy)
127+
xlog.Info("Updated watchdog force eviction when busy setting", "forceEvictionWhenBusy", *settings.ForceEvictionWhenBusy)
128+
}
129+
}
130+
131+
// Update ModelLoader LRU eviction retry settings dynamically
132+
maxRetries := appConfig.LRUEvictionMaxRetries
133+
retryInterval := appConfig.LRUEvictionRetryInterval
134+
if settings.LRUEvictionMaxRetries != nil {
135+
maxRetries = *settings.LRUEvictionMaxRetries
136+
}
137+
if settings.LRUEvictionRetryInterval != nil {
138+
if dur, err := time.ParseDuration(*settings.LRUEvictionRetryInterval); err == nil {
139+
retryInterval = dur
140+
}
141+
}
142+
if settings.LRUEvictionMaxRetries != nil || settings.LRUEvictionRetryInterval != nil {
143+
app.ModelLoader().SetLRUEvictionRetrySettings(maxRetries, retryInterval)
144+
xlog.Info("Updated LRU eviction retry settings", "maxRetries", maxRetries, "retryInterval", retryInterval)
145+
}
146+
114147
// Check if agent job retention changed
115148
agentJobChanged := settings.AgentJobRetentionDays != nil
116149

‎core/http/views/settings.html‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,43 @@ <h2 class="text-xl font-semibold text-[var(--color-text-primary)] mb-4 flex item
136136
:class="!settings.watchdog_enabled ? 'opacity-50 cursor-not-allowed' : ''">
137137
</div>
138138

139+
<!-- Force Eviction When Busy -->
140+
<div class="flex items-center justify-between">
141+
<div>
142+
<label class="text-sm font-medium text-[var(--color-text-primary)]">Force Eviction When Busy</label>
143+
<p class="text-xs text-[var(--color-text-secondary)] mt-1">Allow evicting models even when they have active API calls (default: disabled for safety)</p>
144+
</div>
145+
<label class="relative inline-flex items-center cursor-pointer">
146+
<input type="checkbox" x-model="settings.force_eviction_when_busy"
147+
:disabled="!settings.watchdog_enabled"
148+
class="sr-only peer" :class="!settings.watchdog_enabled ? 'opacity-50' : ''">
149+
<div class="w-11 h-6 bg-[var(--color-bg-primary)] peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-[var(--color-primary-light)] rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-[var(--color-primary)]"></div>
150+
</label>
151+
</div>
152+
153+
<!-- LRU Eviction Max Retries -->
154+
<div>
155+
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">LRU Eviction Max Retries</label>
156+
<p class="text-xs text-[var(--color-text-secondary)] mb-2">Maximum number of retries when waiting for busy models to become idle (default: 30)</p>
157+
<input type="number" x-model="settings.lru_eviction_max_retries"
158+
:disabled="!settings.watchdog_enabled"
159+
min="1"
160+
placeholder="30"
161+
class="w-full px-3 py-2 bg-[var(--color-bg-primary)] border border-[var(--color-primary-border)]/20 rounded text-sm text-[var(--color-text-primary)] focus:outline-none focus:ring-2 focus:ring-[var(--color-primary-border)]"
162+
:class="!settings.watchdog_enabled ? 'opacity-50 cursor-not-allowed' : ''">
163+
</div>
164+
165+
<!-- LRU Eviction Retry Interval -->
166+
<div>
167+
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">LRU Eviction Retry Interval</label>
168+
<p class="text-xs text-[var(--color-text-secondary)] mb-2">Interval between retries when waiting for busy models (e.g., 1s, 2s) (default: 1s)</p>
169+
<input type="text" x-model="settings.lru_eviction_retry_interval"
170+
:disabled="!settings.watchdog_enabled"
171+
placeholder="1s"
172+
class="w-full px-3 py-2 bg-[var(--color-bg-primary)] border border-[var(--color-primary-border)]/20 rounded text-sm text-[var(--color-text-primary)] focus:outline-none focus:ring-2 focus:ring-[var(--color-primary-border)]"
173+
:class="!settings.watchdog_enabled ? 'opacity-50 cursor-not-allowed' : ''">
174+
</div>
175+
139176
<!-- Memory Reclaimer Subsection -->
140177
<div class="mt-6 pt-4 border-t border-[var(--color-primary-border)]/20">
141178
<h3 class="text-md font-medium text-[var(--color-text-primary)] mb-3 flex items-center">
@@ -545,6 +582,9 @@ <h2 class="text-xl font-semibold text-[var(--color-text-primary)] mb-4 flex item
545582
watchdog_idle_timeout: '15m',
546583
watchdog_busy_timeout: '5m',
547584
watchdog_interval: '2s',
585+
force_eviction_when_busy: false,
586+
lru_eviction_max_retries: 30,
587+
lru_eviction_retry_interval: '1s',
548588
max_active_backends: 0,
549589
parallel_backend_requests: false,
550590
memory_reclaimer_enabled: false,
@@ -587,6 +627,9 @@ <h2 class="text-xl font-semibold text-[var(--color-text-primary)] mb-4 flex item
587627
watchdog_idle_timeout: data.watchdog_idle_timeout || '15m',
588628
watchdog_busy_timeout: data.watchdog_busy_timeout || '5m',
589629
watchdog_interval: data.watchdog_interval || '2s',
630+
force_eviction_when_busy: data.force_eviction_when_busy || false,
631+
lru_eviction_max_retries: data.lru_eviction_max_retries || 30,
632+
lru_eviction_retry_interval: data.lru_eviction_retry_interval || '1s',
590633
max_active_backends: data.max_active_backends || 0,
591634
parallel_backend_requests: data.parallel_backend_requests,
592635
memory_reclaimer_enabled: data.memory_reclaimer_enabled || false,
@@ -660,6 +703,15 @@ <h2 class="text-xl font-semibold text-[var(--color-text-primary)] mb-4 flex item
660703
if (this.settings.watchdog_interval) {
661704
payload.watchdog_interval = this.settings.watchdog_interval;
662705
}
706+
if (this.settings.force_eviction_when_busy !== undefined) {
707+
payload.force_eviction_when_busy = this.settings.force_eviction_when_busy;
708+
}
709+
if (this.settings.lru_eviction_max_retries !== undefined) {
710+
payload.lru_eviction_max_retries = parseInt(this.settings.lru_eviction_max_retries) || 30;
711+
}
712+
if (this.settings.lru_eviction_retry_interval) {
713+
payload.lru_eviction_retry_interval = this.settings.lru_eviction_retry_interval;
714+
}
663715
if (this.settings.max_active_backends !== undefined) {
664716
payload.max_active_backends = parseInt(this.settings.max_active_backends) || 0;
665717
}

0 commit comments

Comments
 (0)