Fix infinite loop on high memory pressure.

src/memory.jl

@@ -17,6 +17,12 @@ const soft_frac = Ref{Float64}(0.80)
 const hard_frac = Ref{Float64}(0.90)
 const AUTO_GC_ENABLE = Ref{Bool}(false)
 
+# memory measured right after the last GC
+const post_gc_device_bytes = Atomic{Int64}(0)
+const post_gc_host_bytes = Atomic{Int64}(0)
+# how much new memory must accumulate before GC fires again
+const gc_hysteresis_frac = Ref{Float64}(0.05)
+
 @doc"""
     init_gc!()
 
@@ -90,15 +96,31 @@ end
 function maybe_collect()
     host_bytes = current_host_bytes[]
     device_bytes = current_device_bytes[]
-    if host_bytes > hard_limit() || device_bytes > hard_limit(; host=false)
-        # Aggressive
-        GC.gc(true)
-        recalibrate_allocator!()
-    elseif host_bytes > soft_limit() || device_bytes > soft_limit(; host=false)
-        # Gentle
-        GC.gc(false)
-        recalibrate_allocator!()
+
+    # minimum growth above the post-GC floor needed to re-collect
+    dev_floor = post_gc_device_bytes[]
+    host_floor = post_gc_host_bytes[]
+    dev_delta = Int(round(gc_hysteresis_frac[] * total_device_bytes[]))
+    host_delta = Int(round(gc_hysteresis_frac[] * total_host_bytes[]))
+    grew = device_bytes > dev_floor + dev_delta || host_bytes > host_floor + host_delta
+
+    if device_bytes > hard_limit(; host=false) || host_bytes > hard_limit()
+        grew && _collect!(true)
+    elseif device_bytes > soft_limit(; host=false) || host_bytes > soft_limit()
+        grew && _collect!(false)
+    else
+        # reset floors so the next spike is caught immediately
+        atomic_xchg!(post_gc_device_bytes, 0)
+        atomic_xchg!(post_gc_host_bytes, 0)
     end
 
     return nothing
 end
+
+function _collect!(full::Bool)
+    GC.gc(full)
+    recalibrate_allocator!()
+    atomic_xchg!(post_gc_device_bytes, current_device_bytes[])
+    atomic_xchg!(post_gc_host_bytes, current_host_bytes[])
+    return nothing
+end