Spread hot key reads across replicas by randomising replica selection in Get and BulkGet

Author: cruzelx

load_test/locust.py

@@ -185,3 +185,55 @@ def bulk_get(self):
     @task(1)
     def health(self):
         self.client.get("/health", name="/health")
+
+
+# ---------------------------------------------------------------------------
+# Hot key user — hammers a single key to verify read load is spread across
+# replicas rather than always hitting the same aux node.
+# ---------------------------------------------------------------------------
+HOT_KEY = "hotkey:burn"
+HOT_VALUE = "hot-key-test-value"
+
+
+class HotKeyUser(HttpUser):
+    weight = 10
+    wait_time = between(0.01, 0.05)
+
+    def on_start(self):
+        """Seed the hot key once when this user starts."""
+        self.client.post(
+            "/data",
+            json={"key": HOT_KEY, "value": HOT_VALUE},
+            name="/data PUT (hot-key seed)",
+        )
+
+    @task(10)
+    def read_hot_key(self):
+        """Hammer a single key — all reads resolve to the same hash-ring slot."""
+        with self.client.get(
+            f"/data/{HOT_KEY}",
+            name="/data/[hotkey] GET",
+            catch_response=True,
+        ) as resp:
+            if resp.status_code == 200:
+                body = resp.json()
+                if body.get("value") != HOT_VALUE:
+                    resp.failure(
+                        f"Hot key value mismatch: got {body.get('value')!r}"
+                    )
+                else:
+                    resp.success()
+            elif resp.status_code == 404:
+                # May happen briefly after eviction under memory pressure
+                resp.success()
+            else:
+                resp.failure(f"Unexpected status {resp.status_code}")
+
+    @task(1)
+    def refresh_hot_key(self):
+        """Periodically re-seed so the key survives LRU eviction."""
+        self.client.post(
+            "/data",
+            json={"key": HOT_KEY, "value": HOT_VALUE},
+            name="/data PUT (hot-key refresh)",
+        )

master/controller.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math/rand"
 	"net/http"
 	"os"
 	"strconv"
@@ -188,7 +189,8 @@ func (m *Master) Get(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Try replicas in order; return on the first successful hit.
+	// Shuffle replicas so reads are spread across all replicas, not always hitting node[0].
+	rand.Shuffle(len(nodes), func(i, j int) { nodes[i], nodes[j] = nodes[j], nodes[i] })
 	for _, node := range nodes {
 		resp, err := m.client.Get(fmt.Sprintf("http://%s/data/%s", node, key))
 		if err != nil {
@@ -310,15 +312,16 @@ func (m *Master) BulkGet(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Group keys by primary node.
+	// Group keys by a randomly chosen replica so hot keys spread across replicas.
 	groups := make(map[string][]string)
 	for _, key := range keys {
-		nodes, err := m.hashring.GetNodes(key, 1)
+		nodes, err := m.hashring.GetNodes(key, m.replicationFactor)
 		if err != nil {
 			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
 			return
 		}
-		groups[nodes[0]] = append(groups[nodes[0]], key)
+		chosen := nodes[rand.Intn(len(nodes))]
+		groups[chosen] = append(groups[chosen], key)
 	}
 
 	type nodeResult struct {