A fast, embeddable key-value store for Go with automatic TTL expiration. Designed for telemetry, metrics, sessions, and any data that should disappear after a while.
- Automatic expiration — data expires and is cleaned up without manual intervention
- Time-bucketed storage — old data is deleted by removing entire segment files, no compaction overhead
- Write buffering — batches writes for high throughput
- LRU read cache — hot keys served from memory
- Crash recovery — periodic checkpoints for fast startup
- Atomic compaction — safe segment compaction with atomic file swaps
- Prioritized compaction — segments with most waste are compacted first
- Configurable checksums — CRC32, CRC32C, or CRC64
- Metrics hooks — plug in your own observability (Prometheus, etc.)
- Simple API — just
Put,Get,Delete, and a few query methods
go get github.com/expertblink/ttlstorepackage main
import (
"fmt"
"time"
"github.com/expertblink/ttlstore"
)
func main() {
db, err := ttlstore.Open("./data", ttlstore.DefaultOptions())
if err != nil {
panic(err)
}
defer db.Close()
// Store a value that expires in 5 minutes
db.Put([]byte("sensor:temp"), []byte("21.5"), 5*time.Minute)
// Retrieve it
val, err := db.Get([]byte("sensor:temp"))
if err != nil {
panic(err)
}
fmt.Println(string(val)) // "21.5"
}db, err := ttlstore.Open("./data", ttlstore.DefaultOptions())
if err != nil {
// handle error
}
defer db.Close()Store a key-value pair with a TTL.
err := db.Put([]byte("key"), []byte("value"), time.Hour)Retrieve a value. Returns ErrKeyNotFound if missing or ErrKeyExpired if expired.
val, err := db.Get([]byte("key"))
if err == ttlstore.ErrKeyNotFound {
// key doesn't exist
}
if err == ttlstore.ErrKeyExpired {
// key existed but has expired
}Remove a key immediately.
err := db.Delete([]byte("key"))// All non-expired keys
keys := db.Keys()
// Keys with prefix
keys := db.Scan("sensor:")
// Keys in lexicographical range (inclusive)
keys := db.Range("a", "z")Force write buffer to disk.
db.Flush()stats := db.Stats()
fmt.Printf("Keys: %d, Segments: %d, Cache: %d, Buffered: %d\n",
stats.Keys, stats.Segments, stats.CacheSize, stats.BufferedWrites)opts := ttlstore.Options{
// Time window for each segment file (default: 1 hour)
BucketDuration: time.Hour,
// Fsync after every write (default: false)
SyncWrites: false,
// How often to flush write buffer (default: 100ms)
FlushInterval: 100 * time.Millisecond,
// How often to clean up expired data (default: 1 minute)
CleanupInterval: time.Minute,
// Segments processed per cleanup cycle (default: 10)
CleanupBatchSize: 10,
// LRU cache size (default: 10000)
CacheSize: 10000,
// Max keys in index, 0 = unlimited (default: 0)
MaxKeys: 0,
// Writes buffered before flush (default: 1000)
WriteBufferSize: 1000,
// Compact segment when live ratio drops below (default: 0.5)
CompactionThreshold: 0.5,
// How often to checkpoint the index (default: 30s)
CheckpointInterval: 30 * time.Second,
// Checksum algorithm (default: CRC32IEEE)
ChecksumAlgorithm: ttlstore.ChecksumCRC32IEEE,
// Metrics hook for observability (default: nil)
Metrics: myMetricsHook,
}
db, err := ttlstore.Open("./data", opts)Choose the checksum algorithm based on your needs:
ttlstore.ChecksumCRC32IEEE // Default, fastest
ttlstore.ChecksumCRC32C // Castagnoli, better error detection
ttlstore.ChecksumCRC64ISO // 64-bit (truncated), strongestImplement the MetricsHook interface to receive operational metrics:
type MetricsHook interface {
OnPut(key []byte, valueSize int, ttl time.Duration)
OnGet(key []byte, hit bool, fromCache bool)
OnDelete(key []byte)
OnFlush(entries int, bytes int64, duration time.Duration)
OnCompaction(segment string, beforeBytes, afterBytes int64, duration time.Duration)
OnSegmentCreated(path string)
OnSegmentDeleted(path string)
}Example with Prometheus:
type prometheusMetrics struct {
puts prometheus.Counter
gets prometheus.Counter
cacheHits prometheus.Counter
flushBytes prometheus.Counter
}
func (m *prometheusMetrics) OnPut(key []byte, valueSize int, ttl time.Duration) {
m.puts.Inc()
}
func (m *prometheusMetrics) OnGet(key []byte, hit bool, fromCache bool) {
m.gets.Inc()
if hit && fromCache {
m.cacheHits.Inc()
}
}
func (m *prometheusMetrics) OnFlush(entries int, bytes int64, duration time.Duration) {
m.flushBytes.Add(float64(bytes))
}
// ... implement other methodsfunc SetSession(db *ttlstore.DB, sessionID string, userID string) error {
return db.Put([]byte("session:"+sessionID), []byte(userID), 24*time.Hour)
}
func GetSession(db *ttlstore.DB, sessionID string) (string, error) {
val, err := db.Get([]byte("session:" + sessionID))
if err != nil {
return "", err
}
return string(val), nil
}func CheckRateLimit(db *ttlstore.DB, ip string, limit int) bool {
key := []byte("ratelimit:" + ip)
val, err := db.Get(key)
if err == ttlstore.ErrKeyNotFound || err == ttlstore.ErrKeyExpired {
db.Put(key, []byte("1"), time.Minute)
return true
}
count := atoi(string(val)) + 1
if count > limit {
return false
}
db.Put(key, []byte(itoa(count)), time.Minute)
return true
}func RecordMetric(db *ttlstore.DB, metric string, value float64) error {
key := fmt.Sprintf("metric:%s:%d", metric, time.Now().UnixNano())
val := fmt.Sprintf("%f", value)
return db.Put([]byte(key), []byte(val), time.Hour)
}
func GetRecentMetrics(db *ttlstore.DB, metric string) []string {
return db.Scan("metric:" + metric + ":")
}func GetUser(db *ttlstore.DB, userID string) (*User, error) {
key := []byte("cache:user:" + userID)
val, err := db.Get(key)
if err == nil {
var user User
json.Unmarshal(val, &user)
return &user, nil
}
// Cache miss - fetch from database
user, err := fetchUserFromDB(userID)
if err != nil {
return nil, err
}
// Cache for 10 minutes
data, _ := json.Marshal(user)
db.Put(key, data, 10*time.Minute)
return user, nil
}Data is stored in time-bucketed segment files. Each segment covers a configurable time window (default: 1 hour). When you write a key, it goes to the current segment. When all keys in a segment have expired, the entire file is deleted.
data/
2025-01-23T14-00-00.seg # 14:00-15:00
2025-01-23T15-00-00.seg # 15:00-16:00
checkpoint.dat # index snapshot
This design means:
- Writes are append-only (fast)
- TTL cleanup is O(1) per segment (just delete the file)
- No fragmentation from individual key deletions
Benchmarks on Intel Xeon @ 2.1GHz:
| Operation | Latency | Allocations |
|---|---|---|
| Put | 6 µs | 5 allocs |
| Get (cache miss) | 146 ns | 1 alloc |
| Get (cache hit) | 144 ns | 1 alloc |
| Put (parallel) | 6 µs | 7 allocs |
- Index is in-memory. Use
MaxKeysto bound memory usage. - Keys limited to 64KB, values limited by available memory.
- Not designed for large values (>1MB). Consider storing references instead.
- Single-node only. No replication or clustering.
MIT