From 7932827d59c5ff4d2ad59f4d2a8ed69e16566da9 Mon Sep 17 00:00:00 2001 From: DanieCuevas <43822444+DanielCuevas1208@users.noreply.github.com> Date: Mon, 3 Aug 2026 05:09:56 -0700 Subject: [PATCH] feat: extend cinderstore --- .gitattributes | 5 ++ .github/dependabot.yml | 7 ++ .github/workflows/ci.yml | 5 +- README.md | 90 +++++++++----------- shard.lock | 2 +- shard.yml | 2 +- spec/compaction_spec.cr | 166 +++++++++++++++++++++++++++++++++++++ src/cinderstore/db.cr | 129 ++++++++++++++++++++++------ src/cinderstore/demo.cr | 31 ++++++- src/cinderstore/version.cr | 2 +- 10 files changed, 356 insertions(+), 83 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/dependabot.yml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..972b665 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Normalize every text file to LF on every platform. +# +# LF keeps `crystal tool format --check` stable in CI. The format check +# treats CRLF files as changed, which fails on Windows checkouts. +* text=auto eol=lf diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ff1ffd1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 10 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fdcddd..a4b9de4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Crystal - uses: crystal-lang/setup-crystal@v2 + uses: crystal-lang/install-crystal@v1 with: crystal: 1.21.0 @@ -34,5 +34,8 @@ jobs: - name: Run the test suite run: crystal spec + - name: Run the demo + run: crystal run examples/demo.cr + - name: Build the binary run: shards build --production diff --git a/README.md b/README.md index 2c2aff3..a66604c 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,12 @@ # Cinderstore -[![CI](https://github.com///actions/workflows/ci.yml/badge.svg)](https://github.com///actions/workflows/ci.yml) - +[![CI](https://github.com/DanielCuevas1208/cinderstore/actions/workflows/ci.yml/badge.svg)](https://github.com/DanielCuevas1208/cinderstore/actions/workflows/ci.yml) -Cinderstore is an embeddable key and value store. It is built on a log -structured merge tree (LSM tree). It is written in Crystal and uses only the -Crystal standard library. +Cinderstore is an embeddable key and value store. It is built on a log structured merge tree (LSM tree). It is written in Crystal and uses only the Crystal standard library. -The store keeps a write ahead log for durability. It flushes memory to sorted -files. It merges those files during compaction. It recovers all data after a -restart. It serves `get`, `put`, and `delete` over a local socket. +The store keeps a write ahead log for durability. It flushes memory to sorted files. It merges overflowing files during compaction. It recovers all data after a restart. It serves `get`, `put`, and `delete` over a local socket. -This is release 0.1.0. It is the first coherent release. +This is release 0.2.0. It adds leveled compaction. ## Features @@ -20,6 +15,7 @@ This is release 0.1.0. It is the first coherent release. - Sorted tables with a block index and a bloom filter - Block cache for fast repeated reads - Background flush and compaction +- Multi-level compaction that rewrites only the tables it must touch - Range scans with an iterator API - Crash recovery from the write ahead log - Local TCP server with a line protocol @@ -50,12 +46,10 @@ bin/cinderstore demo ## Demo output -The demo loads a product catalog from `fixtures/catalog.csv`. It writes, -scans, flushes, compacts, deletes, and reopens a database. The output is -deterministic. +The demo loads a product catalog from `fixtures/catalog.csv`. It writes, scans, flushes, compacts, and reopens a database. The output is deterministic. ```text -== Cinderstore 0.1.0 demo == +== Cinderstore 0.2.0 demo == Loaded 24 products from ...\fixtures\catalog.csv Database directory: ...\cinderstore-demo @@ -72,25 +66,36 @@ Database directory: ...\cinderstore-demo ... (24 rows in the store) 3. Flush memtable to a sorted table - tables: 1 (l0: 1, l1: 0), entries: 24 + tables: 1 (levels: [1]), entries: 24 disk bytes: 1649, memtable bytes: 0 4. Delete 4 products, update 2 products, then flush again - tables: 2 (l0: 2, l1: 0), entries: 30 + tables: 2 (levels: [2]), entries: 30 disk bytes: 1902, memtable bytes: 0 + The deleted keys still occupy space in the level-0 tables. -5. Compact merges the tables and drops the deleted keys - tables: 1 (l0: 0, l1: 1), entries: 20 +5. Compact merges the level-0 tables into level 1 + tables: 1 (levels: [0, 1]), entries: 20 disk bytes: 1384, memtable bytes: 0 + The store keeps the newest value for each key. 6. Verify deletes and updates after compaction get SKU-0003 => nil - get SKU-0001 => {"name":"Forge Anvil 45kg","price":175.00,"stock":14} + get SKU-0001 => "{\"name\":\"Forge Anvil 45kg\",\"price\":175.00,\"stock\":14}" scan count => 20 7. Reopen the database and verify recovery rows after restart: 20 +8. Add 32 spare parts across four flushes, then compact + tables: 1 (levels: [0, 0, 1]), entries: 52 + disk bytes: 3366, memtable bytes: 0 + The spare parts cascade into a fresh level-2 table. + +9. Reopen again and verify the full store + rows after restart: 52 + levels after restart: [0, 0, 1] + Demo complete. ``` @@ -127,13 +132,12 @@ Flush and compact explicitly. ```crystal db.flush # Move the memtable into a table. -db.compact # Merge tables and drop deleted keys. +db.compact # Merge overflowing levels into deeper levels. ``` ## Command line tool -The tool uses a database directory. The default directory is -`cinderstore-data`. +The tool uses a database directory. The default directory is `cinderstore-data`. Write a value. @@ -175,9 +179,7 @@ Run `bin/cinderstore help` for the full list of commands. ## Wire protocol -The server listens on `127.0.0.1:7654` by default. Commands are lines of -text. A command ends with a newline. Keys must not contain spaces or -newlines. Values must not contain newlines. +The server listens on `127.0.0.1:7654` by default. Commands are lines of text. A command ends with a newline. Keys must not contain spaces or newlines. Values must not contain newlines. | Command | Meaning | | --- | --- | @@ -208,8 +210,7 @@ printf "PUT forge-hammer steel\nGET forge-hammer\n" | nc 127.0.0.1 7654 ## Architecture -The database stores data in a single directory. The directory contains a -manifest, one write ahead log, and sorted tables. +The database stores data in a single directory. The directory contains a manifest, one write ahead log, and sorted tables. Tables live in levels. ### Write path @@ -218,34 +219,23 @@ A write goes to two places at once. 1. Append the entry to the write ahead log. 2. Insert the entry into the memory table. -The default mode fsyncs after every write. Set `sync_writes` to `false` for -faster, less durable writes. +The default mode fsyncs after every write. Set `sync_writes` to `false` for faster, less durable writes. ### Flush -When the memory table grows past its limit, the database freezes it. A new -memory table starts. A background task writes the frozen table to a sorted -file. The old log is deleted only after the file is durable. +When the memory table grows past its limit, the database freezes it. A new memory table starts. A background task writes the frozen table to a sorted file. The old log is deleted only after the file is durable. ### Compaction -Level-0 tables may overlap. Compaction merges every table into a fresh, -non-overlapping level-1 set. The merge keeps the newest entry for each key. -It drops tombstones, because it includes all data. New writes continue into -the memory table during the merge. +Level-0 tables may overlap. Compaction merges a level that grew past its limit into the next level. Each merge keeps the newest entry for every key. Tables that do not overlap the merge range stay in place. A tombstone is dropped only when the merge covers every copy of its key. New writes continue into the memory table during the merge. ### Read path -A read merges the memory table, any frozen table, and all sorted tables. The -merge yields the newest entry for each key. The bloom filter lets a reader -skip a table that cannot contain the key. The block cache holds decoded -blocks so repeated reads avoid disk. +A read merges the memory table, any frozen table, and all sorted tables. The merge yields the newest entry for each key. The bloom filter lets a reader skip a table that cannot contain the key. The block cache holds decoded blocks so repeated reads avoid disk. ### Recovery -On open, the database replays the write ahead log into the memory table. -Recovery is idempotent. A torn tail is detected by its CRC32 and skipped. -The manifest lists every table. Orphan files from a crash are removed. +On open, the database replays the write ahead log into the memory table. Recovery is idempotent. A torn tail is detected by its CRC32 and skipped. The manifest lists every table. Orphan files from a crash are removed. ## On-disk format @@ -271,10 +261,14 @@ config.bloom_fpp = 0.01 config.cache_blocks = 512 config.sync_writes = true config.l0_compact_threshold = 4 +config.l1_compact_threshold = 4 +config.max_level = 6 config.compact_on_flush = true db = Cinderstore::DB.new("data", config) ``` +The thresholds set the table count that triggers a level merge. The `max_level` value bounds the deepest level. Lower thresholds compact sooner but rewrite more often. + ## Project layout ```text @@ -289,13 +283,9 @@ spec/ Test suite ## Test status -The suite runs with `crystal spec`. It has 82 examples. All pass on Windows -and Linux. It covers the skip list, the memory table, the write ahead log, -the bloom filter, and the block cache. It covers the tables, the iterators, -and the database. It covers compaction, durability, and the server protocol. +The suite runs with `crystal spec`. It has 86 examples. All pass on Windows and Linux. It covers the skip list, the memory table, the write ahead log, the bloom filter, and the block cache. It covers the tables, the iterators, and the database. It covers compaction, leveled cascades, tombstone lifecycle, durability, and the server protocol. -The CI workflow runs on GitHub Actions for Windows and Ubuntu. It checks -formatting, runs the suite, and builds the binary. +The CI workflow runs on GitHub Actions for Windows and Ubuntu. It checks formatting, runs the suite, runs the demo, and builds the binary. ## Limitations @@ -303,12 +293,12 @@ formatting, runs the suite, and builds the binary. - Values are limited to 4 MB. - Keys are limited to 4 KB. - The server protocol is unencrypted. Use it on localhost only. -- Compaction is a full merge. It is correct and simple, not incremental. +- Compaction merges whole levels. It is correct and simple, not incremental within a level. - No multi-threaded runtime is required. The server uses fibers. ## Roadmap -- Release 0.2: incremental compaction by level +- Release 0.2: leveled compaction (complete) - Release 0.3: snapshot iterators and consistent reads - Release 0.4: optional checksum-free fast mode - Release 0.5: batch writes and group commit diff --git a/shard.lock b/shard.lock index 5593cd8..1726e2d 100644 --- a/shard.lock +++ b/shard.lock @@ -1 +1 @@ -version: 0.1.0 +version: 0.2.0 diff --git a/shard.yml b/shard.yml index c8e269b..4fc81c1 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: cinderstore -version: 0.1.0 +version: 0.2.0 description: An embeddable key/value store built on a log structured merge tree. crystal: ">= 1.10.0" license: Apache-2.0 diff --git a/spec/compaction_spec.cr b/spec/compaction_spec.cr index f0c3a09..c7a075e 100644 --- a/spec/compaction_spec.cr +++ b/spec/compaction_spec.cr @@ -136,4 +136,170 @@ describe "Cinderstore compaction" do db.scan.size.should eq(25_000) end end + + it "cascades level-0 tables through the deeper levels" do + config = Cinderstore::SpecHelpers.fast_config + config.l1_compact_threshold = 2 + config.max_level = 3 + Cinderstore::SpecHelpers.with_db("compact-cascade", config) do |db, _path| + db.put("a", "1") + db.put("b", "2") + db.flush + db.put("c", "3") + db.put("d", "4") + db.flush + db.compact + db.stats.levels.should eq([0, 1]) + db.stats.l0.should eq(0) + db.stats.l1.should eq(1) + + db.put("e", "5") + db.put("f", "6") + db.flush + db.put("g", "7") + db.put("h", "8") + db.flush + db.compact + db.stats.levels.should eq([0, 0, 1]) + db.stats.l2.should eq(1) + db.scan.map(&.[0]).should eq(%w[a b c d e f g h]) + db.get("e").should eq("5") + end + end + + it "keeps a tombstone while a deeper level still holds the key" do + config = Cinderstore::SpecHelpers.fast_config + config.l1_compact_threshold = 2 + config.max_level = 3 + Cinderstore::SpecHelpers.with_db("compact-guard", config) do |db, _path| + # Build an old value for "k" at the deepest level. + db.put("k", "old") + db.put("a", "1") + db.put("b", "2") + db.put("c", "3") + db.flush + db.put("m", "4") + db.put("n", "5") + db.put("o", "6") + db.put("p", "7") + db.flush + db.compact + db.put("q", "8") + db.put("r", "9") + db.put("s", "10") + db.put("t", "11") + db.flush + db.put("u", "12") + db.put("v", "13") + db.put("w", "14") + db.put("x", "15") + db.flush + db.compact + db.stats.l2.should eq(1) + db.get("k").should eq("old") + + # Delete "k". The merge into level 1 must keep the tombstone, because + # level 2 still holds the older value for "k". + db.put("k", "new") + db.delete("k") + db.put("y", "16") + db.put("z", "17") + db.flush + db.put("aa", "18") + db.put("bb", "19") + db.flush + db.compact + db.stats.levels.should eq([0, 1, 1]) + db.get("k").should be_nil + db.get("y").should eq("16") + db.scan.map(&.[0]).should eq(%w[a aa b bb c m n o p q r s t u v w x y z]) + end + end + + it "drops a tombstone once the merge covers the deepest level" do + config = Cinderstore::SpecHelpers.fast_config + config.l1_compact_threshold = 2 + config.max_level = 3 + Cinderstore::SpecHelpers.with_db("compact-expire", config) do |db, _path| + db.put("k", "old") + db.put("a", "1") + db.put("b", "2") + db.put("c", "3") + db.flush + db.put("m", "4") + db.put("n", "5") + db.put("o", "6") + db.put("p", "7") + db.flush + db.compact + db.put("q", "8") + db.put("r", "9") + db.put("s", "10") + db.put("t", "11") + db.flush + db.put("u", "12") + db.put("v", "13") + db.put("w", "14") + db.put("x", "15") + db.flush + db.compact + + db.put("k", "new") + db.delete("k") + db.put("y", "16") + db.put("z", "17") + db.flush + db.put("aa", "18") + db.put("bb", "19") + db.flush + db.compact + db.get("k").should be_nil + + # Two disjoint batches push level 1 past its threshold. The cascade + # merges level 1 into level 2, the deepest level. Every copy of "k" + # is inside the merge, so the tombstone is dropped. + db.put("zz", "20") + db.put("zza", "21") + db.flush + db.put("zzb", "22") + db.put("zzc", "23") + db.flush + db.compact + db.stats.levels.should eq([0, 0, 1]) + db.stats.entries.should eq(23) + db.get("k").should be_nil + db.scan.map(&.[0]).should eq(%w[a aa b bb c m n o p q r s t u v w x y z zz zza zzb zzc]) + end + end + + it "recovers a multi-level layout after a restart" do + config = Cinderstore::SpecHelpers.fast_config + config.l1_compact_threshold = 2 + config.max_level = 3 + Cinderstore::SpecHelpers.with_db_path("compact-deep-restart") do |path| + db = Cinderstore::DB.new(path, config) + db.put("a", "1") + db.put("b", "2") + db.flush + db.put("c", "3") + db.put("d", "4") + db.flush + db.compact + db.put("e", "5") + db.put("f", "6") + db.flush + db.put("g", "7") + db.put("h", "8") + db.flush + db.compact + db.stats.levels.should eq([0, 0, 1]) + db.close + + reopened = Cinderstore::DB.new(path, config) + reopened.stats.levels.should eq([0, 0, 1]) + reopened.scan.map(&.[0]).should eq(%w[a b c d e f g h]) + reopened.get("f").should eq("6") + reopened.close + end + end end diff --git a/src/cinderstore/db.cr b/src/cinderstore/db.cr index 9ca6401..13397da 100644 --- a/src/cinderstore/db.cr +++ b/src/cinderstore/db.cr @@ -5,8 +5,9 @@ module Cinderstore # # Writes go to a memtable and a write ahead log. When the memtable grows # past its limit we flush it to a sorted table. Background compaction - # merges tables and drops stale data. Reads merge the memtable and all - # tables, which makes every view consistent with the write order. + # merges overflowing levels into deeper levels and drops stale data. + # Reads merge the memtable and all tables, which makes every view + # consistent with the write order. class DB MANIFEST_NAME = "MANIFEST" WAL_SUFFIX = ".wal" @@ -30,6 +31,10 @@ module Cinderstore property sync_writes : Bool = true # Number of level-0 tables that trigger compaction. property l0_compact_threshold : Int32 = 4 + # Number of tables in a deep level that trigger a merge into the next. + property l1_compact_threshold : Int32 = 4 + # Deepest level that compaction may write into. + property max_level : Int32 = 6 # Start background compaction after a flush when true. property compact_on_flush : Bool = true end @@ -37,8 +42,8 @@ module Cinderstore # A snapshot of database counters for reporting. class Stats getter tables : Int32 - getter l0 : Int32 - getter l1 : Int32 + # Table count per level. Index zero is level 0. + getter levels : Array(Int32) getter entries : Int64 getter disk_bytes : Int64 getter memtable_bytes : Int64 @@ -46,17 +51,28 @@ module Cinderstore getter seq : Int64 getter cache_hits : Int64 getter cache_misses : Int64 + getter l0 : Int32 + getter l1 : Int32 + getter l2 : Int32 + getter l3 : Int32 - def initialize(@tables : Int32, @l0 : Int32, @l1 : Int32, @entries : Int64, + def initialize(@tables : Int32, @levels : Array(Int32), @entries : Int64, @disk_bytes : Int64, @memtable_bytes : Int64, @wal_bytes : Int64, @seq : Int64, @cache_hits : Int64, @cache_misses : Int64) + @l0 = @levels.size > 0 ? @levels[0] : 0 + @l1 = @levels.size > 1 ? @levels[1] : 0 + @l2 = @levels.size > 2 ? @levels[2] : 0 + @l3 = @levels.size > 3 ? @levels[3] : 0 end - def to_h : Hash(String, Int32 | Int64) + def to_h : Hash(String, Int32 | Int64 | Array(Int32)) { "tables" => @tables, + "levels" => @levels, "l0" => @l0, "l1" => @l1, + "l2" => @l2, + "l3" => @l3, "entries" => @entries, "disk_bytes" => @disk_bytes, "memtable_bytes" => @memtable_bytes, @@ -72,7 +88,7 @@ module Cinderstore end def to_s(io : IO) : Nil - io << "tables: #{@tables} (l0: #{@l0}, l1: #{@l1})\n" + io << "tables: #{@tables} (levels: #{@levels})\n" io << "entries: #{@entries}\n" io << "disk bytes: #{@disk_bytes}\n" io << "memtable bytes: #{@memtable_bytes}\n" @@ -212,7 +228,7 @@ module Cinderstore trigger_compact end - # Merges all tables into a fresh, non-overlapping level-1 set. + # Merges tables from overflowing levels into deeper levels. def compact : Nil @flush_lock.synchronize do do_compact @@ -232,8 +248,7 @@ module Cinderstore end Stats.new( tables: table_refs.size, - l0: @levels[0].size, - l1: @levels.size > 1 ? @levels[1].size : 0, + levels: @levels.map(&.size), entries: entries, disk_bytes: disk_bytes, memtable_bytes: @mem.approximate_bytes, @@ -429,7 +444,8 @@ module Cinderstore private def trigger_compact : Nil return unless @config.compact_on_flush - return if @levels[0].size < @config.l0_compact_threshold + needs = @lock.synchronize { select_compaction_level(force: false) } + return unless needs return if @compacting @compacting = true spawn do @@ -446,28 +462,87 @@ module Cinderstore # ------------------------------------------------------------------ private def do_compact : Nil - tables = nil - @lock.synchronize do - return if @closed - tables = @levels.flatten - return if tables.size < 2 + loop do + level = @lock.synchronize do + return if @closed + select_compaction_level(force: true) + end + break unless level + break unless compact_level(level) + end + end + + # Returns the topmost level that needs compaction, or nil. + # + # A background pass compacts level 0 only past its threshold. An + # explicit pass also merges a level-0 pair, so `compact` always makes + # progress when two tables exist. Deeper levels compact when they grow + # past `l1_compact_threshold`. + private def select_compaction_level(force : Bool) : Int32? + l0 = @levels[0].size + return 0 if l0 >= @config.l0_compact_threshold + return 0 if force && l0 >= 2 + (1...@levels.size).each do |i| + next unless i + 1 < @config.max_level + return i if @levels[i].size >= @config.l1_compact_threshold + end + nil + end + + # Merges every table in `level` with the overlapping tables of the + # next level. Outputs land in the next level. Tables that do not + # overlap the source ranges stay in place, so the merge rewrites only + # the tables it must touch. Returns false when nothing can be merged. + private def compact_level(level : Int32) : Bool + sources, overlaps, drop_tombstones = @lock.synchronize do + return false if @closed + src = @levels[level] + tgt = @levels[level + 1]? || ([] of TableRef) + ovl = tgt.select { |ref| overlaps_any?(ref, src) } + deepest = deepest_level + drop = deepest.nil? || deepest.not_nil! <= level + 1 + {src, ovl, drop} end - outputs = merge_tables(tables.not_nil!) + to_merge = sources + overlaps + return false if to_merge.size < 2 + + outputs = merge_tables(to_merge, drop_tombstones) @lock.synchronize do - @levels.flatten.each(&.close) - @levels = [[] of TableRef, outputs.map { |m| TableRef.new(m.id, m.first, m.last, m.count, table_path(m.id)) }] - save_manifest - tables.not_nil!.each do |ref| - path = ref.path - File.delete(path) if File.exists?(path) + return false if @closed + while @levels.size <= level + 1 + @levels << [] of TableRef + end + to_merge.each(&.close) + @levels[level] = [] of TableRef + kept = @levels[level + 1].reject { |ref| overlaps_any?(ref, sources) } + @levels[level + 1] = kept + outputs.map { |m| TableRef.new(m.id, m.first, m.last, m.count, table_path(m.id)) } + to_merge.each do |ref| + File.delete(ref.path) if File.exists?(ref.path) end + save_manifest + true end end - # Merges every table into fresh tables and drops tombstones. - private def merge_tables(tables : Array(TableRef)) : Array(TableMeta) + # Returns the index of the deepest level that holds a table, or nil. + private def deepest_level : Int32? + (@levels.size - 1).downto(0) do |i| + return i unless @levels[i].empty? + end + nil + end + + # Returns true when the key ranges of `ref` and any table in `others` + # overlap. Ranges are inclusive, so touching boundaries count as an + # overlap. A conservative answer is safe: it rewrites more, never less. + private def overlaps_any?(ref : TableRef, others : Array(TableRef)) : Bool + others.any? { |other| ref.first <= other.last && other.first <= ref.last } + end + + # Merges tables into fresh tables and drops tombstones when allowed. + private def merge_tables(tables : Array(TableRef), drop_tombstones : Bool) : Array(TableMeta) readers = @lock.synchronize { tables.map { |ref| ref.reader(@block_cache) } } sources = [] of Store::Iter readers.each { |reader| sources << TableIter.new(reader) } @@ -492,7 +567,7 @@ module Cinderstore } while entry = iter.next? - next unless entry.alive + next if drop_tombstones && !entry.alive if writer.nil? current_id = @lock.synchronize do id = @next_id @@ -502,7 +577,7 @@ module Cinderstore io = File.open(File.join(@path, "#{Util.file_stem(current_id)}#{TMP_SUFFIX}"), "w") writer = SstableWriter.new(io.not_nil!, current_id, @config.block_size, @config.bloom_fpp) end - writer.not_nil!.add(entry.key, entry.value, entry.seq, true) + writer.not_nil!.add(entry.key, entry.value, entry.seq, drop_tombstones ? true : entry.alive) written += 1 finish_output.call if written >= MAX_TABLE_ENTRIES end diff --git a/src/cinderstore/demo.cr b/src/cinderstore/demo.cr index fd224fc..483c955 100644 --- a/src/cinderstore/demo.cr +++ b/src/cinderstore/demo.cr @@ -25,6 +25,8 @@ module Cinderstore config = DB::Config.new config.sync_writes = false config.compact_on_flush = false + config.l1_compact_threshold = 2 + config.max_level = 3 db = DB.new(path, config) puts "== Cinderstore #{VERSION} demo ==" @@ -62,7 +64,7 @@ module Cinderstore puts " The deleted keys still occupy space in the level-0 tables." puts "" - puts "5. Compact merges the tables and drops the deleted keys" + puts "5. Compact merges the level-0 tables into level 1" db.compact print_stats(db) puts " The store keeps the newest value for each key." @@ -82,13 +84,38 @@ module Cinderstore puts " rows after restart: #{count}" reopened.close puts "" + + puts "8. Add 32 spare parts across four flushes, then compact" + writer = DB.new(path, config) + 4.times do |batch| + 8.times do |i| + n = batch * 8 + i + 1 + writer.put("SPARE-%03d" % n, spare_value(n)) + end + writer.flush + end + writer.compact + print_stats(writer) + puts " The spare parts cascade into a fresh level-2 table." + puts "" + + puts "9. Reopen again and verify the full store" + final = DB.new(path, config) + puts " rows after restart: #{final.scan.size}" + puts " levels after restart: #{final.stats.levels}" + final.close + puts "" puts "Demo complete." 0 end + private def spare_value(n : Int32) : String + %({"name":"Spare Part #{n}","price":5.00,"stock":10}) + end + private def print_stats(db : DB) : Nil stats = db.stats - puts " tables: #{stats.tables} (l0: #{stats.l0}, l1: #{stats.l1}), entries: #{stats.entries}" + puts " tables: #{stats.tables} (levels: #{stats.levels}), entries: #{stats.entries}" puts " disk bytes: #{stats.disk_bytes}, memtable bytes: #{stats.memtable_bytes}" end diff --git a/src/cinderstore/version.cr b/src/cinderstore/version.cr index 16eaad2..f883acc 100644 --- a/src/cinderstore/version.cr +++ b/src/cinderstore/version.cr @@ -1,3 +1,3 @@ module Cinderstore - VERSION = "0.1.0" + VERSION = "0.2.0" end