From f0cf59d3a1586148f72de6ba086e4a60a5e21030 Mon Sep 17 00:00:00 2001 From: DanieCuevas <43822444+DanielCuevas1208@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:40:25 -0700 Subject: [PATCH] feat: extend cinderstore --- .github/workflows/ci.yml | 3 + README.md | 78 +++++++++++++++--- shard.lock | 2 +- shard.yml | 2 +- spec/fast_mode_spec.cr | 159 +++++++++++++++++++++++++++++++++++++ spec/wal_spec.cr | 39 ++++++++- src/cinderstore/db.cr | 12 ++- src/cinderstore/demo.cr | 53 +++++++++++-- src/cinderstore/table.cr | 69 +++++++++++----- src/cinderstore/version.cr | 2 +- src/cinderstore/wal.cr | 81 ++++++++++++++++--- src/cli.cr | 26 ++++-- 12 files changed, 462 insertions(+), 64 deletions(-) create mode 100644 spec/fast_mode_spec.cr diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4b9de4..a6148b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,5 +37,8 @@ jobs: - name: Run the demo run: crystal run examples/demo.cr + - name: Run the server demo + run: crystal run examples/server_demo.cr + - name: Build the binary run: shards build --production diff --git a/README.md b/README.md index d9f2522..32a7c2b 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,13 @@ restart. It serves `get`, `put`, and `delete` over a local socket. Snapshots give you a consistent view of the store at one instant. A snapshot never blocks new writes. Release it when you are done. -This is release 0.3.0. It adds point-in-time snapshots and consistent reads. +This is release 0.4.0. It adds an optional checksum-free fast mode. ## Features - Memory table with ordered writes - Durable write ahead log with CRC32 framing +- Optional checksum-free fast mode - Sorted tables with a block index and a bloom filter - Block cache for fast repeated reads - Background flush and compaction @@ -59,7 +60,7 @@ scans, flushes, compacts, deletes, snapshots, and reopens a database. The output is deterministic. ```text -== Cinderstore 0.3.0 demo == +== Cinderstore 0.4.0 demo == Loaded 24 products from ...\fixtures\catalog.csv Database directory: ...\cinderstore-demo @@ -106,12 +107,24 @@ Database directory: ...\cinderstore-demo 8. Reopen the database and verify recovery rows after restart: 21 +9. Fast mode skips checksums + checksummed disk bytes: 1649 + fast disk bytes: 1645 + fast mode saves 4 bytes on disk + fast mode scan count: 24 + fast mode snapshot rows: 24 + fast mode rows after restart: 24 + Demo complete. ``` Step 7 shows the value of a snapshot. The live store drops SKU-0001 and adds two products. The snapshot still sees the state before those writes. +Step 9 shows the fast mode. Both databases hold the same data. The fast +table drops the CRC32 guards, so it is smaller. Reads, snapshots, and +recovery work the same. + ## Use the library Require the library. @@ -133,6 +146,20 @@ end db.close ``` +### Use fast mode + +Set `checksums` to `false` to skip CRC32 checks. + +```crystal +config = Cinderstore::DB::Config.new +config.checksums = false +db = Cinderstore::DB.new("data/my-store", config) +``` + +Fast mode writes smaller files and does less CPU work. It cannot detect torn +or corrupt data. Use it when a little risk is acceptable. Keep the default +when data integrity matters most. + ### Read a snapshot A snapshot is a consistent view at one instant. Take a snapshot, read it, @@ -225,6 +252,15 @@ Start the local server. bin/cinderstore server --db data --port 7654 ``` +Run the demo without checksums. + +```console +bin/cinderstore demo --no-checksums +``` + +The `server`, `put`, `del`, `flush`, and `compact` commands also accept +`--no-checksums`. + Run `bin/cinderstore help` for the full list of commands. ## Wire protocol @@ -275,6 +311,19 @@ A write goes to two places at once. The default mode fsyncs after every write. Set `sync_writes` to `false` for faster, less durable writes. +Every file records its own format. The log keeps a header. The table keeps a +version in the footer. The reader detects the format, so a database can mix +modes. + +### Fast mode + +Checksums find corruption. Each log record and each table block carries a +CRC32. The footer carries one too. Verifying them costs CPU time. + +Fast mode skips every CRC32. Files are smaller and reads do less work. +Corruption may go unnoticed. A fast file and a checked file can live in one +database. Toggle `checksums` at any time. + ### Flush When the memory table grows past its limit, the database freezes it. A new @@ -313,8 +362,9 @@ blocks so repeated reads avoid disk. ### Recovery On open, the database replays the write ahead log into the memory table. -Recovery is idempotent. A torn tail is detected by its CRC32 and skipped. -The manifest lists every table. Orphan files from a crash are removed. +Recovery is idempotent. A torn tail is skipped. With checksums on, the CRC32 +catches a torn tail. Without them, a short record stops the replay. The +manifest lists every table. Orphan files from a crash are removed. ## On-disk format @@ -324,7 +374,8 @@ Tables use a compact binary format. - A block index maps the first key of each block to its offset. - A bloom filter covers every key in the table. - A footer stores offsets, a version, and a CRC32. -- Each block and each log record carries a CRC32. +- Each block and each log record carries a CRC32 in the default format. +- The fast format writes no CRC32 guards. Sequence numbers make versions unique. They are per-write and never reused. @@ -339,6 +390,7 @@ config.memtable_limit = 4_i64 * 1024 * 1024 config.bloom_fpp = 0.01 config.cache_blocks = 512 config.sync_writes = true +config.checksums = true config.l0_compact_threshold = 4 config.compact_on_flush = true db = Cinderstore::DB.new("data", config) @@ -359,14 +411,14 @@ spec/ Test suite ## Test status -The suite runs with `crystal spec`. It has 97 examples. All pass on Windows +The suite runs with `crystal spec`. It has 108 examples. All pass on Windows and Linux. It covers the skip list, the memory table, the write ahead log, the bloom filter, and the block cache. It covers the tables, the iterators, -and the database. It covers compaction, durability, snapshots, and the -server protocol. +and the database. It covers compaction, durability, snapshots, the server +protocol, and the fast mode. The CI workflow runs on GitHub Actions for Windows and Ubuntu. It checks -formatting, runs the suite, runs the demo, and builds the binary. +formatting, runs the suite, runs both demos, and builds the binary. ## Limitations @@ -377,20 +429,22 @@ formatting, runs the suite, runs the demo, and builds the binary. - Compaction is a full merge. It is correct and simple, not incremental. - No multi-threaded runtime is required. The server uses fibers. - Release snapshots before you close the database. +- Fast mode cannot detect corruption. It skips every CRC32 guard. ## Roadmap Planned: -- Release 0.2: incremental compaction by level -- Release 0.4: optional checksum-free fast mode - Release 0.5: batch writes and group commit -- Release 0.6: secondary indexes +- Release 0.6: incremental compaction by level +- Release 0.7: secondary indexes Delivered: - Release 0.3: snapshot iterators and consistent reads. Snapshots give a stable view of the store. Compaction keeps referenced files alive. +- Release 0.4: optional checksum-free fast mode. Every file records its + format. Readers detect it, so modes can mix. ## License diff --git a/shard.lock b/shard.lock index c1dbcd2..8a8a3ae 100644 --- a/shard.lock +++ b/shard.lock @@ -1 +1 @@ -version: 0.3.0 +version: 0.4.0 diff --git a/shard.yml b/shard.yml index e335eb5..5225a79 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: cinderstore -version: 0.3.0 +version: 0.4.0 description: An embeddable key/value store built on a log structured merge tree. crystal: ">= 1.10.0" license: Apache-2.0 diff --git a/spec/fast_mode_spec.cr b/spec/fast_mode_spec.cr new file mode 100644 index 0000000..c0a1c16 --- /dev/null +++ b/spec/fast_mode_spec.cr @@ -0,0 +1,159 @@ +require "./spec_helper" + +describe "Cinderstore fast mode" do + it "stores and reads values without checksums" do + config = Cinderstore::SpecHelpers.fast_config + config.checksums = false + Cinderstore::SpecHelpers.with_db("fast-basic", config) do |db, _path| + db.put("a", "1") + db.put("b", "2") + db.get("a").should eq("1") + db.get("b").should eq("2") + db.get("c").should be_nil + end + end + + it "recovers writes after a restart" do + config = Cinderstore::SpecHelpers.fast_config + config.checksums = false + Cinderstore::SpecHelpers.with_db_path("fast-restart") do |path| + db = Cinderstore::DB.new(path, config) + db.put("a", "1") + db.put("b", "2") + db.close + + reopened = Cinderstore::DB.new(path, config) + reopened.get("a").should eq("1") + reopened.get("b").should eq("2") + reopened.close + end + end + + it "flushes into a table and reads it back" do + config = Cinderstore::SpecHelpers.fast_config + config.checksums = false + Cinderstore::SpecHelpers.with_db("fast-flush", config) do |db, _path| + 20.times { |i| db.put("k%02d" % i, "v#{i}") } + db.flush + db.stats.tables.should eq(1) + db.scan.size.should eq(20) + db.get("k10").should eq("v10") + end + end + + it "compacts fast tables into a valid level-1 set" do + config = Cinderstore::SpecHelpers.fast_config + config.checksums = false + Cinderstore::SpecHelpers.with_db("fast-compact", config) do |db, _path| + db.put("a", "1") + db.flush + db.put("b", "2") + db.flush + db.compact + db.stats.l1.should eq(1) + db.get("a").should eq("1") + db.get("b").should eq("2") + db.scan.size.should eq(2) + end + end + + it "keeps snapshots consistent in fast mode" do + config = Cinderstore::SpecHelpers.fast_config + config.checksums = false + Cinderstore::SpecHelpers.with_db("fast-snapshot", config) do |db, _path| + db.put("a", "old") + snap = db.snapshot + db.put("b", "new") + db.flush + snap.get("a").should eq("old") + snap.get("b").should be_nil + snap.count.should eq(1_i64) + db.scan.size.should eq(2) + end + end + + it "writes smaller tables than checksummed mode" do + entries = 300.times.map { |i| Cinderstore::Entry.new("key-%04d" % i, i.to_i64, true, "value-#{i}") }.to_a + dir = Cinderstore::SpecHelpers.tmp_db_path("fast-size") + Dir.mkdir_p(dir) + checksummed_path = File.join(dir, "000001.sst") + fast_path = File.join(dir, "000002.sst") + + File.open(checksummed_path, "w") do |io| + writer = Cinderstore::SstableWriter.new(io, 1_i64, 256, 0.01, true) + entries.each { |entry| writer.add(entry.key, entry.value, entry.seq, entry.alive) } + writer.finish + end + File.open(fast_path, "w") do |io| + writer = Cinderstore::SstableWriter.new(io, 2_i64, 256, 0.01, false) + entries.each { |entry| writer.add(entry.key, entry.value, entry.seq, entry.alive) } + writer.finish + end + + File.size(fast_path).should be < File.size(checksummed_path) + end + + it "round trips a fast table through a reader" do + dir = Cinderstore::SpecHelpers.tmp_db_path("fast-table") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.sst") + entries = 300.times.map { |i| Cinderstore::Entry.new("key-%04d" % i, i.to_i64, true, "value-#{i}") }.to_a + File.open(path, "w") do |io| + writer = Cinderstore::SstableWriter.new(io, 1_i64, 128, 0.01, false) + entries.each { |entry| writer.add(entry.key, entry.value, entry.seq, entry.alive) } + writer.finish + end + + reader = Cinderstore::SstableReader.new(path, 1_i64, nil) + all = [] of Cinderstore::Entry + reader.block_count.times { |i| all.concat(reader.load_block_entries(i)) } + all.should eq(entries) + reader.close + end + + it "keeps the intact prefix of a torn fast log" do + config = Cinderstore::SpecHelpers.fast_config + config.checksums = false + Cinderstore::SpecHelpers.with_db_path("fast-torn") do |path| + db = Cinderstore::DB.new(path, config) + db.put("alpha", "1") + db.put("beta", "2") + db.put("gamma", "3") + db.close + + wal = Dir.children(path).find { |n| n.ends_with?(".wal") }.not_nil! + Cinderstore::SpecHelpers.truncate(File.join(path, wal), File.size(File.join(path, wal)) - 3) + + reopened = Cinderstore::DB.new(path, config) + reopened.get("alpha").should eq("1") + reopened.get("beta").should eq("2") + reopened.get("gamma").should be_nil + reopened.close + end + end + + it "mixes fast and checksummed tables in one database" do + Cinderstore::SpecHelpers.with_db_path("fast-mix") do |path| + db = Cinderstore::DB.new(path, Cinderstore::SpecHelpers.fast_config) + db.put("a", "1") + db.put("b", "2") + db.flush + db.close + + fast_config = Cinderstore::SpecHelpers.fast_config + fast_config.checksums = false + reopened = Cinderstore::DB.new(path, fast_config) + reopened.put("c", "3") + reopened.put("d", "4") + reopened.flush + reopened.scan.size.should eq(4) + reopened.get("a").should eq("1") + reopened.get("d").should eq("4") + reopened.close + + again = Cinderstore::DB.new(path, Cinderstore::SpecHelpers.fast_config) + again.scan.size.should eq(4) + again.close + end + end +end diff --git a/spec/wal_spec.cr b/spec/wal_spec.cr index 3860506..414fd62 100644 --- a/spec/wal_spec.cr +++ b/spec/wal_spec.cr @@ -52,9 +52,11 @@ describe Cinderstore::Wal do writer.append(Cinderstore::Entry.new("alpha", 1_i64, true, "1")) writer.close - # Flip a byte inside the record body. + # Flip a byte inside the record body. The header, length, and framing + # stay intact, so only the checksum can reveal the corruption. The + # body starts after the nine-byte header and the length varint. bytes = File.read(path).to_slice.dup - bytes[1] = (bytes[1] ^ 0xFF).to_u8 + bytes[10] = (bytes[10] ^ 0xFF).to_u8 File.open(path, "w") { |f| f.write(bytes) } mem = Cinderstore::MemTable.new @@ -62,6 +64,39 @@ describe Cinderstore::Wal do mem.empty?.should be_true end + it "recovers a legacy log that has no header" do + dir = Cinderstore::SpecHelpers.tmp_db_path("wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + File.open(path, "w") do |io| + io.write(Cinderstore::Wal.encode(Cinderstore::Entry.new("a", 1_i64, true, "one"))) + io.write(Cinderstore::Wal.encode(Cinderstore::Entry.new("b", 2_i64, false, ""))) + end + + mem = Cinderstore::MemTable.new + seq = Cinderstore::Wal.recover(path, mem, 0_i64) + seq.should eq(2_i64) + mem.get("a").should eq("one") + mem.get("b").should be_nil + mem.get_entry("b").not_nil!.alive.should be_false + end + + it "replays a fast log that has no checksums" do + dir = Cinderstore::SpecHelpers.tmp_db_path("wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, false, false) + writer.append(Cinderstore::Entry.new("a", 1_i64, true, "one")) + writer.append(Cinderstore::Entry.new("b", 2_i64, true, "two")) + writer.close + + mem = Cinderstore::MemTable.new + seq = Cinderstore::Wal.recover(path, mem, 0_i64) + seq.should eq(2_i64) + mem.get("a").should eq("one") + mem.get("b").should eq("two") + end + it "survives a restart through the database" do Cinderstore::SpecHelpers.with_db_path("wal-durable") do |path| db = Cinderstore::DB.new(path) diff --git a/src/cinderstore/db.cr b/src/cinderstore/db.cr index f5707f9..a1bd58f 100644 --- a/src/cinderstore/db.cr +++ b/src/cinderstore/db.cr @@ -28,6 +28,10 @@ module Cinderstore property cache_blocks : Int32 = 512 # Fsync after every write when true. property sync_writes : Bool = true + # Write a CRC32 guard per record and per block when true. + # Set to false to skip checksum work at the cost of corruption + # detection. + property checksums : Bool = true # Number of level-0 tables that trigger compaction. property l0_compact_threshold : Int32 = 4 # Start background compaction after a flush when true. @@ -374,7 +378,7 @@ module Cinderstore private def create_active_wal : Nil id = @next_id @next_id += 1 - @wal = Wal::Writer.new(wal_path(id), @config.sync_writes) + @wal = Wal::Writer.new(wal_path(id), @config.sync_writes, @config.checksums) end # ------------------------------------------------------------------ @@ -429,7 +433,7 @@ module Cinderstore @next_id += 1 wal_id = @next_id @next_id += 1 - @wal = Wal::Writer.new(wal_path(wal_id), @config.sync_writes) + @wal = Wal::Writer.new(wal_path(wal_id), @config.sync_writes, @config.checksums) end end @@ -459,7 +463,7 @@ module Cinderstore final_path = table_path(table_id) meta = nil File.open(tmp_path, "w") do |io| - writer = SstableWriter.new(io, table_id, @config.block_size, @config.bloom_fpp) + writer = SstableWriter.new(io, table_id, @config.block_size, @config.bloom_fpp, @config.checksums) mem.each_entry do |entry| writer.add(entry.key, entry.value, entry.seq, entry.alive) end @@ -557,7 +561,7 @@ module Cinderstore id end io = File.open(File.join(@path, "#{Util.file_stem(current_id)}#{TMP_SUFFIX}"), "w") - writer = SstableWriter.new(io.not_nil!, current_id, @config.block_size, @config.bloom_fpp) + writer = SstableWriter.new(io.not_nil!, current_id, @config.block_size, @config.bloom_fpp, @config.checksums) end writer.not_nil!.add(entry.key, entry.value, entry.seq, true) written += 1 diff --git a/src/cinderstore/demo.cr b/src/cinderstore/demo.cr index 206635f..33fec1b 100644 --- a/src/cinderstore/demo.cr +++ b/src/cinderstore/demo.cr @@ -5,12 +5,13 @@ module Cinderstore # # The demo loads a small product catalog, writes it, scans a range, # flushes, compacts, and verifies recovery. Its output is deterministic. + # Set `checksums` to false to run the walkthrough in fast mode. class Demo - def self.run(db_path : String? = nil, fixture : String? = nil) : Int32 - new(db_path, fixture).run + def self.run(db_path : String? = nil, fixture : String? = nil, checksums : Bool = true) : Int32 + new(db_path, fixture, checksums).run end - def initialize(@db_path : String? = nil, @fixture : String? = nil) + def initialize(@db_path : String? = nil, @fixture : String? = nil, @checksums : Bool = true) end def run : Int32 @@ -25,6 +26,7 @@ module Cinderstore config = DB::Config.new config.sync_writes = false config.compact_on_flush = false + config.checksums = @checksums db = DB.new(path, config) puts "== Cinderstore #{VERSION} demo ==" @@ -86,12 +88,12 @@ module Cinderstore puts " live SKU-0001 => #{db.get("SKU-0001").inspect}" puts " snapshot SKU-0001 => #{snap.get("SKU-0001").inspect}" puts " snapshot iterator SKU-0010 to SKU-0016" - rows = [] of String + keys = [] of String iter = snap.iter("SKU-0010") while (entry = iter.next?) && entry.key < "SKU-0016" - rows << entry.key + keys << entry.key end - puts " #{rows.join(", ")}" + puts " #{keys.join(", ")}" snap.release puts "" @@ -103,6 +105,45 @@ module Cinderstore puts " rows after restart: #{count}" reopened.close puts "" + + puts "9. Fast mode skips checksums" + ref_config = DB::Config.new + ref_config.sync_writes = false + ref_config.compact_on_flush = false + ref_config.checksums = true + fast_config = DB::Config.new + fast_config.sync_writes = false + fast_config.compact_on_flush = false + fast_config.checksums = false + ref_path = "#{path}-ref" + fast_path = "#{path}-fast" + FileUtils.rm_rf(ref_path) if File.exists?(ref_path) + FileUtils.rm_rf(fast_path) if File.exists?(fast_path) + ref_db = DB.new(ref_path, ref_config) + fast_db = DB.new(fast_path, fast_config) + rows.each do |sku, name, price, stock| + value = %({"name":"#{name}","price":#{price},"stock":#{stock}}) + ref_db.put(sku, value) + fast_db.put(sku, value) + end + ref_db.flush + fast_db.flush + puts " checksummed disk bytes: #{ref_db.stats.disk_bytes}" + puts " fast disk bytes: #{fast_db.stats.disk_bytes}" + puts " fast mode saves #{ref_db.stats.disk_bytes - fast_db.stats.disk_bytes} bytes on disk" + puts " fast mode scan count: #{fast_db.scan.size}" + snap = fast_db.snapshot + puts " fast mode snapshot rows: #{snap.count}" + snap.release + ref_db.close + fast_db.close + fast_reopened = DB.new(fast_path, fast_config) + puts " fast mode rows after restart: #{fast_reopened.scan.size}" + fast_reopened.close + FileUtils.rm_rf(ref_path) rescue nil + FileUtils.rm_rf(fast_path) rescue nil + puts "" + puts "Demo complete." 0 end diff --git a/src/cinderstore/table.cr b/src/cinderstore/table.cr index bcd8bd3..da57fdc 100644 --- a/src/cinderstore/table.cr +++ b/src/cinderstore/table.cr @@ -32,17 +32,25 @@ module Cinderstore # A data block holds the block length, a batch of serialized entries, # and a CRC32. The block index maps the first key of each block to its # file offset and total length. The bloom filter lets a reader skip a - # table that cannot contain a key. The footer stores offsets and a CRC32 - # over the entire footer. + # table that cannot contain a key. The footer stores offsets, a format + # version, and a CRC32 over the entire footer. + # + # Fast tables use version two and skip every CRC32. The footer keeps its + # fixed size, so the reader can find the version before it decides how + # to verify the rest of the file. class SstableWriter - MAGIC = 0x43494E4445525F31_u64 - VERSION = 1_u32 - FOOTER_SIZE = 48 + MAGIC = 0x43494E4445525F31_u64 + # Format version that writes a CRC32 per block and on the footer. + VERSION_CHECKSUMMED = 1_u32 + # Format version that writes no CRC32 guards. + VERSION_FAST = 2_u32 + FOOTER_SIZE = 48 @io : IO @id : Int64 @block_size : Int32 @fpp : Float64 + @checksums : Bool @pending : IO::Memory @pending_keys : Array(String) @keys : Array(String) @@ -51,7 +59,8 @@ module Cinderstore @first : String? @last : String? - def initialize(io : IO, @id : Int64, @block_size : Int32 = 4096, @fpp : Float64 = 0.01) + def initialize(io : IO, @id : Int64, @block_size : Int32 = 4096, @fpp : Float64 = 0.01, + @checksums : Bool = true) @io = io @pending = IO::Memory.new @pending_keys = [] of String @@ -100,10 +109,14 @@ module Cinderstore footer.write_bytes(index_length.to_u64, IO::ByteFormat::LittleEndian) footer.write_bytes(bloom_offset.to_u64, IO::ByteFormat::LittleEndian) footer.write_bytes(bloom_length.to_u64, IO::ByteFormat::LittleEndian) - footer.write_bytes(VERSION, IO::ByteFormat::LittleEndian) + footer.write_bytes(@checksums ? VERSION_CHECKSUMMED : VERSION_FAST, IO::ByteFormat::LittleEndian) footer_bytes = footer.to_slice @io.write(footer_bytes) - @io.write_bytes(Util.crc32(footer_bytes), IO::ByteFormat::LittleEndian) + if @checksums + @io.write_bytes(Util.crc32(footer_bytes), IO::ByteFormat::LittleEndian) + else + @io.write(Bytes.new(4, 0_u8)) + end @io.flush TableMeta.new(@id, @first.not_nil!, @last.not_nil!, @count) @@ -114,8 +127,11 @@ module Cinderstore offset = @io.pos Util.write_varint(@io, data.size.to_u64) @io.write(data) - @io.write_bytes(Util.crc32(data), IO::ByteFormat::LittleEndian) - total = Util.varint_len(data.size.to_u64) + data.size + 4 + total = Util.varint_len(data.size.to_u64) + data.size + if @checksums + @io.write_bytes(Util.crc32(data), IO::ByteFormat::LittleEndian) + total += 4 + end @index << BlockIndexEntry.new(@pending_keys.first, offset.to_u64, total.to_u64) @pending = IO::Memory.new @pending_keys = [] of String @@ -136,6 +152,7 @@ module Cinderstore @index_length : UInt64 @bloom_offset : UInt64 @bloom_length : UInt64 + @version : UInt32 def initialize(@path : String, @file_id : Int64, @cache : BlockCache? = nil) @file = File.open(@path, "r") @@ -147,6 +164,7 @@ module Cinderstore @index_length = 0_u64 @bloom_offset = 0_u64 @bloom_length = 0_u64 + @version = 0_u32 read_footer read_index read_bloom @@ -215,14 +233,18 @@ module Cinderstore private def read_block_raw(entry : BlockIndexEntry) : Bytes @file.pos = entry.offset data_len = Util.read_varint(@file).to_i - if data_len.to_u64 + Util.varint_len(data_len.to_u64) + 4 != entry.length + expected = data_len.to_u64 + Util.varint_len(data_len.to_u64).to_u64 + expected += 4_u64 if @version == SstableWriter::VERSION_CHECKSUMMED + if expected != entry.length raise CorruptDataError.new("block length mismatch in #{@path}") end data = Bytes.new(data_len) @file.read_fully(data) - stored_crc = @file.read_bytes(UInt32, IO::ByteFormat::LittleEndian) - actual_crc = Util.crc32(data) - raise CorruptDataError.new("block checksum mismatch in #{@path}") unless stored_crc == actual_crc + if @version == SstableWriter::VERSION_CHECKSUMMED + stored_crc = @file.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + actual_crc = Util.crc32(data) + raise CorruptDataError.new("block checksum mismatch in #{@path}") unless stored_crc == actual_crc + end data end @@ -232,11 +254,6 @@ module Cinderstore footer = Bytes.new(SstableWriter::FOOTER_SIZE) @file.read_fully(footer) - crc_bytes = footer[SstableWriter::FOOTER_SIZE - 4, 4] - stored_crc = IO::Memory.new(crc_bytes).read_bytes(UInt32, IO::ByteFormat::LittleEndian) - actual_crc = Util.crc32(footer[0, SstableWriter::FOOTER_SIZE - 4]) - raise CorruptDataError.new("footer checksum mismatch in #{@path}") unless stored_crc == actual_crc - io = IO::Memory.new(footer) magic = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) raise CorruptDataError.new("bad magic in #{@path}") unless magic == SstableWriter::MAGIC @@ -244,8 +261,18 @@ module Cinderstore @index_length = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) @bloom_offset = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) @bloom_length = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) - version = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) - raise CorruptDataError.new("unsupported version #{version} in #{@path}") unless version == SstableWriter::VERSION + @version = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + case @version + when SstableWriter::VERSION_CHECKSUMMED + crc_bytes = footer[SstableWriter::FOOTER_SIZE - 4, 4] + stored_crc = IO::Memory.new(crc_bytes).read_bytes(UInt32, IO::ByteFormat::LittleEndian) + actual_crc = Util.crc32(footer[0, SstableWriter::FOOTER_SIZE - 4]) + raise CorruptDataError.new("footer checksum mismatch in #{@path}") unless stored_crc == actual_crc + when SstableWriter::VERSION_FAST + # No checksum to verify in the fast format. + else + raise CorruptDataError.new("unsupported version #{@version} in #{@path}") + end end private def read_index : Nil diff --git a/src/cinderstore/version.cr b/src/cinderstore/version.cr index 4b9d872..3b31a8c 100644 --- a/src/cinderstore/version.cr +++ b/src/cinderstore/version.cr @@ -1,3 +1,3 @@ module Cinderstore - VERSION = "0.3.0" + VERSION = "0.4.0" end diff --git a/src/cinderstore/wal.cr b/src/cinderstore/wal.cr index 04735b6..b3dffdf 100644 --- a/src/cinderstore/wal.cr +++ b/src/cinderstore/wal.cr @@ -1,19 +1,42 @@ module Cinderstore # The write ahead log (WAL). # - # The WAL frames each entry with a length, a payload, and a CRC32. We - # fsync after every append when durability is enabled. A torn write at - # the tail is truncated during recovery. + # The WAL frames each entry with a length and a payload. A CRC32 guard + # follows each payload when checksums are enabled. We fsync after every + # append when durability is enabled. A torn write at the tail is + # truncated during recovery. + # + # Every WAL starts with a header that records its format. The header + # makes each file self describing, so a database can mix formats after a + # configuration change. class Wal + MAGIC = Bytes[0x43, 0x49, 0x4E, 0x44, 0x57, 0x41, 0x4C, 0x00] + + # Format version that writes a CRC32 guard per record. + FORMAT_CHECKSUMMED = 1_u8 + # Format version that writes no CRC32 guards. + FORMAT_FAST = 2_u8 + + # Header size: magic plus the format byte. + HEADER_SIZE = 9_i64 + + # Upper bound for one record. Recovery uses it to reject garbage + # lengths before allocating a buffer. + MAX_RECORD_BYTES = 8 * 1024 * 1024 + # Serializes an entry into a framed record. - def self.encode(entry : Entry) : Bytes + # + # A CRC32 guard follows the payload when `checksums` is true. + def self.encode(entry : Entry, checksums : Bool = true) : Bytes body = IO::Memory.new Cinderstore.write_entry(body, entry) payload = body.to_slice framed = IO::Memory.new Util.write_varint(framed, payload.size.to_u64) framed.write(payload) - framed.write_bytes(Util.crc32(payload), IO::ByteFormat::LittleEndian) + if checksums + framed.write_bytes(Util.crc32(payload), IO::ByteFormat::LittleEndian) + end framed.to_slice end @@ -21,21 +44,27 @@ module Cinderstore # # Returns the largest sequence number seen, or `base_seq`. A partial # or corrupt trailing record stops the replay. Everything before it is - # preserved. + # preserved. The format comes from the file header, so the caller does + # not need to know how the log was written. def self.recover(path : String, mem : MemTable, base_seq : Int64) : Int64 max_seq = base_seq return max_seq unless File.exists?(path) File.open(path, "r") do |io| + format, start = detect_format(io) + io.pos = start loop do break if io.pos >= io.size begin body_len = Util.read_varint(io).to_i + raise CorruptDataError.new("wal record is too large") if body_len > MAX_RECORD_BYTES body = Bytes.new(body_len) io.read_fully(body) - stored_crc = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) - actual_crc = Util.crc32(body) - raise CorruptDataError.new("wal checksum mismatch") unless stored_crc == actual_crc + if format == FORMAT_CHECKSUMMED + stored_crc = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + actual_crc = Util.crc32(body) + raise CorruptDataError.new("wal checksum mismatch") unless stored_crc == actual_crc + end entry = Cinderstore.read_entry(IO::Memory.new(body)) if entry.alive mem.put(entry.key, entry.value, entry.seq) @@ -53,18 +82,39 @@ module Cinderstore max_seq end + # Returns the format of `io` and the offset where records begin. + # + # Logs written before the header existed have no magic. They use the + # checksummed format, which keeps old files readable. + def self.detect_format(io : IO) : Tuple(UInt8, Int64) + return {FORMAT_CHECKSUMMED, 0_i64} if io.size < HEADER_SIZE + io.pos = 0 + head = Bytes.new(MAGIC.size) + io.read_fully(head) + return {FORMAT_CHECKSUMMED, 0_i64} unless head == MAGIC + {io.read_byte.not_nil!, HEADER_SIZE} + end + # Appends framed records to a WAL file. class Writer getter path : String getter size : Int64 - def initialize(@path : String, @sync_each_write : Bool = true) + def initialize(@path : String, @sync_each_write : Bool = true, checksums : Bool = true) @file = File.open(@path, "a+") @size = @file.size + @checksums = checksums + if @size == 0 + write_header + else + # An existing log keeps its own format. New records must match + # the records already on disk so recovery can read them together. + @checksums = Wal.detect_format(@file).first == FORMAT_CHECKSUMMED + end end def append(entry : Entry) : Nil - record = Wal.encode(entry) + record = Wal.encode(entry, @checksums) @file.write(record) @file.flush @file.fsync if @sync_each_write @@ -75,6 +125,15 @@ module Cinderstore @file.fsync @file.close end + + private def write_header : Nil + io = IO::Memory.new + io.write(MAGIC) + io.write_byte(@checksums ? FORMAT_CHECKSUMMED : FORMAT_FAST) + @file.write(io.to_slice) + @file.flush + @size = HEADER_SIZE + end end end end diff --git a/src/cli.cr b/src/cli.cr index 8a91070..b44f5d4 100644 --- a/src/cli.cr +++ b/src/cli.cr @@ -39,18 +39,20 @@ module Cinderstore db_path = "cinderstore-data" host = "127.0.0.1" port = 7654 + no_checksums = false help = false parser = OptionParser.new parser.on("--db PATH", "Database directory") { |v| db_path = v } parser.on("--host HOST", "Bind address") { |v| host = v } parser.on("--port PORT", "Listen on this port") { |v| port = v.to_i } + parser.on("--no-checksums", "Skip CRC32 checks (faster, less safe)") { no_checksums = true } parser.on("-h", "--help", "Show this help") { help = true } parser.parse(rest) if help puts parser return 0 end - db = DB.new(db_path) + db = DB.new(db_path, db_config(no_checksums)) server = Server.new(db, host, port) server.run db.close @@ -61,11 +63,13 @@ module Cinderstore db_path = "cinderstore-data" key = "" value = "" + no_checksums = false help = false parser = OptionParser.new parser.on("--db PATH", "Database directory") { |v| db_path = v } parser.on("--key KEY", "Key to write") { |v| key = v } parser.on("--value VALUE", "Value to write") { |v| value = v } + parser.on("--no-checksums", "Skip CRC32 checks (faster, less safe)") { no_checksums = true } parser.on("-h", "--help", "Show this help") { help = true } parser.parse(rest) if help @@ -73,7 +77,7 @@ module Cinderstore return 0 end raise Error.new("missing --key") if key.empty? - db = DB.new(db_path) + db = DB.new(db_path, db_config(no_checksums)) begin db.put(key, value) puts "ok" @@ -114,10 +118,12 @@ module Cinderstore private def run_delete(rest : Array(String)) : Int32 db_path = "cinderstore-data" key = "" + no_checksums = false help = false parser = OptionParser.new parser.on("--db PATH", "Database directory") { |v| db_path = v } parser.on("--key KEY", "Key to delete") { |v| key = v } + parser.on("--no-checksums", "Skip CRC32 checks (faster, less safe)") { no_checksums = true } parser.on("-h", "--help", "Show this help") { help = true } parser.parse(rest) if help @@ -125,7 +131,7 @@ module Cinderstore return 0 end raise Error.new("missing --key") if key.empty? - db = DB.new(db_path) + db = DB.new(db_path, db_config(no_checksums)) begin db.delete(key) puts "ok" @@ -166,16 +172,18 @@ module Cinderstore private def run_simple(action : String, rest : Array(String)) : Int32 db_path = "cinderstore-data" + no_checksums = false help = false parser = OptionParser.new parser.on("--db PATH", "Database directory") { |v| db_path = v } + parser.on("--no-checksums", "Skip CRC32 checks (faster, less safe)") { no_checksums = true } parser.on("-h", "--help", "Show this help") { help = true } parser.parse(rest) if help puts parser return 0 end - db = DB.new(db_path) + db = DB.new(db_path, db_config(no_checksums)) begin case action when "stats" @@ -196,17 +204,25 @@ module Cinderstore private def run_demo(rest : Array(String)) : Int32 db_path = nil fixture = nil + no_checksums = false help = false parser = OptionParser.new parser.on("--db PATH", "Database directory") { |v| db_path = v } parser.on("--fixture PATH", "CSV fixture file") { |v| fixture = v } + parser.on("--no-checksums", "Skip CRC32 checks (faster, less safe)") { no_checksums = true } parser.on("-h", "--help", "Show this help") { help = true } parser.parse(rest) if help puts parser return 0 end - Demo.run(db_path, fixture) + Demo.run(db_path, fixture, !no_checksums) + end + + private def db_config(no_checksums : Bool) : DB::Config + config = DB::Config.new + config.checksums = false if no_checksums + config end private def print_help : Nil