diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4b9de4..213bd9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Check out the repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Crystal uses: crystal-lang/install-crystal@v1 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ef704d7 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,25 @@ +# Changelog + +Cinderstore follows a numbered release track. Each release keeps the on-disk +format compatible with earlier releases. + +## 0.4.0 - 2026-08-03 + +Added the optional checksum-free fast mode. + +- Add the `checksums` config option. It defaults to `true`. +- Skip CRC32 work on writes and reads when the option is off. +- Record the checksum mode in the table footer and the WAL header. +- Read version 1 tables and logs from release 0.3 unchanged. +- Report the durable checksum mode through `stats`. +- Add the `--no-checksums` flag to the command line tool. +- Add 12 tests for the checksum-free mode. + +## 0.3.0 - 2026-08-03 + +Added point-in-time snapshots. + +- Create an immutable snapshot with `snapshot`. +- Keep snapshot reads consistent while the store keeps writing. +- Keep compacted table files alive until snapshots release them. +- Add snapshot iterators and range scans. diff --git a/README.md b/README.md index d9f2522..c8f3aaa 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,11 @@ restart. It serves `get`, `put`, and `delete` over a local socket. Snapshots give you a consistent view of the store at one instant. A snapshot never blocks new writes. Release it when you are done. -This is release 0.3.0. It adds point-in-time snapshots and consistent reads. +An optional fast mode skips checksums. It writes faster and reads faster. It +also loses the ability to detect silent data corruption. Each file records its +own mode, so readers always know how to decode it. + +This is release 0.4.0. It adds the checksum-free fast mode. ## Features @@ -26,6 +30,7 @@ This is release 0.3.0. It adds point-in-time snapshots and consistent reads. - Snapshot iterators that stay valid during writes - Range scans with an iterator API - Crash recovery from the write ahead log +- Optional checksum-free fast mode - Local TCP server with a line protocol - Zero runtime dependencies @@ -55,11 +60,11 @@ bin/cinderstore demo ## Demo output The demo loads a product catalog from `fixtures/catalog.csv`. It writes, -scans, flushes, compacts, deletes, snapshots, and reopens a database. The -output is deterministic. +scans, flushes, compacts, deletes, snapshots, and reopens a database. It also +shows the checksum-free fast mode. The output is deterministic. ```text -== Cinderstore 0.3.0 demo == +== Cinderstore 0.4.0 demo == Loaded 24 products from ...\fixtures\catalog.csv Database directory: ...\cinderstore-demo @@ -77,16 +82,16 @@ Database directory: ...\cinderstore-demo 3. Flush memtable to a sorted table tables: 1 (l0: 1, l1: 0), entries: 24 - disk bytes: 1649, memtable bytes: 0 + disk bytes: 1650, memtable bytes: 0 4. Delete 4 products, update 2 products, then flush again tables: 2 (l0: 2, l1: 0), entries: 30 - disk bytes: 1902, memtable bytes: 0 + disk bytes: 1904, memtable bytes: 0 The deleted keys still occupy space in the level-0 tables. 5. Compact merges the tables and drops the deleted keys tables: 1 (l0: 0, l1: 1), entries: 20 - disk bytes: 1384, memtable bytes: 0 + disk bytes: 1385, memtable bytes: 0 The store keeps the newest value for each key. 6. Verify deletes and updates after compaction @@ -106,12 +111,20 @@ Database directory: ...\cinderstore-demo 8. Reopen the database and verify recovery rows after restart: 21 +9. Fast mode skips checksums + checksums: false, tables: 1 + get SKU-0032 => "{\"name\":\"Damper Wrench\",\"price\":18.75,\"stock\":23}" + default reopen reads: 3 rows + Demo complete. ``` Step 7 shows the value of a snapshot. The live store drops SKU-0001 and adds two products. The snapshot still sees the state before those writes. +Step 9 writes a database without checksums. The default reopen reads it with +no extra configuration. Each file records its own checksum mode. + ## Use the library Require the library. @@ -184,6 +197,29 @@ db.flush # Move the memtable into a table. db.compact # Merge tables and drop deleted keys. ``` +### Fast mode + +Fast mode disables checksums. It reduces CPU use on every write and every +read. Use it when you control the storage medium and can tolerate silent +corruption. The write ahead log still frames records. It just skips the CRC32. + +```crystal +config = Cinderstore::DB::Config.new +config.checksums = false +db = Cinderstore::DB.new("data", config) +db.put("forge-hammer", "steel") +``` + +Each table file records its own mode. Each write ahead log records its own +mode. A reader always follows the flag in the file. You can reopen the same +directory in either mode. Old files keep working unchanged. + +Check the mode of the durable data. + +```crystal +db.stats.checksums # => false +``` + ## Command line tool The tool uses a database directory. The default directory is @@ -227,6 +263,15 @@ bin/cinderstore server --db data --port 7654 Run `bin/cinderstore help` for the full list of commands. +Pass `--no-checksums` to any writing command for fast mode. The flag applies +to the server, writes, deletes, flushes, and compactions. + +```console +bin/cinderstore server --db data --port 7654 --no-checksums +``` + +Read-only commands follow the mode stored in each file. They need no flag. + ## Wire protocol The server listens on `127.0.0.1:7654` by default. Commands are lines of @@ -275,12 +320,17 @@ A write goes to two places at once. The default mode fsyncs after every write. Set `sync_writes` to `false` for faster, less durable writes. +The default mode also computes a CRC32 for every record. Set `checksums` to +`false` to skip that work. The WAL header records the mode of that file. + ### Flush When the memory table grows past its limit, the database freezes it. A new memory table starts. A background task writes the frozen table to a sorted file. The old log is deleted only after the file is durable. +The new table follows the `checksums` setting. Its footer records that mode. + ### Compaction Level-0 tables may overlap. Compaction merges every table into a fresh, @@ -316,6 +366,9 @@ On open, the database replays the write ahead log into the memory table. Recovery is idempotent. A torn tail is detected by its CRC32 and skipped. The manifest lists every table. Orphan files from a crash are removed. +In fast mode, a torn tail is not detected. Recovery reads the mode from the +WAL header and skips verification for that file. + ## On-disk format Tables use a compact binary format. @@ -323,8 +376,15 @@ Tables use a compact binary format. - Data blocks hold serialized entries. - A block index maps the first key of each block to its offset. - A bloom filter covers every key in the table. -- A footer stores offsets, a version, and a CRC32. -- Each block and each log record carries a CRC32. +- A footer stores offsets, a version, and a flags byte. +- Each block and each log record carries a CRC32 by default. + +The table footer flags byte records the checksum mode. Version 2 files carry +this byte. Version 1 files from earlier releases always used checksums. +Readers accept both versions. + +The write ahead log carries a small header in version 2. The header records +the checksum mode. Version 1 logs have no header and always used checksums. Sequence numbers make versions unique. They are per-write and never reused. @@ -339,11 +399,15 @@ config.memtable_limit = 4_i64 * 1024 * 1024 config.bloom_fpp = 0.01 config.cache_blocks = 512 config.sync_writes = true +config.checksums = true config.l0_compact_threshold = 4 config.compact_on_flush = true db = Cinderstore::DB.new("data", config) ``` +Set `checksums` to `false` for fast mode. The value controls new writes. It +applies to the write ahead log and to every table the database creates. + ## Project layout ```text @@ -359,11 +423,11 @@ spec/ Test suite ## Test status -The suite runs with `crystal spec`. It has 97 examples. All pass on Windows +The suite runs with `crystal spec`. It has 109 examples. All pass on Windows and Linux. It covers the skip list, the memory table, the write ahead log, the bloom filter, and the block cache. It covers the tables, the iterators, -and the database. It covers compaction, durability, snapshots, and the -server protocol. +and the database. It covers compaction, durability, snapshots, the +checksum-free fast mode, and the server protocol. The CI workflow runs on GitHub Actions for Windows and Ubuntu. It checks formatting, runs the suite, runs the demo, and builds the binary. @@ -376,14 +440,13 @@ formatting, runs the suite, runs the demo, and builds the binary. - The server protocol is unencrypted. Use it on localhost only. - Compaction is a full merge. It is correct and simple, not incremental. - No multi-threaded runtime is required. The server uses fibers. +- Fast mode skips checksums. It cannot detect silent data corruption. - Release snapshots before you close the database. ## Roadmap Planned: -- Release 0.2: incremental compaction by level -- Release 0.4: optional checksum-free fast mode - Release 0.5: batch writes and group commit - Release 0.6: secondary indexes @@ -391,6 +454,8 @@ Delivered: - Release 0.3: snapshot iterators and consistent reads. Snapshots give a stable view of the store. Compaction keeps referenced files alive. +- Release 0.4: optional checksum-free fast mode. Each file records its own + checksum mode. Readers decode old and new files without configuration. ## License diff --git a/shard.lock b/shard.lock index c1dbcd2..8a8a3ae 100644 --- a/shard.lock +++ b/shard.lock @@ -1 +1 @@ -version: 0.3.0 +version: 0.4.0 diff --git a/shard.yml b/shard.yml index e335eb5..5225a79 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: cinderstore -version: 0.3.0 +version: 0.4.0 description: An embeddable key/value store built on a log structured merge tree. crystal: ">= 1.10.0" license: Apache-2.0 diff --git a/spec/checksum_spec.cr b/spec/checksum_spec.cr new file mode 100644 index 0000000..a3f11d9 --- /dev/null +++ b/spec/checksum_spec.cr @@ -0,0 +1,248 @@ +require "./spec_helper" + +# Writes a table file with the given checksum mode and returns its meta. +def write_table_with_checksums(path : String, entries : Array(Cinderstore::Entry), checksums : Bool, block_size : Int32 = 4096) + meta = nil + File.open(path, "w") do |io| + writer = Cinderstore::SstableWriter.new(io, 7_i64, block_size, 0.01, checksums) + entries.each do |entry| + writer.add(entry.key, entry.value, entry.seq, entry.alive) + end + meta = writer.finish + end + meta.not_nil! +end + +# Rewrites a version 2 footer as a version 1 footer without the flags byte. +def downgrade_footer_to_v1(path : String) : Nil + size = File.size(path) + File.open(path, "r+") do |f| + f.pos = size - 49 + v2 = Bytes.new(49) + f.read_fully(v2) + footer = Bytes.new(48) + footer.copy_from(v2[0, 8]) + v2[9, 32].each_with_index { |byte, i| footer[8 + i] = byte } + footer[40] = 1_u8 + footer[41] = 0_u8 + footer[42] = 0_u8 + footer[43] = 0_u8 + crc = Cinderstore::Util.crc32(footer[0, 44]) + IO::Memory.new(footer[44, 4]).write_bytes(crc, IO::ByteFormat::LittleEndian) + f.pos = size - 49 + f.write(footer) + f.truncate(size - 1) + end +end + +def collect_entries(reader : Cinderstore::SstableReader) + all = [] of Cinderstore::Entry + reader.block_count.times do |i| + all.concat(reader.load_block_entries(i)) + end + all +end + +# Returns the file offset of the first block's CRC slot for `entry`. +def block_crc_offset(entry : Cinderstore::Entry) : Int32 + body = IO::Memory.new + Cinderstore.write_entry(body, entry) + Cinderstore::Util.varint_len(body.size.to_u64) + body.size +end + +describe Cinderstore do + describe "checksum-free mode" do + it "defaults to checksums enabled" do + Cinderstore::DB::Config.new.checksums.should be_true + end + + it "round trips table entries without checksums" do + dir = Cinderstore::SpecHelpers.tmp_db_path("fast-table") + Dir.mkdir_p(dir) + path = File.join(dir, "000007.sst") + entries = 200.times.map { |i| Cinderstore::Entry.new("key-%04d" % i, i.to_i64, true, "value-#{i}") }.to_a + write_table_with_checksums(path, entries, false, 128) + reader = Cinderstore::SstableReader.new(path, 7_i64, nil) + collect_entries(reader).should eq(entries) + reader.close + end + + it "skips block checksums in a checksum-free table" do + dir = Cinderstore::SpecHelpers.tmp_db_path("fast-table") + Dir.mkdir_p(dir) + path = File.join(dir, "000007.sst") + entry = Cinderstore::Entry.new("key-0000", 1_i64, true, "value-0") + write_table_with_checksums(path, [entry], false) + + File.open(path, "r+") do |f| + f.pos = block_crc_offset(entry) + byte = f.read_byte.not_nil! + f.pos = block_crc_offset(entry) + f.write_byte((byte ^ 0xFF).to_u8) + end + + reader = Cinderstore::SstableReader.new(path, 7_i64, nil) + collect_entries(reader).should eq([entry]) + reader.close + end + + it "detects a corrupt block checksum in a checksummed table" do + dir = Cinderstore::SpecHelpers.tmp_db_path("fast-table") + Dir.mkdir_p(dir) + path = File.join(dir, "000007.sst") + entry = Cinderstore::Entry.new("key-0000", 1_i64, true, "value-0") + write_table_with_checksums(path, [entry], true) + + File.open(path, "r+") do |f| + f.pos = block_crc_offset(entry) + byte = f.read_byte.not_nil! + f.pos = block_crc_offset(entry) + f.write_byte((byte ^ 0xFF).to_u8) + end + + reader = Cinderstore::SstableReader.new(path, 7_i64, nil) + expect_raises(Cinderstore::CorruptDataError) do + reader.load_block_entries(0) + end + reader.close + end + + it "reads a version 1 table from before the mode existed" do + dir = Cinderstore::SpecHelpers.tmp_db_path("fast-table") + Dir.mkdir_p(dir) + path = File.join(dir, "000007.sst") + entries = 100.times.map { |i| Cinderstore::Entry.new("key-%03d" % i, i.to_i64, true, "value-#{i}") }.to_a + write_table_with_checksums(path, entries, true, 64) + downgrade_footer_to_v1(path) + + reader = Cinderstore::SstableReader.new(path, 7_i64, nil) + collect_entries(reader).should eq(entries) + reader.close + end + end + + describe "checksum-free write ahead log" do + it "recovers records written without checksums" do + dir = Cinderstore::SpecHelpers.tmp_db_path("fast-wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, false, false) + writer.append(Cinderstore::Entry.new("a", 1_i64, true, "one")) + writer.append(Cinderstore::Entry.new("b", 2_i64, false, "")) + writer.close + + mem = Cinderstore::MemTable.new + Cinderstore::Wal.recover(path, mem, 0_i64).should eq(2_i64) + mem.get("a").should eq("one") + mem.get("b").should be_nil + mem.get_entry("b").not_nil!.alive.should be_false + end + + it "ignores corruption in a checksum-free log" do + dir = Cinderstore::SpecHelpers.tmp_db_path("fast-wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, false, false) + writer.append(Cinderstore::Entry.new("alpha", 1_i64, true, "1")) + writer.close + + record = Cinderstore::Wal.encode(Cinderstore::Entry.new("alpha", 1_i64, true, "1"), false) + crc_offset = Cinderstore::Wal::HEADER_SIZE + record.size - 4 + bytes = File.read(path).to_slice.dup + bytes[crc_offset] = (bytes[crc_offset] ^ 0xFF).to_u8 + File.open(path, "w") { |f| f.write(bytes) } + + mem = Cinderstore::MemTable.new + Cinderstore::Wal.recover(path, mem, 0_i64).should eq(1_i64) + mem.get("alpha").should eq("1") + end + + it "still verifies a checksummed log that has a header" do + dir = Cinderstore::SpecHelpers.tmp_db_path("fast-wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, false, true) + writer.append(Cinderstore::Entry.new("alpha", 1_i64, true, "1")) + writer.close + + record = Cinderstore::Wal.encode(Cinderstore::Entry.new("alpha", 1_i64, true, "1"), true) + crc_offset = Cinderstore::Wal::HEADER_SIZE + record.size - 4 + bytes = File.read(path).to_slice.dup + bytes[crc_offset] = (bytes[crc_offset] ^ 0xFF).to_u8 + File.open(path, "w") { |f| f.write(bytes) } + + mem = Cinderstore::MemTable.new + Cinderstore::Wal.recover(path, mem, 0_i64).should eq(0_i64) + mem.empty?.should be_true + end + + it "replays an old log without a header" do + dir = Cinderstore::SpecHelpers.tmp_db_path("fast-wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + File.open(path, "w") do |io| + io.write(Cinderstore::Wal.encode(Cinderstore::Entry.new("a", 1_i64, true, "one"), true)) + io.write(Cinderstore::Wal.encode(Cinderstore::Entry.new("b", 2_i64, true, "two"), true)) + end + + mem = Cinderstore::MemTable.new + Cinderstore::Wal.recover(path, mem, 0_i64).should eq(2_i64) + mem.get("a").should eq("one") + mem.get("b").should eq("two") + end + end + + describe "database in checksum-free mode" do + it "survives a restart and is read by a default reopen" do + config = Cinderstore::SpecHelpers.fast_config + config.checksums = false + Cinderstore::SpecHelpers.with_db_path("db-fast") do |path| + db = Cinderstore::DB.new(path, config) + db.put("a", "1") + db.put("b", "2") + db.flush + db.put("c", "3") + db.compact + db.close + + reopened = Cinderstore::DB.new(path) + reopened.get("a").should eq("1") + reopened.get("b").should eq("2") + reopened.get("c").should eq("3") + reopened.scan.map(&.[0]).should eq(%w[a b c]) + reopened.close + end + end + + it "keeps the checksum mode readable across an upgrade" do + config = Cinderstore::SpecHelpers.fast_config + config.checksums = false + Cinderstore::SpecHelpers.with_db_path("db-fast") do |path| + db = Cinderstore::DB.new(path, config) + db.put("a", "1") + db.flush + db.close + + upgraded = Cinderstore::DB.new(path) + upgraded.get("a").should eq("1") + upgraded.put("b", "2") + upgraded.flush + upgraded.compact + upgraded.get("a").should eq("1") + upgraded.get("b").should eq("2") + upgraded.close + end + end + + it "reports the checksum mode in stats" do + config = Cinderstore::SpecHelpers.fast_config + config.checksums = false + Cinderstore::SpecHelpers.with_db("db-faststats", config) do |db, _path| + db.stats.checksums.should be_false + end + Cinderstore::SpecHelpers.with_db("db-cksstats") do |db, _path| + db.stats.checksums.should be_true + end + end + end +end diff --git a/src/cinderstore/db.cr b/src/cinderstore/db.cr index f5707f9..0f3d698 100644 --- a/src/cinderstore/db.cr +++ b/src/cinderstore/db.cr @@ -28,6 +28,9 @@ module Cinderstore property cache_blocks : Int32 = 512 # Fsync after every write when true. property sync_writes : Bool = true + # Write and verify checksums when true. Disable for faster, less + # durable I/O. Readers always follow the flag stored in each file. + property checksums : Bool = true # Number of level-0 tables that trigger compaction. property l0_compact_threshold : Int32 = 4 # Start background compaction after a flush when true. @@ -46,13 +49,15 @@ module Cinderstore getter seq : Int64 getter cache_hits : Int64 getter cache_misses : Int64 + getter checksums : Bool def initialize(@tables : Int32, @l0 : Int32, @l1 : Int32, @entries : Int64, @disk_bytes : Int64, @memtable_bytes : Int64, @wal_bytes : Int64, - @seq : Int64, @cache_hits : Int64, @cache_misses : Int64) + @seq : Int64, @cache_hits : Int64, @cache_misses : Int64, + @checksums : Bool) end - def to_h : Hash(String, Int32 | Int64) + def to_h : Hash(String, Int32 | Int64 | Bool) { "tables" => @tables, "l0" => @l0, @@ -64,6 +69,7 @@ module Cinderstore "seq" => @seq, "cache_hits" => @cache_hits, "cache_misses" => @cache_misses, + "checksums" => @checksums, } end @@ -78,7 +84,8 @@ module Cinderstore io << "memtable bytes: #{@memtable_bytes}\n" io << "wal bytes: #{@wal_bytes}\n" io << "sequence: #{@seq}\n" - io << "cache hits: #{@cache_hits}, misses: #{@cache_misses}" + io << "cache hits: #{@cache_hits}, misses: #{@cache_misses}\n" + io << "checksums: #{@checksums}" end end @@ -286,6 +293,13 @@ module Cinderstore Dir.children(@path).each do |name| wal_bytes += file_size(File.join(@path, name)) if name.ends_with?(WAL_SUFFIX) end + # The mode of the newest table describes the durable data. Before + # any table exists, the session config decides the mode. + checksums = if ref = table_refs.max_by?(&.id) + ref.reader(@block_cache).checksums? + else + @config.checksums + end Stats.new( tables: table_refs.size, l0: @levels[0].size, @@ -297,6 +311,7 @@ module Cinderstore seq: @seq, cache_hits: @block_cache.hits, cache_misses: @block_cache.misses, + checksums: checksums, ) end end @@ -374,7 +389,7 @@ module Cinderstore private def create_active_wal : Nil id = @next_id @next_id += 1 - @wal = Wal::Writer.new(wal_path(id), @config.sync_writes) + @wal = Wal::Writer.new(wal_path(id), @config.sync_writes, @config.checksums) end # ------------------------------------------------------------------ @@ -429,7 +444,7 @@ module Cinderstore @next_id += 1 wal_id = @next_id @next_id += 1 - @wal = Wal::Writer.new(wal_path(wal_id), @config.sync_writes) + @wal = Wal::Writer.new(wal_path(wal_id), @config.sync_writes, @config.checksums) end end @@ -459,7 +474,7 @@ module Cinderstore final_path = table_path(table_id) meta = nil File.open(tmp_path, "w") do |io| - writer = SstableWriter.new(io, table_id, @config.block_size, @config.bloom_fpp) + writer = SstableWriter.new(io, table_id, @config.block_size, @config.bloom_fpp, @config.checksums) mem.each_entry do |entry| writer.add(entry.key, entry.value, entry.seq, entry.alive) end @@ -557,7 +572,7 @@ module Cinderstore id end io = File.open(File.join(@path, "#{Util.file_stem(current_id)}#{TMP_SUFFIX}"), "w") - writer = SstableWriter.new(io.not_nil!, current_id, @config.block_size, @config.bloom_fpp) + writer = SstableWriter.new(io.not_nil!, current_id, @config.block_size, @config.bloom_fpp, @config.checksums) end writer.not_nil!.add(entry.key, entry.value, entry.seq, true) written += 1 diff --git a/src/cinderstore/demo.cr b/src/cinderstore/demo.cr index 206635f..a6242e4 100644 --- a/src/cinderstore/demo.cr +++ b/src/cinderstore/demo.cr @@ -103,6 +103,30 @@ module Cinderstore puts " rows after restart: #{count}" reopened.close puts "" + + puts "9. Fast mode skips checksums" + fast_path = File.join(Dir.tempdir, "cinderstore-demo-fast") + FileUtils.rm_rf(fast_path) if File.exists?(fast_path) + fast_config = DB::Config.new + fast_config.sync_writes = false + fast_config.checksums = false + fast_db = DB.new(fast_path, fast_config) + fast_db.put("SKU-0030", %({"name":"Tinder Box Brass","price":12.50,"stock":28})) + fast_db.put("SKU-0031", %({"name":"Coal Scuttle Iron","price":39.90,"stock":12})) + fast_db.flush + fast_db.put("SKU-0032", %({"name":"Damper Wrench","price":18.75,"stock":23})) + fast_db.compact + puts " checksums: #{fast_db.stats.checksums}, tables: #{fast_db.stats.tables}" + puts " get SKU-0032 => #{fast_db.get("SKU-0032").inspect}" + fast_db.close + # A default reopen reads the fast-mode files. Each file records its + # own checksum mode, so readers need no configuration. + reopened_fast = DB.new(fast_path) + puts " default reopen reads: #{reopened_fast.scan.size} rows" + reopened_fast.close + FileUtils.rm_rf(fast_path) + puts "" + puts "Demo complete." 0 end diff --git a/src/cinderstore/table.cr b/src/cinderstore/table.cr index bcd8bd3..015fa80 100644 --- a/src/cinderstore/table.cr +++ b/src/cinderstore/table.cr @@ -35,14 +35,21 @@ module Cinderstore # table that cannot contain a key. The footer stores offsets and a CRC32 # over the entire footer. class SstableWriter - MAGIC = 0x43494E4445525F31_u64 - VERSION = 1_u32 - FOOTER_SIZE = 48 + MAGIC = 0x43494E4445525F31_u64 + VERSION = 2_u32 + VERSION_V1 = 1_u32 + VERSION_V2 = 2_u32 + FOOTER_SIZE_V1 = 48 + FOOTER_SIZE_V2 = 49 + FOOTER_SIZE = FOOTER_SIZE_V2 + # Version 2 footer flag. Cleared when checksums are disabled. + FLAG_CHECKSUMS = 0x01_u8 @io : IO @id : Int64 @block_size : Int32 @fpp : Float64 + @checksums : Bool @pending : IO::Memory @pending_keys : Array(String) @keys : Array(String) @@ -51,7 +58,8 @@ module Cinderstore @first : String? @last : String? - def initialize(io : IO, @id : Int64, @block_size : Int32 = 4096, @fpp : Float64 = 0.01) + def initialize(io : IO, @id : Int64, @block_size : Int32 = 4096, @fpp : Float64 = 0.01, + @checksums : Bool = true) @io = io @pending = IO::Memory.new @pending_keys = [] of String @@ -96,6 +104,7 @@ module Cinderstore footer = IO::Memory.new footer.write_bytes(MAGIC, IO::ByteFormat::LittleEndian) + footer.write_byte(@checksums ? FLAG_CHECKSUMS : 0_u8) footer.write_bytes(index_offset.to_u64, IO::ByteFormat::LittleEndian) footer.write_bytes(index_length.to_u64, IO::ByteFormat::LittleEndian) footer.write_bytes(bloom_offset.to_u64, IO::ByteFormat::LittleEndian) @@ -103,7 +112,11 @@ module Cinderstore footer.write_bytes(VERSION, IO::ByteFormat::LittleEndian) footer_bytes = footer.to_slice @io.write(footer_bytes) - @io.write_bytes(Util.crc32(footer_bytes), IO::ByteFormat::LittleEndian) + if @checksums + @io.write_bytes(Util.crc32(footer_bytes), IO::ByteFormat::LittleEndian) + else + @io.write_bytes(0_u32, IO::ByteFormat::LittleEndian) + end @io.flush TableMeta.new(@id, @first.not_nil!, @last.not_nil!, @count) @@ -114,7 +127,11 @@ module Cinderstore offset = @io.pos Util.write_varint(@io, data.size.to_u64) @io.write(data) - @io.write_bytes(Util.crc32(data), IO::ByteFormat::LittleEndian) + if @checksums + @io.write_bytes(Util.crc32(data), IO::ByteFormat::LittleEndian) + else + @io.write_bytes(0_u32, IO::ByteFormat::LittleEndian) + end total = Util.varint_len(data.size.to_u64) + data.size + 4 @index << BlockIndexEntry.new(@pending_keys.first, offset.to_u64, total.to_u64) @pending = IO::Memory.new @@ -131,6 +148,7 @@ module Cinderstore @size : Int64 @index : Array(BlockIndexEntry) @bloom : BloomFilter? + @checksums : Bool @closed : Bool @index_offset : UInt64 @index_length : UInt64 @@ -142,6 +160,7 @@ module Cinderstore @size = @file.size @index = [] of BlockIndexEntry @bloom = nil + @checksums = true @closed = false @index_offset = 0_u64 @index_length = 0_u64 @@ -160,6 +179,11 @@ module Cinderstore @file_id end + # Returns whether this table's blocks carry checksums. + def checksums? : Bool + @checksums + end + def block_count : Int32 @index.size end @@ -220,32 +244,52 @@ module Cinderstore end data = Bytes.new(data_len) @file.read_fully(data) - stored_crc = @file.read_bytes(UInt32, IO::ByteFormat::LittleEndian) - actual_crc = Util.crc32(data) - raise CorruptDataError.new("block checksum mismatch in #{@path}") unless stored_crc == actual_crc + if @checksums + stored_crc = @file.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + actual_crc = Util.crc32(data) + raise CorruptDataError.new("block checksum mismatch in #{@path}") unless stored_crc == actual_crc + else + @file.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + end data end private def read_footer : Nil - raise CorruptDataError.new("file too small: #{@path}") if @size < SstableWriter::FOOTER_SIZE - @file.pos = @size - SstableWriter::FOOTER_SIZE - footer = Bytes.new(SstableWriter::FOOTER_SIZE) + raise CorruptDataError.new("file too small: #{@path}") if @size < SstableWriter::FOOTER_SIZE_V1 + # The version field sits directly before the footer CRC in both the + # v1 and v2 layouts, so it is always found at size - 8. + @file.pos = @size - 8 + version = @file.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + footer_size = case version + when SstableWriter::VERSION_V1 then SstableWriter::FOOTER_SIZE_V1 + when SstableWriter::VERSION_V2 then SstableWriter::FOOTER_SIZE_V2 + else + raise CorruptDataError.new("unsupported version #{version} in #{@path}") + end + raise CorruptDataError.new("file too small: #{@path}") if @size < footer_size + + @file.pos = @size - footer_size + footer = Bytes.new(footer_size) @file.read_fully(footer) - crc_bytes = footer[SstableWriter::FOOTER_SIZE - 4, 4] + crc_bytes = footer[footer_size - 4, 4] stored_crc = IO::Memory.new(crc_bytes).read_bytes(UInt32, IO::ByteFormat::LittleEndian) - actual_crc = Util.crc32(footer[0, SstableWriter::FOOTER_SIZE - 4]) - raise CorruptDataError.new("footer checksum mismatch in #{@path}") unless stored_crc == actual_crc + footer_body = footer[0, footer_size - 4] + @checksums = version == SstableWriter::VERSION_V1 || + (footer[8] & SstableWriter::FLAG_CHECKSUMS) != 0 + if @checksums + actual_crc = Util.crc32(footer_body) + raise CorruptDataError.new("footer checksum mismatch in #{@path}") unless stored_crc == actual_crc + end io = IO::Memory.new(footer) magic = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) raise CorruptDataError.new("bad magic in #{@path}") unless magic == SstableWriter::MAGIC + io.read_byte if version == SstableWriter::VERSION_V2 @index_offset = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) @index_length = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) @bloom_offset = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) @bloom_length = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) - version = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) - raise CorruptDataError.new("unsupported version #{version} in #{@path}") unless version == SstableWriter::VERSION end private def read_index : Nil diff --git a/src/cinderstore/version.cr b/src/cinderstore/version.cr index 4b9d872..3b31a8c 100644 --- a/src/cinderstore/version.cr +++ b/src/cinderstore/version.cr @@ -1,3 +1,3 @@ module Cinderstore - VERSION = "0.3.0" + VERSION = "0.4.0" end diff --git a/src/cinderstore/wal.cr b/src/cinderstore/wal.cr index 04735b6..157968d 100644 --- a/src/cinderstore/wal.cr +++ b/src/cinderstore/wal.cr @@ -4,16 +4,36 @@ module Cinderstore # The WAL frames each entry with a length, a payload, and a CRC32. We # fsync after every append when durability is enabled. A torn write at # the tail is truncated during recovery. + # + # A WAL can skip checksums. Such a file starts with a header that records + # the mode. Recovery reads the header and skips verification for records + # in that file. Older WALs without a header always carried checksums. class Wal + # Version 2 header magic. The leading zero byte can never begin a v1 + # record, because every v1 record starts with a non-zero length varint. + HEADER_MAGIC = Bytes[0x00_u8, 0x43_u8, 0x49_u8, 0x4E_u8] + HEADER_SIZE = (HEADER_MAGIC.size + 1).to_i64 + FLAG_CHECKSUMS = 0x01_u8 + + # Returns the version 2 header for a writer. + def self.header(checksums : Bool) : Bytes + Bytes[HEADER_MAGIC[0], HEADER_MAGIC[1], HEADER_MAGIC[2], HEADER_MAGIC[3], + checksums ? FLAG_CHECKSUMS : 0_u8] + end + # Serializes an entry into a framed record. - def self.encode(entry : Entry) : Bytes + def self.encode(entry : Entry, checksums : Bool = true) : Bytes body = IO::Memory.new Cinderstore.write_entry(body, entry) payload = body.to_slice framed = IO::Memory.new Util.write_varint(framed, payload.size.to_u64) framed.write(payload) - framed.write_bytes(Util.crc32(payload), IO::ByteFormat::LittleEndian) + if checksums + framed.write_bytes(Util.crc32(payload), IO::ByteFormat::LittleEndian) + else + framed.write_bytes(0_u32, IO::ByteFormat::LittleEndian) + end framed.to_slice end @@ -27,15 +47,20 @@ module Cinderstore return max_seq unless File.exists?(path) File.open(path, "r") do |io| + checksums = read_header(io) loop do break if io.pos >= io.size begin body_len = Util.read_varint(io).to_i body = Bytes.new(body_len) io.read_fully(body) - stored_crc = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) - actual_crc = Util.crc32(body) - raise CorruptDataError.new("wal checksum mismatch") unless stored_crc == actual_crc + if checksums + stored_crc = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + actual_crc = Util.crc32(body) + raise CorruptDataError.new("wal checksum mismatch") unless stored_crc == actual_crc + else + io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + end entry = Cinderstore.read_entry(IO::Memory.new(body)) if entry.alive mem.put(entry.key, entry.value, entry.seq) @@ -53,18 +78,37 @@ module Cinderstore max_seq end + # Consumes a version 2 header when present. Returns whether the records + # that follow carry checksums. Files without a header are v1 and always + # carry checksums. + private def self.read_header(io : IO) : Bool + first = io.read_byte + if first == HEADER_MAGIC[0] && io.size >= HEADER_SIZE + rest = Bytes.new(HEADER_MAGIC.size - 1) + io.read_fully(rest) + if rest[0] == HEADER_MAGIC[1] && rest[1] == HEADER_MAGIC[2] && rest[2] == HEADER_MAGIC[3] + flags = io.read_byte.not_nil! + return (flags & FLAG_CHECKSUMS) != 0 + end + end + io.pos = 0 + true + end + # Appends framed records to a WAL file. class Writer getter path : String getter size : Int64 + getter checksums : Bool - def initialize(@path : String, @sync_each_write : Bool = true) + def initialize(@path : String, @sync_each_write : Bool = true, @checksums : Bool = true) @file = File.open(@path, "a+") @size = @file.size + write_header if @size == 0 end def append(entry : Entry) : Nil - record = Wal.encode(entry) + record = Wal.encode(entry, @checksums) @file.write(record) @file.flush @file.fsync if @sync_each_write @@ -75,6 +119,12 @@ module Cinderstore @file.fsync @file.close end + + private def write_header : Nil + @file.write(Wal.header(@checksums)) + @file.flush + @size = Wal::HEADER_SIZE + end end end end diff --git a/src/cli.cr b/src/cli.cr index 8a91070..c55fc1b 100644 --- a/src/cli.cr +++ b/src/cli.cr @@ -39,18 +39,20 @@ module Cinderstore db_path = "cinderstore-data" host = "127.0.0.1" port = 7654 + checksums = true help = false parser = OptionParser.new parser.on("--db PATH", "Database directory") { |v| db_path = v } parser.on("--host HOST", "Bind address") { |v| host = v } parser.on("--port PORT", "Listen on this port") { |v| port = v.to_i } + parser.on("--no-checksums", "Skip checksums for faster writes") { checksums = false } parser.on("-h", "--help", "Show this help") { help = true } parser.parse(rest) if help puts parser return 0 end - db = DB.new(db_path) + db = DB.new(db_path, build_config(checksums)) server = Server.new(db, host, port) server.run db.close @@ -61,11 +63,13 @@ module Cinderstore db_path = "cinderstore-data" key = "" value = "" + checksums = true help = false parser = OptionParser.new parser.on("--db PATH", "Database directory") { |v| db_path = v } parser.on("--key KEY", "Key to write") { |v| key = v } parser.on("--value VALUE", "Value to write") { |v| value = v } + parser.on("--no-checksums", "Skip checksums for faster writes") { checksums = false } parser.on("-h", "--help", "Show this help") { help = true } parser.parse(rest) if help @@ -73,7 +77,7 @@ module Cinderstore return 0 end raise Error.new("missing --key") if key.empty? - db = DB.new(db_path) + db = DB.new(db_path, build_config(checksums)) begin db.put(key, value) puts "ok" @@ -114,10 +118,12 @@ module Cinderstore private def run_delete(rest : Array(String)) : Int32 db_path = "cinderstore-data" key = "" + checksums = true help = false parser = OptionParser.new parser.on("--db PATH", "Database directory") { |v| db_path = v } parser.on("--key KEY", "Key to delete") { |v| key = v } + parser.on("--no-checksums", "Skip checksums for faster writes") { checksums = false } parser.on("-h", "--help", "Show this help") { help = true } parser.parse(rest) if help @@ -125,7 +131,7 @@ module Cinderstore return 0 end raise Error.new("missing --key") if key.empty? - db = DB.new(db_path) + db = DB.new(db_path, build_config(checksums)) begin db.delete(key) puts "ok" @@ -166,16 +172,18 @@ module Cinderstore private def run_simple(action : String, rest : Array(String)) : Int32 db_path = "cinderstore-data" + checksums = true help = false parser = OptionParser.new parser.on("--db PATH", "Database directory") { |v| db_path = v } + parser.on("--no-checksums", "Skip checksums for faster writes") { checksums = false } parser.on("-h", "--help", "Show this help") { help = true } parser.parse(rest) if help puts parser return 0 end - db = DB.new(db_path) + db = DB.new(db_path, build_config(checksums)) begin case action when "stats" @@ -209,6 +217,12 @@ module Cinderstore Demo.run(db_path, fixture) end + private def build_config(checksums : Bool) : DB::Config + config = DB::Config.new + config.checksums = checksums + config + end + private def print_help : Nil puts <<-HELP Cinderstore #{VERSION} - an embeddable log structured merge tree.