diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6148b5..cb2b9e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Check out the repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Crystal uses: crystal-lang/install-crystal@v1 diff --git a/.gitignore b/.gitignore index 006c2aa..ff33f72 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /lib/ /.shards/ +/.crystal/ # Crystal *.dwarf *.dwarf.* diff --git a/README.md b/README.md index cc17337..991d4ce 100644 --- a/README.md +++ b/README.md @@ -3,37 +3,70 @@ [![CI](https://github.com/DanielCuevas1208/cinderstore/actions/workflows/ci.yml/badge.svg)](https://github.com/DanielCuevas1208/cinderstore/actions/workflows/ci.yml) Cinderstore is an embeddable key and value store. It is built on a log -structured merge tree (LSM tree). It is written in Crystal and uses only the -Crystal standard library. +structured merge tree. It is written in Crystal and uses only the Crystal +standard library. The store keeps a write ahead log for durability. It flushes memory to sorted files. It merges those files during compaction. It recovers all data after a -restart. It serves `get`, `put`, and `delete` over a local socket. +restart. It serves `get`, `put`, `delete`, `scan`, and `query` over a local socket. Snapshots give you a consistent view of the store at one instant. A snapshot never blocks new writes. Release it when you are done. -This is release 0.4.0. It adds level-based compaction. Tables cascade from -level 0 to deeper levels, and each level stays within its size target. +Secondary indexes map derived keys to primary keys. The store keeps each +index in sync with every write. Query an index to find the primary keys +behind one value. + +This is release 0.12.0. It streams primary-key scans over the local protocol. +It adds bounded range scans and prefix scans without materializing result arrays. + +Release 0.10 added prefix scans. A prefix scan returns live key/value pairs +whose keys start with the prefix bytes. + +Prefix scans return arrays or streaming iterators. They use the existing +ordered merge path. Database iterators own snapshots. Snapshot iterators borrow +snapshots. + +Release 0.9 added streaming iterators for primary-key range scans. A scan +iterator reads a snapshot. It yields live key/value pairs one at a time. +You can stop early and close it. + +Release 0.8 added streaming iterators for secondary index queries. A query +iterator yields primary keys one at a time. The server streams query results +over the socket. + +Release 0.7 added secondary indexes. An index writes into a reserved part +of the log structured merge tree. The write ahead log keeps the index +atomic with the data. Compaction drops stale index entries. ## Features - Memory table with ordered writes - Durable write ahead log with CRC32 framing +- Atomic batch writes in one log record +- Group commit that shares one fsync between writers +- Optional checksums for higher write throughput - Sorted tables with a block index and a bloom filter - Block cache for fast repeated reads - Level-based compaction with size targets per level - Background flush and compaction - Point-in-time snapshots with consistent reads - Snapshot iterators that stay valid during writes -- Range scans with an iterator API +- Streaming primary-key range scans +- Streaming primary-key prefix scans +- Streaming primary-key scans over the local protocol +- Secondary indexes with automatic maintenance +- JSON field indexes with one API call +- Exact and range queries over an index +- Streaming iterators for exact and range index queries - Crash recovery from the write ahead log +- Read-only storage verification for tables and write ahead logs - Local TCP server with a line protocol - Zero runtime dependencies ## Quick start -You need Crystal 1.10 or newer. +You need Crystal 1.21 or newer. ```console crystal spec @@ -54,14 +87,26 @@ Run the demo. bin/cinderstore demo ``` +Run the demo in fast mode. + +```console +bin/cinderstore demo --no-checksums +``` + +Run the wire example. + +```console +crystal run examples/server_demo.cr +``` + ## Demo output The demo loads a product catalog from `fixtures/catalog.csv`. It writes, -scans, flushes, compacts, deletes, snapshots, and reopens a database. The -output is deterministic. +scans, flushes, compacts, deletes, snapshots, and reopens a database. It +creates secondary indexes and queries them. The output is deterministic. ```text -== Cinderstore 0.4.0 demo == +== Cinderstore 0.12.0 demo == Loaded 24 products from ...\fixtures\catalog.csv Database directory: ...\cinderstore-demo @@ -79,16 +124,16 @@ Database directory: ...\cinderstore-demo 3. Flush memtable to a sorted table tables: 1 (l0: 1, l1: 0), entries: 24 - disk bytes: 1649, memtable bytes: 0 + disk bytes: 1653, memtable bytes: 0 4. Delete 4 products, update 2 products, then flush again tables: 2 (l0: 2, l1: 0), entries: 30 - disk bytes: 1902, memtable bytes: 0 + disk bytes: 1910, memtable bytes: 0 The deleted keys still occupy space in the level-0 tables. 5. Compact merges the tables and drops the deleted keys tables: 1 (l0: 0, l1: 1), entries: 20 - disk bytes: 1384, memtable bytes: 0 + disk bytes: 1388, memtable bytes: 0 The store keeps the newest value for each key. 6. Verify deletes and updates after compaction @@ -113,6 +158,51 @@ Database directory: ...\cinderstore-demo after compact_levels: [0, 0, 1] all 100 rows still readable +10. Fast mode skips checksums on new writes + checksummed: wal 5231 bytes, disk 5692 bytes + fast mode: wal 4943 bytes, disk 5548 bytes + all 72 rows readable after a fast-mode restart + +11. Batch writes and group commit + one batch wrote 24 rows as 1 write operation + sequence after the batch: 24 + all 24 rows recovered after a restart + + concurrent writes: 200, durability commits: 25 + all 200 rows recovered after a restart + +12. Secondary indexes track derived views + created indexes on name, price, and stock + query by-name "Ash Rake Forged" => SKU-0010 + query by-price "14.0" => SKU-0010 + query by-stock "31" => SKU-0010 + stock range 20 to 30 => SKU-0004, SKU-0023, SKU-0011, SKU-0003, SKU-0022, SKU-0006 + update SKU-0010 price to 15.50, then flush and compact + query by-price "14.0" => (none) + query by-price "15.5" => SKU-0010 + delete SKU-0011, then flush and compact + query by-name "Coal Shovel Small" => (none) + +13. Streaming iterators for secondary index queries + streamed by-price "15.5" => SKU-0010 + stock range 20 to 30, first 3 => SKU-0004, SKU-0023, SKU-0003 + snapshot stream "Ember Tray Brass" => SKU-0012 + live query after the delete => (none) + reopen and query by-stock "31" => SKU-0010 + +14. Streaming primary-key range scans + primary range SKU-0010 to SKU-0016 => SKU-0010, SKU-0013, SKU-0014, SKU-0015 + snapshot still sees SKU-0010 => SKU-0010, SKU-0013, SKU-0014, SKU-0015 + +15. Prefix scans group keys + live prefix SKU-001 => SKU-0013, SKU-0014, SKU-0015 + +16. Read-only storage verification + valid: true + tables checked: 1, WAL files checked: 1 + table entries: 92, WAL records: 1 + bytes checked: + Demo complete. ``` @@ -123,6 +213,50 @@ Step 9 shows the value of leveled compaction. Five flushes create five level-0 tables. `compact_levels` merges them into a fresh level-2 table, so the store stays tidy as it grows. +Step 10 shows the value of fast mode. It writes the same 72 rows twice. Fast +mode skips the CRC32 on every log record and every table block. Its log is +288 bytes smaller, and its table is 144 bytes smaller. The data still round +trips after a restart. + +Step 11 shows the value of batches and group commit. One batch writes 24 rows +in a single log record. Eight writers share one durability sync per round. +They write 200 rows with 25 syncs instead of 200. The store still recovers +every row after a restart. + +Step 12 shows the value of secondary indexes. Three JSON indexes cover the +catalog. A price update moves SKU-0010 between index keys. A delete removes +SKU-0011 from its indexes. The store recovers every index after a restart. + +Step 13 shows the value of streaming iterators. An exact query streams its +matches one at a time. A range query stops early after three rows. A +snapshot stream still sees SKU-0012 after a delete removes it. The store +recovers the index after a restart. + +Step 14 shows bounded primary-key streaming. The iterator returns live pairs +in key order. The database iterator owns its snapshot until `close`. +The snapshot iterator borrows its snapshot. A delete does not change its view. + +Step 15 shows prefix scans. The scan starts at the prefix and stops at its +byte successor. The iterator filters the range when no successor exists. + +Step 16 shows storage verification. The check reads every persistent table +and log. It reports corruption and metadata mismatches without repair. + +## Wire demo + +The wire demo starts a local server, writes two values, reads them, deletes one, and streams a prefix scan. + +```text +PUT forge-hammer => OK +PUT forge-tongs => OK +GET forge-hammer => VALUE steel 1.5kg +GET missing => NOT_FOUND +DEL forge-tongs => OK +SCANPREFIX => ROW forge-hammer steel 1.5kg +SCANPREFIX => END +SHUTDOWN => BYE +``` + ## Use the library Require the library. @@ -144,6 +278,31 @@ end db.close ``` +### Write a batch + +A batch applies many changes at once. All changes succeed together, or none +do. A batch is one write ahead log record. A crash cannot apply half of it. + +```crystal +batch = Cinderstore::Batch.new +batch.put("bellows-copper", "in stock") +batch.put("chimney-brush", "sold out") +batch.delete("ash-rake") +db.write(batch) +``` + +Build a batch in a block. + +```crystal +db.write do |b| + b.put("bellows-copper", "in stock") + b.delete("ash-rake") +end +``` + +A later operation in the batch overrides an earlier one. The operations use +consecutive sequence numbers in queue order. + ### Read a snapshot A snapshot is a consistent view at one instant. Take a snapshot, read it, @@ -188,6 +347,135 @@ db.each("SKU-0100", "SKU-0200") do |key, value| end ``` +### Stream a primary-key range + +Stream live pairs from a half-open primary-key range. The iterator yields one +pair at a time. + +```crystal +iter = db.scan_iter("SKU-0100", "SKU-0200") +while pair = iter.next? + process(pair[0], pair[1]) +end +iter.close +``` + +Close a database iterator when you finish. It owns the snapshot it reads. + +Snapshot iterators borrow the snapshot. Closing one does not release the +snapshot. + +```crystal +db.snapshot do |snap| + iter = snap.scan_iter("SKU-0100", "SKU-0200") + iter.each { |key, value| process(key, value) } + iter.close +end +``` + +### Scan a key prefix + +Read every live key that starts with a prefix. + +```crystal +pairs = db.scan_prefix("SKU-001") +pairs.each { |key, value| process(key, value) } +``` + +Stream a prefix when the result can grow. + +```crystal +iter = db.scan_prefix_iter("SKU-001") +iter.each { |key, value| process(key, value) } +iter.close +``` + +An empty prefix reads every live key. A negative limit means no limit. + +### Use a secondary index + +Create an index with an extractor. The block gets the primary key and its +value. It returns the index keys for that value. + +```crystal +db.create_index("by-builder") do |key, value| + [extract_builder(value)] +end +``` + +For JSON values, index one field with a name and a field name. A scalar +field yields one index key. An array field yields one key per element. + +```crystal +db.create_json_index("by-price", "price") +db.create_json_index("by-name", "name") +``` + +Every put and delete updates each index. A value change moves its primary +key between index keys. A delete removes the key from every index. + +Find the primary keys behind one index key. + +```crystal +db.query("by-price", "14.0") # => ["SKU-0010"] +``` + +Find the primary keys behind a range of index keys. The range is +`[start, finish)`. + +```crystal +db.query_range("by-price", "10", "20") # => ["SKU-0022", "SKU-0010"] +``` + +Query a snapshot for a consistent result. + +```crystal +db.snapshot do |snap| + snap.query("by-name", "Ash Rake Forged") +end +``` + +A query reads the stored index entries. It works without a registered +index in the current process. Index keys sort by byte value, like primary +keys. + +### Stream the matches of a query + +Stream the matches of an exact query. Each call to `next?` returns the +next primary key. Call `close` when you are done. + +```crystal +iter = db.query_iter("by-price", "15.5") +while key = iter.next? + process(key) +end +iter.close +``` + +The iterator reads a snapshot it owns. It stays consistent while the store +keeps writing. Closing it releases the snapshot. + +Stream a range the same way. The range is `[start, finish)`. + +```crystal +iter = db.query_range_iter("by-stock", "20", "30") +iter.each { |key| process(key) } +iter.close +``` + +A snapshot exposes the same iterators. Close a snapshot iterator before +you release the snapshot. Closing it never releases the snapshot. + +```crystal +db.snapshot do |snap| + snap.query_iter("by-name", "Ash Rake Forged").each { |key| process(key) } + snap.query_range_iter("by-price", "10", "20").each { |key| process(key) } +end +``` + +Streaming avoids building a result array. A large query keeps a small +memory footprint. You can stop reading at any time. + Flush and compact explicitly. ```crystal @@ -203,6 +491,22 @@ pass touches only two levels. It returns true when any table moved. moved = db.compact_levels ``` +### Verify storage + +Check persistent files without changing them. + +```crystal +report = db.verify +raise "storage check failed" unless report.valid? +puts report +``` + +The report checks table blocks, bloom filters, WAL records, and manifest +metadata. It returns errors instead of changing files. + +Use `DB.verify("data")` when the database is closed. This form does not open, +recover, or remove files. + ## Command line tool The tool uses a database directory. The default directory is @@ -232,18 +536,54 @@ Scan a range. bin/cinderstore scan --start a --finish z --limit 100 ``` +Scan a key prefix. + +```console +bin/cinderstore scan --prefix SKU-001 --limit 10 +``` + +Query a secondary index. + +```console +bin/cinderstore query --index by-price --key 14.0 +``` + +Query an index range. + +```console +bin/cinderstore query --index by-stock --start 20 --finish 30 +``` + Show counters. ```console bin/cinderstore stats ``` +Verify persistent files. + +```console +bin/cinderstore verify --db data +``` + Start the local server. ```console bin/cinderstore server --db data --port 7654 ``` +Start the server with JSON field indexes. + +```console +bin/cinderstore server --db data --port 7654 --index by-price:price +``` + +Run the server in fast mode. + +```console +bin/cinderstore server --db data --port 7654 --no-checksums +``` + Run `bin/cinderstore help` for the full list of commands. ## Wire protocol @@ -254,25 +594,36 @@ newlines. Values must not contain newlines. | Command | Meaning | | --- | --- | +| `SCANPREFIX prefix limit` | List keys with a prefix | | `PUT key value` | Write a value | | `GET key` | Read a value | | `DEL key` | Delete a key | | `SCAN start finish limit` | List a range | +| `QUERY index index_key` | Query one index key | +| `QUERYRANGE index start finish` | Query an index range | | `STATS` | Show counters | | `PING` | Check the server | | `FLUSH` | Flush the memtable | | `COMPACT` | Compact the tables | | `SHUTDOWN` | Stop the server | -The server answers with one line per command. +The server returns one response line or one streamed response. - `OK` for a successful write - `VALUE value` for a read - `NOT_FOUND` for a missing key - `ROW key value` for each scan result, then `END` +- `ROW key` for each index query result, then `END` - `STATS {...}` for the counters - `ERR message` for an error +A `SCAN` command streams live key/value pairs in `[start, finish)`. A +`SCANPREFIX` command streams live pairs whose keys start with `prefix`. +A missing limit returns all rows. A zero limit returns no rows. + +`QUERY` and `QUERYRANGE` stream primary keys. The server writes each row +as the iterator produces it, then writes `END`. + Use a tool such as `nc` or the included example client. ```console @@ -291,8 +642,17 @@ A write goes to two places at once. 1. Append the entry to the write ahead log. 2. Insert the entry into the memory table. -The default mode fsyncs after every write. Set `sync_writes` to `false` for -faster, less durable writes. +The default mode syncs each commit group. Concurrent writers share one +group, so one fsync makes several writes durable. Set `sync_writes` to +`false` for faster, less durable writes. Set `checksums` to `false` to skip +the CRC32 on new data. + +A batch follows the same path. Its entries become one log record. Group +commit still applies, so a batch waits for the same shared fsync. + +A secondary index rides the same path. A write that changes an index appends +one batch record. The record holds the primary entry and its index entries. +The log keeps them atomic. A crash drops the whole record or nothing. ### Flush @@ -324,6 +684,13 @@ merges one level at a time. The background task uses `compact_levels`. Compaction keeps a table file on disk while a snapshot references it. The file is deleted only after the last snapshot releases it. +### Wire reads + +Each scan command opens one database iterator. The iterator owns one snapshot. +The server writes each row as soon as the iterator returns it. + +A limit stops the iterator early. The server closes the iterator after END. + ### Snapshots A snapshot is an immutable view of the store at one instant. Creation copies @@ -336,6 +703,31 @@ can replace those files without breaking the snapshot. Snapshots never block writes. Reads over a snapshot use the same merge path as normal reads. Release a snapshot when you are done with it. +### Primary-key scans + +A primary-key scan uses this merge path. It yields live pairs in key order. +A database-level scan owns its snapshot. A snapshot-level scan borrows its +snapshot. + +### Secondary indexes + +An index entry is an ordinary entry in a reserved key namespace. The entry +key holds the index name, the index key, and the primary key. The namespace +uses the NUL byte, so it sorts before every user key. + +The store maintains each index during a write. It reads the previous value +of the primary key. It runs the extractor on the old and the new value. A +new index key becomes a live entry. A removed index key becomes a tombstone. + +The write ahead log keeps the primary entry and the index entries in one +batch record. Recovery restores both together. A newer tombstone supersedes +a stale entry, exactly as it does for primary keys. Compaction drops the +stale entries. + +A query scans the index range in the merge path. It returns primary keys in +index key order, then primary key order. User reads never see the index +namespace. Scans, snapshots, and counters skip it. + ### Read path A read merges the memory table, any frozen table, and all sorted tables. The @@ -349,6 +741,15 @@ On open, the database replays the write ahead log into the memory table. Recovery is idempotent. A torn tail is detected by its CRC32 and skipped. The manifest lists every table. Orphan files from a crash are removed. +### Verification + +`DB#verify` waits for a flush, then blocks writes during the check. It uses +fresh readers, so cached blocks do not hide file corruption. Fast-mode files +still allow layout checks, but they cannot expose silent bit rot. + +The check compares each table with its manifest entry. It verifies every WAL +record and checksum. The CLI returns exit code 1 when the report is invalid. + ## On-disk format Tables use a compact binary format. @@ -356,11 +757,24 @@ Tables use a compact binary format. - Data blocks hold serialized entries. - A block index maps the first key of each block to its offset. - A bloom filter covers every key in the table. -- A footer stores offsets, a version, and a CRC32. -- Each block and each log record carries a CRC32. +- A footer stores offsets, a version, a flag, and a CRC32. +- Each block carries a CRC32 unless the flag turns it off. + +The write ahead log starts with a header. The header holds a magic value and +a flag byte. The flag says whether records carry checksums and a kind byte. +The kind byte marks a single entry or a batch. The reader detects both +settings from the file itself. + +Files from earlier releases have no header, or no kind byte. Their records +always carry checksums. The reader handles every layout, so an upgrade does +not lose data. Sequence numbers make versions unique. They are per-write and never reused. +Index entries use a reserved key namespace inside the tables. The manifest +records how many index entries each table holds. Counters use this number +to hide the internal entries. + ## Configuration Tune the database with `Cinderstore::DB::Config`. @@ -372,6 +786,7 @@ config.memtable_limit = 4_i64 * 1024 * 1024 config.bloom_fpp = 0.01 config.cache_blocks = 512 config.sync_writes = true +config.checksums = true config.l0_compact_threshold = 4 config.compact_on_flush = true config.max_levels = 7 @@ -383,11 +798,26 @@ db = Cinderstore::DB.new("data", config) deeper level may hold. A higher ratio compacts less often. Level L holds up to `memtable_limit * level_ratio ** L` bytes. +`checksums` controls new writes. Set it to `false` to skip CRC32 checksums. +This raises write throughput and shrinks each log record by four bytes. Each +table block also saves four bytes. The reader still verifies files that +carry checksums. Use fast mode for ephemeral data or data you can rebuild. + +`sync_writes` controls durability. When it is `true`, writers share commit +groups and wait for one shared fsync. A single sequential writer still syncs +once per write. When it is `false`, writes never fsync and return at once. + ## Project layout ```text src/cinderstore.cr Library entry point src/cinderstore/ Core components +src/cinderstore/batch.cr Atomic batch writes +src/cinderstore/commit_group.cr Shared durability syncs +src/cinderstore/index.cr Secondary index namespace +src/cinderstore/index_query.cr Streaming index query iterators +src/cinderstore/scan_iter.cr Streaming primary-key range and prefix iterators +src/cinderstore/util.cr Prefix range upper bounds src/cinderstore/snapshot.cr Point-in-time snapshot support src/cli.cr Command line tool examples/demo.cr Library walkthrough @@ -398,36 +828,73 @@ spec/ Test suite ## Test status -The suite runs with `crystal spec`. It has 108 examples. All pass on Windows -and Linux. It covers the skip list, the memory table, the write ahead log, -the bloom filter, and the block cache. It covers the tables, the iterators, -and the database. It covers compaction by level, durability, snapshots, and -the server protocol. +The suite runs with `crystal spec`. It contains 199 examples. It covers the +storage layers, recovery, durability, snapshots, indexes, and server protocol. +It covers streaming index queries, range scans, prefix scans, and verification. The CI workflow runs on GitHub Actions for Windows and Ubuntu. It checks formatting, runs the suite, runs both demos, and builds the binary. +Local validation for this release: + +- Formatting passed with `crystal tool format --check`. +- No-codegen type checks passed for the CLI, demos, and all 21 spec files. +- Full specs and production builds are blocked by restricted Windows process pipes in the MSVC toolchain. + ## Limitations -- Keys sort by byte value. +- Keys sort by byte value. Index keys sort the same way. - Values are limited to 4 MB. - Keys are limited to 4 KB. +- Keys that start with the reserved index prefix are rejected. +- Keys with a NUL byte cannot be indexed. +- An index only covers writes made after it is created. - The server protocol is unencrypted. Use it on localhost only. +- Fast mode cannot detect silent bit rot. It detects torn writes only. +- Group commit needs concurrent writers to share a sync. A single writer + still syncs once per write. +- Index maintenance reads the previous value of each written key. +- Indexers must be deterministic. The same value must give the same keys. - `compact_levels` merges a whole level at once. A very large level makes a large merge. The level targets keep that merge rare. - No multi-threaded runtime is required. The server uses fibers. - Release snapshots before you close the database. +- `scan_iter` has no built-in limit. Stop reading when you reach your limit. +- A database-level scan, prefix, or query iterator holds a snapshot until you close it. +- A wire scan holds a snapshot until it sends `END` or reaches its limit. +- Wire scan limits must be `-1` or greater. +- Verification does not inspect the active in-memory table. ## Roadmap -Planned: - -- Release 0.5: optional checksum-free fast mode -- Release 0.6: batch writes and group commit -- Release 0.7: secondary indexes - -Delivered: - +Complete: + +- Release 0.12: streaming primary-key range and prefix scans over the local + protocol. Limits stop iterators before the next row. +- Release 0.11: read-only storage verification. The check covers tables, WALs, + and manifest metadata without repairing files. +- Release 0.10: primary-key prefix scans. Materialized scans and streaming + iterators use one ordered merge path. Snapshot iterators borrow snapshots. +- Release 0.9: streaming primary-key range iterators. A database-level iterator + owns a snapshot. A snapshot-level iterator borrows its snapshot. + +Remaining: + +- No later release scope is selected. + +Earlier releases: + +- Release 0.8: streaming iterators for secondary index queries. A query + iterator reads a snapshot and yields primary keys one at a time. The + server streams query results over the socket. +- Release 0.7: secondary indexes. Index entries share the log structured + merge tree. The store maintains each index on every write. Queries return + the primary keys behind an index key. +- Release 0.6: batch writes and group commit. A batch is one atomic write + ahead log record. Concurrent writers share one durability sync per group. +- Release 0.5: optional checksum-free fast mode. New data can skip CRC32 + checksums. The on-disk format records the mode, so old and new files mix + safely. - Release 0.4: level-based compaction. Tables cascade from level 0 to deeper levels. Each level stays within its size target. Tombstones survive until the deepest level. diff --git a/examples/server_demo.cr b/examples/server_demo.cr index 1805edc..178323d 100644 --- a/examples/server_demo.cr +++ b/examples/server_demo.cr @@ -30,6 +30,12 @@ puts "GET forge-hammer => #{send(sock, "GET forge-hammer")}" puts "GET missing => #{send(sock, "GET nope")}" puts "DEL forge-tongs => #{send(sock, "DEL forge-tongs")}" puts "STATS => #{send(sock, "STATS")}" +sock << "SCANPREFIX forge- 1\n" +sock.flush +while line = sock.gets + puts "SCANPREFIX => #{line}" + break if line == "END" +end puts "SHUTDOWN => #{send(sock, "SHUTDOWN")}" sock.close diff --git a/shard.lock b/shard.lock index 8a8a3ae..edeb7ad 100644 --- a/shard.lock +++ b/shard.lock @@ -1 +1 @@ -version: 0.4.0 +version: 0.12.0 diff --git a/shard.yml b/shard.yml index 5225a79..18339fb 100644 --- a/shard.yml +++ b/shard.yml @@ -1,7 +1,7 @@ name: cinderstore -version: 0.4.0 +version: 0.12.0 description: An embeddable key/value store built on a log structured merge tree. -crystal: ">= 1.10.0" +crystal: ">= 1.21.0" license: Apache-2.0 authors: - Cinderstore Contributors diff --git a/spec/batch_spec.cr b/spec/batch_spec.cr new file mode 100644 index 0000000..42fb26b --- /dev/null +++ b/spec/batch_spec.cr @@ -0,0 +1,213 @@ +require "./spec_helper" + +describe Cinderstore::Batch do + it "counts queued operations" do + batch = Cinderstore::Batch.new + batch.empty?.should be_true + batch.put("a", "1") + batch.delete("b") + batch.empty?.should be_false + batch.size.should eq(2) + end + + it "yields operations in queue order" do + batch = Cinderstore::Batch.new + batch.put("a", "1") + batch.delete("b") + batch.put("c", "3") + ops = [] of Tuple(String, String, Bool) + batch.each do |key, value, alive| + ops << {key, value, alive} + end + ops.should eq([{"a", "1", true}, {"b", "", false}, {"c", "3", true}]) + end +end + +describe Cinderstore::DB do + it "writes a batch and reads it back" do + Cinderstore::SpecHelpers.with_db("batch-basic") do |db, _path| + batch = Cinderstore::Batch.new + batch.put("alpha", "one") + batch.put("beta", "two") + db.write(batch) + db.get("alpha").should eq("one") + db.get("beta").should eq("two") + db.scan.size.should eq(2) + end + end + + it "writes a batch built in a block" do + Cinderstore::SpecHelpers.with_db("batch-block") do |db, _path| + db.write do |b| + b.put("a", "1") + b.put("b", "2") + end + db.scan.map(&.[0]).should eq(%w[a b]) + end + end + + it "applies puts and deletes in one batch" do + Cinderstore::SpecHelpers.with_db("batch-mixed") do |db, _path| + db.put("keep", "v") + db.put("drop", "v") + db.write do |b| + b.put("add", "v") + b.delete("drop") + end + db.scan.map(&.[0]).should eq(%w[add keep]) + db.get("drop").should be_nil + end + end + + it "lets a later batch operation win for the same key" do + Cinderstore::SpecHelpers.with_db("batch-overwrite") do |db, _path| + db.write do |b| + b.put("k", "first") + b.put("k", "second") + end + db.get("k").should eq("second") + db.scan.size.should eq(1) + end + end + + it "lets a delete in a batch hide an earlier put" do + Cinderstore::SpecHelpers.with_db("batch-hide") do |db, _path| + db.write do |b| + b.put("k", "v") + b.delete("k") + end + db.get("k").should be_nil + db.scan.size.should eq(0) + end + end + + it "lets a put in a batch override an earlier delete" do + Cinderstore::SpecHelpers.with_db("batch-resurrect") do |db, _path| + db.write do |b| + b.delete("k") + b.put("k", "v") + end + db.get("k").should eq("v") + db.scan.size.should eq(1) + end + end + + it "treats an empty batch as a no-op" do + Cinderstore::SpecHelpers.with_db("batch-empty") do |db, _path| + batch = Cinderstore::Batch.new + db.write(batch) + db.stats.seq.should eq(0) + db.stats.writes.should eq(0) + db.scan.size.should eq(0) + end + end + + it "assigns consecutive sequence numbers within a batch" do + Cinderstore::SpecHelpers.with_db("batch-seq") do |db, _path| + db.write do |b| + 3.times { |i| b.put("k#{i}", "v") } + end + db.stats.seq.should eq(3) + end + end + + it "counts one batch as one write operation" do + Cinderstore::SpecHelpers.with_db("batch-stats") do |db, _path| + db.write do |b| + 10.times { |i| b.put("k%02d" % i, "v") } + end + db.stats.writes.should eq(1) + db.stats.entries.should eq(10) + end + end + + it "rejects an oversized value and leaves the store unchanged" do + Cinderstore::SpecHelpers.with_db("batch-invalid") do |db, _path| + db.put("before", "v") + batch = Cinderstore::Batch.new + batch.put("good", "v") + batch.put("bad", "x" * (Cinderstore::DB::MAX_VALUE_BYTES + 1)) + expect_raises(Cinderstore::InvalidValueError) { db.write(batch) } + db.get("before").should eq("v") + db.get("good").should be_nil + db.stats.writes.should eq(1) + end + end + + it "rejects an invalid key inside a batch" do + Cinderstore::SpecHelpers.with_db("batch-key") do |db, _path| + batch = Cinderstore::Batch.new + batch.put("", "v") + expect_raises(Cinderstore::InvalidKeyError) { db.write(batch) } + end + end + + it "recovers a batch across a restart" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db_path("batch-restart") do |path| + db = Cinderstore::DB.new(path, config) + db.write do |b| + 20.times { |i| b.put("row-%02d" % i, "value-#{i}") } + end + db.close + + reopened = Cinderstore::DB.new(path, config) + reopened.scan.size.should eq(20) + reopened.get("row-07").should eq("value-7") + reopened.close + end + end + + it "recovers a batch written in fast mode" do + config = Cinderstore::SpecHelpers.fast_config + config.checksums = false + Cinderstore::SpecHelpers.with_db_path("batch-fast") do |path| + db = Cinderstore::DB.new(path, config) + db.write do |b| + 10.times { |i| b.put("k%02d" % i, "v-#{i}") } + end + db.close + + reopened = Cinderstore::DB.new(path) + reopened.scan.size.should eq(10) + reopened.get("k03").should eq("v-3") + reopened.close + end + end + + it "flushes and compacts batch-written data" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("batch-compact", config) do |db, _path| + db.write do |b| + 40.times { |i| b.put("k%02d" % i, "v-#{i}") } + end + db.flush + db.put("k00", "updated") + db.flush + db.compact + db.stats.tables.should eq(1) + db.get("k00").should eq("updated") + db.scan.size.should eq(40) + end + end + + it "keeps a batch-created snapshot view stable" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("batch-snapshot", config) do |db, _path| + db.write do |b| + b.put("a", "1") + b.put("b", "2") + end + snap = db.snapshot + db.write do |b| + b.put("c", "3") + b.delete("a") + end + snap.get("a").should eq("1") + snap.get("c").should be_nil + snap.release + db.get("a").should be_nil + db.get("c").should eq("3") + end + end +end diff --git a/spec/commit_group_spec.cr b/spec/commit_group_spec.cr new file mode 100644 index 0000000..6a25062 --- /dev/null +++ b/spec/commit_group_spec.cr @@ -0,0 +1,89 @@ +require "./spec_helper" + +# A helper that runs `writers` concurrent fibers, each calling `work`, and +# waits for every fiber to finish. +private def run_concurrently(count : Int32, &work : Int32 ->) : Nil + done = Channel(Nil).new + count.times do |i| + spawn do + work.call(i) + done.send(nil) + end + end + count.times { done.receive } +end + +describe Cinderstore::CommitGroup do + it "syncs once for a burst of concurrent writers" do + dir = Cinderstore::SpecHelpers.tmp_db_path("group") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, true) + group = Cinderstore::CommitGroup.new + begin + run_concurrently(8) do |i| + writer.append(Cinderstore::Entry.new("k#{i}", i.to_i64, true, "v")) + group.commit(writer) + end + group.commits.should eq(1_i64) + ensure + group.shutdown + writer.close + end + end + + it "syncs each sequential write" do + dir = Cinderstore::SpecHelpers.tmp_db_path("group") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, true) + group = Cinderstore::CommitGroup.new + begin + 5.times do |i| + writer.append(Cinderstore::Entry.new("k#{i}", i.to_i64, true, "v")) + group.commit(writer) + end + group.commits.should eq(5_i64) + ensure + group.shutdown + writer.close + end + end +end + +describe Cinderstore::DB do + it "coalesces durability syncs for concurrent writers" do + config = Cinderstore::SpecHelpers.fast_config + config.sync_writes = true + Cinderstore::SpecHelpers.with_db("db-group", config) do |db, _path| + run_concurrently(8) do |i| + 25.times do |j| + db.put("k%d-%02d" % {i, j}, "v") + end + end + db.stats.writes.should eq(200) + db.stats.commits.should be >= 1 + db.stats.commits.should be <= 25 + db.scan.size.should eq(200) + end + end + + it "recovers group-committed writes after a restart" do + config = Cinderstore::SpecHelpers.fast_config + config.sync_writes = true + Cinderstore::SpecHelpers.with_db_path("db-group-restart") do |path| + db = Cinderstore::DB.new(path, config) + run_concurrently(4) do |i| + 10.times do |j| + db.put("k%d-%02d" % {i, j}, "value-#{j}") + end + end + db.close + + reopened = Cinderstore::DB.new(path, config) + reopened.scan.size.should eq(40) + reopened.get("k2-07").should eq("value-7") + reopened.close + end + end +end diff --git a/spec/db_spec.cr b/spec/db_spec.cr index ad5703e..c9aa7af 100644 --- a/spec/db_spec.cr +++ b/spec/db_spec.cr @@ -142,6 +142,65 @@ describe Cinderstore::DB do end end + it "round trips a fast-mode database across a restart" do + config = Cinderstore::SpecHelpers.fast_config + config.checksums = false + Cinderstore::SpecHelpers.with_db_path("db-fast") do |path| + db = Cinderstore::DB.new(path, config) + db.put("a", "1") + db.put("b", "2") + db.flush + db.put("c", "3") + db.close + + # The reader learns the checksum mode from the files themselves, so + # the default config opens the fast-mode directory correctly. + reopened = Cinderstore::DB.new(path) + reopened.get("a").should eq("1") + reopened.get("b").should eq("2") + reopened.get("c").should eq("3") + reopened.scan.map(&.[0]).should eq(%w[a b c]) + reopened.close + end + end + + it "recovers a fast-mode torn write ahead log" do + config = Cinderstore::SpecHelpers.fast_config + config.checksums = false + Cinderstore::SpecHelpers.with_db_path("db-fast-torn") do |path| + db = Cinderstore::DB.new(path, config) + db.put("alpha", "1") + db.put("beta", "2") + db.put("gamma", "3") + db.close + + wal = Dir.children(path).find { |n| n.ends_with?(".wal") }.not_nil! + Cinderstore::SpecHelpers.truncate(File.join(path, wal), File.size(File.join(path, wal)) - 3) + + reopened = Cinderstore::DB.new(path, config) + reopened.get("alpha").should eq("1") + reopened.get("beta").should eq("2") + reopened.get("gamma").should be_nil + reopened.close + end + end + + it "compacts fast-mode tables without losing data" do + config = Cinderstore::SpecHelpers.fast_config + config.checksums = false + Cinderstore::SpecHelpers.with_db("db-fast-compact", config) do |db, _path| + db.put("k", "v1") + db.flush + db.put("k", "v2") + db.flush + db.stats.tables.should eq(2) + db.compact + db.stats.tables.should eq(1) + db.get("k").should eq("v2") + db.scan.size.should eq(1) + end + end + it "recovers the intact prefix of a torn write ahead log" do config = Cinderstore::SpecHelpers.fast_config Cinderstore::SpecHelpers.with_db_path("db-torn") do |path| @@ -192,6 +251,21 @@ describe Cinderstore::DB do end end + it "reports the number of writes in stats" do + Cinderstore::SpecHelpers.with_db("db-writes") do |db, _path| + db.put("a", "1") + db.delete("a") + db.write do |b| + b.put("x", "1") + b.put("y", "2") + end + stats = db.stats + stats.writes.should eq(3) + stats.to_h.keys.should contain("writes") + stats.to_h.keys.should contain("commits") + end + end + it "rejects empty keys" do Cinderstore::SpecHelpers.with_db("db-invalid") do |db, _path| expect_raises(Cinderstore::InvalidKeyError) { db.put("", "v") } diff --git a/spec/index_iter_spec.cr b/spec/index_iter_spec.cr new file mode 100644 index 0000000..d99ae10 --- /dev/null +++ b/spec/index_iter_spec.cr @@ -0,0 +1,276 @@ +require "./spec_helper" +require "socket" + +private def wait_until_ready(server : Cinderstore::Server, timeout : Time::Span = 5.seconds) : Nil + deadline = Time.instant + timeout + until server.ready? + raise "server did not start in time" if Time.instant >= deadline + sleep 10.milliseconds + end +end + +private def with_server(db : Cinderstore::DB, &) + server = Cinderstore::Server.new(db, "127.0.0.1", 0) + spawn { server.run } + wait_until_ready(server) + begin + yield server + ensure + server.stop + sleep 10.milliseconds + end +end + +private def send_command(sock : TCPSocket, command : String) : String? + sock << command << "\n" + sock.flush + sock.gets +end + +describe Cinderstore::DB do + it "streams the primary keys behind one index key in order" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-exact", config) do |db, _path| + db.create_index("by-group") { |key, _value| [key[0..0]] } + db.put("apple", "1") + db.put("apricot", "2") + db.put("avocado", "3") + db.put("plum", "4") + iter = db.query_iter("by-group", "a") + keys = [] of String + while key = iter.next? + keys << key + end + iter.close + keys.should eq(%w[apple apricot avocado]) + end + end + + it "matches the materialized query result" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-parity", config) do |db, _path| + db.create_json_index("by-price", "price") + db.put("a", %({"price":10})) + db.put("b", %({"price":20})) + db.put("c", %({"price":20})) + db.put("d", %({"price":30})) + iter = db.query_iter("by-price", "20") + iter.to_a.should eq(db.query("by-price", "20")) + iter.close + end + end + + it "stops early and still releases the snapshot on close" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-early", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + 5.times { |i| db.put("k#{i}", "v") } + iter = db.query_iter("by-value", "v") + iter.next?.should eq("k0") + iter.next?.should eq("k1") + iter.close + expect_raises(Cinderstore::SnapshotReleasedError) { iter.next? } + end + end + + it "streams a range in index key order with an upper bound" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-range", config) do |db, _path| + db.create_json_index("by-price", "price") + db.put("a", %({"price":10})) + db.put("b", %({"price":20})) + db.put("c", %({"price":30})) + db.put("d", %({"price":40})) + iter = db.query_range_iter("by-price", "10", "30") + iter.to_a.should eq(%w[a b]) + iter.close + iter = db.query_range_iter("by-price") + iter.to_a.should eq(%w[a b c d]) + iter.close + end + end + + it "streams empty results without error" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-empty", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "x") + iter = db.query_iter("by-value", "missing") + iter.next?.should be_nil + iter.close + iter = db.query_range_iter("by-value", "y", "z") + iter.to_a.should be_empty + iter.close + end + end + + it "skips deleted keys and moved values" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-live", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "x") + db.put("b", "x") + db.delete("a") + db.put("c", "y") + iter = db.query_iter("by-value", "x") + iter.to_a.should eq(%w[b]) + iter.close + iter = db.query_iter("by-value", "y") + iter.to_a.should eq(%w[c]) + iter.close + end + end + + it "streams across flush and compaction boundaries" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-tables", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "x") + db.flush + db.put("b", "x") + db.flush + db.put("c", "x") + db.flush + db.compact + iter = db.query_iter("by-value", "x") + iter.to_a.should eq(%w[a b c]) + iter.close + end + end + + it "reads from tables even without a registered index" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db_path("iter-materialized") do |path| + db = Cinderstore::DB.new(path, config) + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "x") + db.put("b", "x") + db.close + + reopened = Cinderstore::DB.new(path, config) + iter = reopened.query_iter("by-value", "x") + iter.to_a.should eq(%w[a b]) + iter.close + reopened.close + end + end + + it "yields keys through the block form" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-block", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "x") + db.put("b", "x") + iter = db.query_iter("by-value", "x") + keys = [] of String + iter.each { |key| keys << key } + iter.close + keys.should eq(%w[a b]) + end + end + + it "rejects invalid index names" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-invalid", config) do |db, _path| + expect_raises(Cinderstore::InvalidIndexError) do + db.query_iter("", "v") + end + expect_raises(Cinderstore::InvalidIndexError) do + db.query_range_iter("a\u{0000}b") + end + end + end + + it "keeps a snapshot iterator stable across later writes" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-snapshot", config) do |db, _path| + db.create_json_index("by-x", "x") + db.put("a", %({"x":1})) + snap = db.snapshot + db.put("b", %({"x":1})) + db.delete("a") + iter = snap.query_iter("by-x", "1") + iter.to_a.should eq(%w[a]) + iter.close + db.query_iter("by-x", "1").to_a.should eq(%w[b]) + snap.release + end + end + + it "does not release the caller snapshot on close" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-norelease", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "x") + snap = db.snapshot + iter = snap.query_iter("by-value", "x") + iter.to_a.should eq(%w[a]) + iter.close + snap.count.should eq(1) + snap.get("a").should eq("x") + snap.release + end + end + + it "keeps a snapshot range stream stable across a delete" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-snapshot-range", config) do |db, _path| + db.create_json_index("by-x", "x") + db.put("a", %({"x":1})) + db.put("b", %({"x":2})) + snap = db.snapshot + db.delete("a") + iter = snap.query_range_iter("by-x") + iter.to_a.should eq(%w[a b]) + iter.close + snap.release + end + end +end + +describe Cinderstore::Server do + it "streams query results over the socket" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-server", config) do |db, _path| + db.create_json_index("by-price", "price") + with_server(db) do |server| + sock = TCPSocket.new("127.0.0.1", server.port) + send_command(sock, "PUT a {\"price\":10}").should eq("OK") + send_command(sock, "PUT b {\"price\":20}").should eq("OK") + send_command(sock, "PUT c {\"price\":20}").should eq("OK") + + sock << "QUERY by-price 20\n" + sock.flush + rows = [] of String + while (line = sock.gets) && line != "END" + rows << line + end + rows.should eq(["ROW b", "ROW c"]) + + sock << "QUERYRANGE by-price 10 20\n" + sock.flush + rows = [] of String + while (line = sock.gets) && line != "END" + rows << line + end + rows.should eq(["ROW a"]) + sock.close + end + end + end + + it "streams an empty query as a single END line" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("iter-server-empty", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + with_server(db) do |server| + sock = TCPSocket.new("127.0.0.1", server.port) + send_command(sock, "PUT a x").should eq("OK") + sock << "QUERY by-value missing\n" + sock.flush + sock.gets.should eq("END") + sock.close + end + end + end +end diff --git a/spec/index_spec.cr b/spec/index_spec.cr new file mode 100644 index 0000000..53e6660 --- /dev/null +++ b/spec/index_spec.cr @@ -0,0 +1,346 @@ +require "./spec_helper" + +describe Cinderstore::DB do + it "indexes new values and returns the matching primary keys" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-basic", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "one") + db.put("b", "two") + db.query("by-value", "one").should eq(%w[a]) + db.query("by-value", "two").should eq(%w[b]) + db.query("by-value", "three").should eq([] of String) + end + end + + it "returns primary keys in primary key order for one index key" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-order", config) do |db, _path| + db.create_index("by-group") { |key, _value| [key[0..0]] } + db.put("apple", "1") + db.put("apricot", "2") + db.put("avocado", "3") + db.query("by-group", "a").should eq(%w[apple apricot avocado]) + end + end + + it "supports a limit on exact queries" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-limit", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + 3.times { |i| db.put("k#{i}", "v") } + db.query("by-value", "v", 2).should eq(%w[k0 k1]) + end + end + + it "moves a key between index values when its value changes" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-update", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "x") + db.put("a", "y") + db.query("by-value", "x").should eq([] of String) + db.query("by-value", "y").should eq(%w[a]) + end + end + + it "removes a key from every index on delete" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-delete", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "x") + db.put("b", "x") + db.delete("a") + db.query("by-value", "x").should eq(%w[b]) + end + end + + it "re-indexes a key after a delete" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-resurrect", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "x") + db.delete("a") + db.put("a", "y") + db.query("by-value", "x").should eq([] of String) + db.query("by-value", "y").should eq(%w[a]) + end + end + + it "keeps the index correct across flush boundaries" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-flush", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "x") + db.flush + db.put("a", "y") + db.flush + db.query("by-value", "x").should eq([] of String) + db.query("by-value", "y").should eq(%w[a]) + end + end + + it "drops stale index entries during compaction" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-compact", config) do |db, path| + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "x") + db.flush + db.put("a", "y") + db.flush + db.put("b", "x") + db.flush + db.compact + db.query("by-value", "x").should eq(%w[b]) + db.query("by-value", "y").should eq(%w[a]) + db.scan.map(&.[0]).should eq(%w[a b]) + end + end + + it "recovers the index after a restart" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db_path("idx-restart") do |path| + db = Cinderstore::DB.new(path, config) + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "x") + db.put("b", "x") + db.flush + db.put("c", "y") + db.close + + reopened = Cinderstore::DB.new(path, config) + reopened.query("by-value", "x").should eq(%w[a b]) + reopened.query("by-value", "y").should eq(%w[c]) + reopened.close + end + end + + it "answers queries without a registered index" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db_path("idx-materialized") do |path| + db = Cinderstore::DB.new(path, config) + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "x") + db.close + + reopened = Cinderstore::DB.new(path, config) + reopened.query("by-value", "x").should eq(%w[a]) + reopened.query("by-value", "x").should eq(%w[a]) + reopened.close + end + end + + it "maintains the index through batch writes" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-batch", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + db.write do |b| + b.put("a", "x") + b.put("a", "y") + end + db.query("by-value", "x").should eq([] of String) + db.query("by-value", "y").should eq(%w[a]) + end + end + + it "recovers the index after a batch restart" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db_path("idx-batchrestart") do |path| + db = Cinderstore::DB.new(path, config) + db.create_index("by-value") { |_key, value| [value] } + db.write do |b| + b.put("a", "x") + b.put("b", "y") + end + db.close + + reopened = Cinderstore::DB.new(path, config) + reopened.query("by-value", "x").should eq(%w[a]) + reopened.query("by-value", "y").should eq(%w[b]) + reopened.close + end + end + + it "hides index entries from scans, gets, and stats" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-hidden", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + db.put("a", "1") + db.put("b", "2") + db.flush + db.scan.map(&.[0]).should eq(%w[a b]) + db.get("a").should eq("1") + db.stats.entries.should eq(2_i64) + db.stats.entries.should eq(db.scan.size.to_i64) + end + end + + it "extracts scalar and array JSON fields" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-json", config) do |db, _path| + db.create_json_index("by-tag", "tags") + db.create_json_index("by-price", "price") + db.put("k1", %({"tags":["red","blue"],"price":14.00})) + db.put("k2", %({"tags":["green"],"price":20.50})) + db.query("by-tag", "red").should eq(%w[k1]) + db.query("by-tag", "blue").should eq(%w[k1]) + db.query("by-tag", "green").should eq(%w[k2]) + db.query("by-price", "14.0").should eq(%w[k1]) + db.query("by-price", "20.5").should eq(%w[k2]) + end + end + + it "skips values that are not JSON or lack the field" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-jsonmiss", config) do |db, _path| + db.create_json_index("by-x", "x") + db.put("k1", %({"y":1})) + db.put("k2", "not json") + db.put("k3", %({"x":"found"})) + db.query("by-x", "found").should eq(%w[k3]) + db.scan.map(&.[0]).should eq(%w[k1 k2 k3]) + end + end + + it "returns range results in index key order" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-range", config) do |db, _path| + db.create_json_index("by-price", "price") + db.put("a", %({"price":10})) + db.put("b", %({"price":20})) + db.put("c", %({"price":30})) + db.query_range("by-price", "10", "30").should eq(%w[a b]) + db.query_range("by-price", "", "40").should eq(%w[a b c]) + db.query_range("by-price", "", nil, 2).should eq(%w[a b]) + db.query_range("by-price").should eq(%w[a b c]) + end + end + + it "keeps a snapshot query stable across later writes" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-snapshot", config) do |db, _path| + db.create_json_index("by-x", "x") + db.put("a", %({"x":1})) + snap = db.snapshot + db.put("b", %({"x":1})) + db.delete("a") + snap.query("by-x", "1").should eq(%w[a]) + db.query("by-x", "1").should eq(%w[b]) + snap.release + end + end + + it "supports querying a snapshot that spans tables" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-snaptables", config) do |db, _path| + db.create_json_index("by-x", "x") + db.put("a", %({"x":1})) + db.flush + db.put("b", %({"x":1})) + db.flush + snap = db.snapshot + db.put("c", %({"x":1})) + snap.query("by-x", "1").should eq(%w[a b]) + db.query("by-x", "1").should eq(%w[a b c]) + snap.release + end + end + + it "lists registered indexes" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-list", config) do |db, _path| + db.create_index("one") { |_key, _value| [] of String } + db.create_json_index("two", "price") + db.indexes.should eq(%w[one two]) + db.index?("one").should be_true + db.index?("nope").should be_false + end + end + + it "rejects invalid index names" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-invalid", config) do |db, _path| + expect_raises(Cinderstore::InvalidIndexError) do + db.create_index("") { |_key, _value| [] of String } + end + expect_raises(Cinderstore::InvalidIndexError) do + db.create_index("a\u{0000}b") { |_key, _value| [] of String } + end + db.create_index("dup") { |_key, _value| [] of String } + expect_raises(Cinderstore::InvalidIndexError) do + db.create_index("dup") { |_key, _value| [] of String } + end + end + end + + it "rejects index keys that contain a NUL byte" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-nulkey", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + expect_raises(Cinderstore::InvalidIndexError) { db.put("a", "bad\u{0000}key") } + end + end + + it "rejects primary keys in the reserved namespace" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("idx-reserved", config) do |db, _path| + reserved = Cinderstore::Index::NAMESPACE + "x" + expect_raises(Cinderstore::InvalidKeyError) { db.put(reserved, "v") } + expect_raises(Cinderstore::InvalidKeyError) { db.get(reserved) } + end + end + + it "rejects NUL-containing keys only while an index exists" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db_path("idx-nulprimary") do |path| + db = Cinderstore::DB.new(path, config) + db.put("a\u{0000}b", "v") + db.get("a\u{0000}b").should eq("v") + db.close + + indexed = Cinderstore::DB.new(path, config) + indexed.create_index("by-value") { |_key, value| [value] } + expect_raises(Cinderstore::InvalidKeyError) { indexed.put("a\u{0000}b", "w") } + indexed.close + end + end + + it "stays consistent under concurrent writes" do + config = Cinderstore::SpecHelpers.fast_config + config.memtable_limit = 1_i64 << 30 + Cinderstore::SpecHelpers.with_db("idx-concurrent", config) do |db, _path| + db.create_index("by-value") { |_key, value| [value] } + done = Channel(Nil).new + 8.times do |i| + spawn do + 20.times do |j| + db.put("k-%d-%02d" % {i, j}, "v-#{i}") + end + done.send(nil) + end + end + 8.times { done.receive } + 8.times do |i| + keys = db.query("by-value", "v-#{i}") + keys.size.should eq(20) + keys.each { |key| key.should start_with("k-#{i}-") } + end + end + end + + it "keeps existing data readable when an index is added later" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db_path("idx-parity") do |path| + plain = Cinderstore::DB.new(path, config) + 5.times { |i| plain.put("k#{i}", "v#{i}") } + plain.close + + indexed = Cinderstore::DB.new(path, config) + indexed.create_index("by-value") { |_key, value| [value] } + indexed.put("extra", "v9") + indexed.scan.map(&.[0]).should eq(%w[extra k0 k1 k2 k3 k4]) + indexed.stats.entries.should eq(6_i64) + indexed.close + end + end +end diff --git a/spec/prefix_spec.cr b/spec/prefix_spec.cr new file mode 100644 index 0000000..d59e154 --- /dev/null +++ b/spec/prefix_spec.cr @@ -0,0 +1,37 @@ +require "./spec_helper" + +describe Cinderstore::DB do + it %q(streams keys with a prefix and applies a limit after filtering) do + Cinderstore::SpecHelpers.with_db(%q(scan-prefix), Cinderstore::SpecHelpers.fast_config) do |db, _path| + %w[SKU-0010 SKU-0011 SKU-0020 SKU-0012].each { |key| db.put(key, key) } + + db.scan_prefix(%q(SKU-001), 2).map(&.[0]).should eq(%w[SKU-0010 SKU-0011]) + db.scan_prefix(%q()).map(&.[0]).should eq(%w[SKU-0010 SKU-0011 SKU-0012 SKU-0020]) + iter = db.scan_prefix_iter(%q(SKU-001)) + iter.to_a.map(&.[0]).should eq(%w[SKU-0010 SKU-0011 SKU-0012]) + iter.close + end + end + + it %q(keeps a snapshot prefix stable across writes) do + Cinderstore::SpecHelpers.with_db(%q(scan-prefix-snapshot)) do |db, _path| + db.put(%q(user:1), %q(old)) + snapshot = db.snapshot + db.put(%q(user:2), %q(new)) + db.delete(%q(user:1)) + + snapshot.scan_prefix(%q(user:)).should eq([{"user:1", "old"}]) + iter = snapshot.scan_prefix_iter(%q(user:)) + iter.to_a.should eq([{"user:1", "old"}]) + iter.close + snapshot.release + end + end +end + +describe Cinderstore::Util do + it %q(computes a byte upper bound for a prefix) do + Cinderstore::Util.prefix_end(%q(SKU-001)).should eq(%q(SKU-002)) + Cinderstore::Util.prefix_end(%q()).should be_nil + end +end diff --git a/spec/scan_iter_spec.cr b/spec/scan_iter_spec.cr new file mode 100644 index 0000000..419ff47 --- /dev/null +++ b/spec/scan_iter_spec.cr @@ -0,0 +1,61 @@ +require "./spec_helper" + +describe Cinderstore::DB do + it "streams live pairs in a half-open primary-key range" do + Cinderstore::SpecHelpers.with_db("scan-iter") do |db, _path| + %w[apple banana cherry date].each { |key| db.put(key, key.upcase) } + + iter = db.scan_iter("banana", "date") + iter.next?.should eq({"banana", "BANANA"}) + iter.next?.should eq({"cherry", "CHERRY"}) + iter.next?.should be_nil + iter.close + end + end + + it "keeps a database iterator at its creation snapshot" do + Cinderstore::SpecHelpers.with_db("scan-iter-snapshot") do |db, _path| + db.put("a", "1") + db.put("b", "2") + iter = db.scan_iter + db.put("c", "3") + db.delete("b") + + iter.to_a.should eq([{"a", "1"}, {"b", "2"}]) + iter.close + expect_raises(Cinderstore::SnapshotReleasedError) { iter.next? } + end + end + + it "keeps a caller snapshot alive when its iterator closes" do + Cinderstore::SpecHelpers.with_db("scan-iter-caller-snapshot") do |db, _path| + db.put("a", "1") + db.put("b", "2") + snapshot = db.snapshot + iter = snapshot.scan_iter("a") + keys = [] of String + iter.each { |key, _value| keys << key } + iter.close + + keys.should eq(%w[a b]) + snapshot.get("a").should eq("1") + snapshot.release + end + end + + it "streams across tables and skips tombstones" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("scan-iter-tables", config) do |db, _path| + db.put("a", "one") + db.put("b", "two") + db.flush + db.delete("b") + db.put("c", "three") + db.flush + + iter = db.scan_iter + iter.to_a.should eq([{"a", "one"}, {"c", "three"}]) + iter.close + end + end +end diff --git a/spec/server_spec.cr b/spec/server_spec.cr index 51fef0f..76de06d 100644 --- a/spec/server_spec.cr +++ b/spec/server_spec.cr @@ -53,7 +53,7 @@ describe Cinderstore::Server do send_command(sock, "PUT b two").should eq("OK") send_command(sock, "PUT c three").should eq("OK") - sock << "SCAN a c\n" + sock << "SCAN a c 2\n" sock.flush rows = [] of String while (line = sock.gets) && line != "END" @@ -65,6 +65,53 @@ describe Cinderstore::Server do end end + it "streams a prefix scan and validates its limit" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("server-prefix", config) do |db, _path| + db.put("forge-a", "one") + db.put("forge-b", "two") + db.put("other", "three") + with_server(db) do |server| + sock = TCPSocket.new("127.0.0.1", server.port) + sock << "SCANPREFIX forge- 1\n" + sock.flush + rows = [] of String + while (line = sock.gets) && line != "END" + rows << line + end + rows.should eq(["ROW forge-a one"]) + + sock << "SCANPREFIX forge- 0\n" + sock.flush + sock.gets.should eq("END") + + sock << "SCANPREFIX forge- nope\n" + sock.flush + sock.gets.not_nil!.should start_with("ERR limit must be an integer") + sock.close + end + end + end + + it "serves queries over a secondary index" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("server-query", config) do |db, _path| + db.create_json_index("by-price", "price") + with_server(db) do |server| + sock = TCPSocket.new("127.0.0.1", server.port) + send_command(sock, "PUT a {\"price\":10}").should eq("OK") + sock << "QUERY by-price 10\n" + sock.flush + rows = [] of String + while (line = sock.gets) && line != "END" + rows << line + end + rows.should eq(["ROW a"]) + sock.close + end + end + end + it "reports protocol errors" do config = Cinderstore::SpecHelpers.fast_config Cinderstore::SpecHelpers.with_db("server-errors", config) do |db, _path| diff --git a/spec/table_spec.cr b/spec/table_spec.cr index ee76cb5..e5ffa11 100644 --- a/spec/table_spec.cr +++ b/spec/table_spec.cr @@ -20,6 +20,33 @@ def read_all(reader : Cinderstore::SstableReader) all end +# Rewrites a version-2 table file into the version-1 layout. +# +# The data blocks, index, and bloom filter are identical between the two +# versions. Only the footer differs. The version-1 footer drops the flag +# byte and the reserved bytes, and it carries its own CRC32. +def rewrite_as_legacy_v1(path : String) : Nil + bytes = File.read(path).to_slice + footer_start = bytes.size - Cinderstore::SstableWriter::FOOTER_SIZE + prefix = bytes[0, footer_start] + footer = bytes[footer_start, 44].dup + + # Patch the version field from 2 to 1. It sits 40 bytes into the footer. + version_io = IO::Memory.new(footer) + version_io.pos = 40 + version_io.write_bytes(1_u32, IO::ByteFormat::LittleEndian) + + # A version-1 footer is the fixed fields plus a CRC32. + legacy_footer = IO::Memory.new + legacy_footer.write(footer) + legacy_footer.write_bytes(Cinderstore::Util.crc32(footer), IO::ByteFormat::LittleEndian) + + rebuilt = IO::Memory.new + rebuilt.write(prefix) + rebuilt.write(legacy_footer.to_slice) + File.write(path, rebuilt.to_slice) +end + describe Cinderstore::SstableWriter do it "round trips entries through a reader" do dir = Cinderstore::SpecHelpers.tmp_db_path("table") @@ -51,6 +78,74 @@ describe Cinderstore::SstableWriter do reader.close end + it "round trips entries without checksums" do + dir = Cinderstore::SpecHelpers.tmp_db_path("table") + Dir.mkdir_p(dir) + path = File.join(dir, "000007.sst") + entries = 300.times.map { |i| Cinderstore::Entry.new("key-%04d" % i, i.to_i64, true, "value-#{i}") }.to_a + File.open(path, "w") do |io| + writer = Cinderstore::SstableWriter.new(io, 7_i64, 4096, 0.01, false) + entries.each do |entry| + writer.add(entry.key, entry.value, entry.seq, entry.alive) + end + writer.finish + end + + reader = Cinderstore::SstableReader.new(path, 7_i64, nil) + read_all(reader).should eq(entries) + reader.close + end + + it "does not verify data blocks that carry no checksums" do + dir = Cinderstore::SpecHelpers.tmp_db_path("table") + Dir.mkdir_p(dir) + path = File.join(dir, "000007.sst") + entries = 50.times.map { |i| Cinderstore::Entry.new("k%03d" % i, i.to_i64, true, "v") }.to_a + File.open(path, "w") do |io| + writer = Cinderstore::SstableWriter.new(io, 7_i64, 64, 0.01, false) + entries.each { |entry| writer.add(entry.key, entry.value, entry.seq, entry.alive) } + writer.finish + end + + # Corrupt the value byte of the first entry in the first block. The + # value byte of a four-byte key sits eight bytes into the payload. + File.open(path, "r+") do |f| + length = Cinderstore::Util.read_varint(f).to_i + target = f.pos + 8 + f.pos = target + byte = f.read_byte.not_nil! + f.pos = target + f.write_byte((byte ^ 0xFF).to_u8) + end + + reader = Cinderstore::SstableReader.new(path, 7_i64, nil) + decoded = read_all(reader) + decoded.size.should eq(entries.size) + decoded.first.value.should_not eq("v") + reader.close + end + + it "reads a legacy version-1 table" do + dir = Cinderstore::SpecHelpers.tmp_db_path("table") + Dir.mkdir_p(dir) + path = File.join(dir, "000007.sst") + entries = [ + Cinderstore::Entry.new("a", 1_i64, true, "1"), + Cinderstore::Entry.new("b", 2_i64, true, "2"), + Cinderstore::Entry.new("c", 3_i64, true, "3"), + ] + File.open(path, "w") do |io| + writer = Cinderstore::SstableWriter.new(io, 7_i64, 256, 0.01, true) + entries.each { |entry| writer.add(entry.key, entry.value, entry.seq, entry.alive) } + writer.finish + end + rewrite_as_legacy_v1(path) + + reader = Cinderstore::SstableReader.new(path, 7_i64, nil) + read_all(reader).should eq(entries) + reader.close + end + it "splits data into multiple blocks" do dir = Cinderstore::SpecHelpers.tmp_db_path("table") Dir.mkdir_p(dir) diff --git a/spec/verification_spec.cr b/spec/verification_spec.cr new file mode 100644 index 0000000..64d02f3 --- /dev/null +++ b/spec/verification_spec.cr @@ -0,0 +1,90 @@ +require "./spec_helper" + +describe Cinderstore::DB::VerificationReport do + it "reports valid tables and write ahead logs" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db("verify-valid", config) do |db, _path| + db.put("alpha", "one") + db.put("beta", "two") + + before_flush = db.verify + before_flush.valid?.should be_true + before_flush.tables.should eq(0) + before_flush.wal_files.should eq(1) + before_flush.wal_records.should eq(2) + + db.flush + report = db.verify + report.valid?.should be_true + report.tables.should eq(1) + report.table_entries.should eq(2) + report.wal_files.should eq(1) + report.wal_records.should eq(0) + report.bytes_checked.should be > 0 + report.to_h["valid"].should be_true + report.to_json.should contain("table_entries") + end + end +end + +describe Cinderstore::DB do + it "reports a corrupt table block" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db_path("verify-table") do |path| + db = Cinderstore::DB.new(path, config) + db.put("alpha", "one") + db.flush + db.close + + table_name = Dir.children(path).find { |name| name.ends_with?(".sst") }.not_nil! + table_path = File.join(path, table_name) + bytes = File.read(table_path).to_slice.dup + bytes[0] = (bytes[0] ^ 0xFF).to_u8 + File.open(table_path, "w") { |io| io.write(bytes) } + + report = Cinderstore::DB.verify(path) + report.valid?.should be_false + report.errors.first.should contain(table_name) + end + end + + it "reports a manifest mismatch without changing the table" do + config = Cinderstore::SpecHelpers.fast_config + Cinderstore::SpecHelpers.with_db_path("verify-manifest") do |path| + db = Cinderstore::DB.new(path, config) + db.put("alpha", "one") + db.flush + db.close + + manifest_path = File.join(path, Cinderstore::DB::MANIFEST_NAME) + manifest = Cinderstore::Manifest.load(manifest_path) + manifest.levels[0][0].count += 1 + manifest.save(manifest_path) + + report = Cinderstore::DB.verify(path) + report.valid?.should be_false + report.errors.should contain("000001.sst: manifest metadata mismatch") + end + end +end + +describe Cinderstore::Wal do + it "rejects a checksum error during verification" do + dir = Cinderstore::SpecHelpers.tmp_db_path("verify-wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + begin + writer = Cinderstore::Wal::Writer.new(path, false) + writer.append(Cinderstore::Entry.new("alpha", 1_i64, true, "one")) + writer.close + + bytes = File.read(path).to_slice.dup + bytes[bytes.size - 1] = (bytes[bytes.size - 1] ^ 0xFF).to_u8 + File.open(path, "w") { |io| io.write(bytes) } + + expect_raises(Cinderstore::CorruptDataError) { Cinderstore::Wal.verify(path) } + ensure + FileUtils.rm_rf(dir) + end + end +end diff --git a/spec/wal_spec.cr b/spec/wal_spec.cr index 3860506..15461a8 100644 --- a/spec/wal_spec.cr +++ b/spec/wal_spec.cr @@ -52,9 +52,10 @@ describe Cinderstore::Wal do writer.append(Cinderstore::Entry.new("alpha", 1_i64, true, "1")) writer.close - # Flip a byte inside the record body. + # Flip a byte inside the record payload. The header is five bytes long, + # so the value byte of the first entry starts at offset 15. bytes = File.read(path).to_slice.dup - bytes[1] = (bytes[1] ^ 0xFF).to_u8 + bytes[15] = (bytes[15] ^ 0xFF).to_u8 File.open(path, "w") { |f| f.write(bytes) } mem = Cinderstore::MemTable.new @@ -62,6 +63,208 @@ describe Cinderstore::Wal do mem.empty?.should be_true end + it "round trips without checksums in fast mode" do + dir = Cinderstore::SpecHelpers.tmp_db_path("wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, false, false) + writer.append(Cinderstore::Entry.new("a", 1_i64, true, "one")) + writer.append(Cinderstore::Entry.new("b", 2_i64, false, "")) + writer.close + + mem = Cinderstore::MemTable.new + seq = Cinderstore::Wal.recover(path, mem, 0_i64) + seq.should eq(2_i64) + mem.get("a").should eq("one") + mem.get("b").should be_nil + mem.get_entry("b").not_nil!.alive.should be_false + end + + it "does not verify payloads in fast mode" do + dir = Cinderstore::SpecHelpers.tmp_db_path("wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, false, false) + writer.append(Cinderstore::Entry.new("alpha", 1_i64, true, "1")) + writer.close + + # Flip the value byte. No checksum exists to catch the change. The + # header is five bytes, then the record length and kind bytes, then the + # key length and key, so the value starts at offset 16. + bytes = File.read(path).to_slice.dup + bytes[16] = (bytes[16] ^ 0xFF).to_u8 + File.open(path, "w") { |f| f.write(bytes) } + + mem = Cinderstore::MemTable.new + Cinderstore::Wal.recover(path, mem, 0_i64) + mem.get("alpha").should_not eq("1") + mem.get_entry("alpha").not_nil!.seq.should eq(1_i64) + end + + it "stops at a torn tail in fast mode" do + dir = Cinderstore::SpecHelpers.tmp_db_path("wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, false, false) + writer.append(Cinderstore::Entry.new("alpha", 1_i64, true, "1")) + writer.append(Cinderstore::Entry.new("beta", 2_i64, true, "2")) + writer.append(Cinderstore::Entry.new("gamma", 3_i64, true, "3")) + writer.close + + # Cut the last record in half. This simulates a torn write. + Cinderstore::SpecHelpers.truncate(path, File.size(path) - 3) + + mem = Cinderstore::MemTable.new + seq = Cinderstore::Wal.recover(path, mem, 0_i64) + seq.should eq(2_i64) + mem.get("alpha").should eq("1") + mem.get("beta").should eq("2") + mem.get("gamma").should be_nil + end + + it "replays a legacy version-1 log that has no header" do + dir = Cinderstore::SpecHelpers.tmp_db_path("wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + # A version-1 log stores framed records back to back with no header. + # Each record carries a checksum. + File.open(path, "w") do |io| + io.write(Cinderstore::Wal.encode(Cinderstore::Entry.new("a", 1_i64, true, "one"))) + io.write(Cinderstore::Wal.encode(Cinderstore::Entry.new("b", 2_i64, true, "two"))) + end + + mem = Cinderstore::MemTable.new + seq = Cinderstore::Wal.recover(path, mem, 0_i64) + seq.should eq(2_i64) + mem.get("a").should eq("one") + mem.get("b").should eq("two") + end + + it "replays a version-2 log whose records have no kind byte" do + dir = Cinderstore::SpecHelpers.tmp_db_path("wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + # A version-2 log from release 0.5 has a header but no kind byte. + # Its records are single entries with checksums. + File.open(path, "w") do |io| + Cinderstore::Wal.write_header(io, true, false) + io.write(Cinderstore::Wal.encode(Cinderstore::Entry.new("a", 1_i64, true, "one"))) + io.write(Cinderstore::Wal.encode(Cinderstore::Entry.new("b", 2_i64, true, "two"))) + end + + mem = Cinderstore::MemTable.new + seq = Cinderstore::Wal.recover(path, mem, 0_i64) + seq.should eq(2_i64) + mem.get("a").should eq("one") + mem.get("b").should eq("two") + end + + it "replays a batch of entries as one record" do + dir = Cinderstore::SpecHelpers.tmp_db_path("wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, false) + writer.append_batch([ + Cinderstore::Entry.new("a", 1_i64, true, "one"), + Cinderstore::Entry.new("b", 2_i64, false, ""), + Cinderstore::Entry.new("c", 3_i64, true, "three"), + ]) + writer.close + + mem = Cinderstore::MemTable.new + seq = Cinderstore::Wal.recover(path, mem, 0_i64) + seq.should eq(3_i64) + mem.get("a").should eq("one") + mem.get("c").should eq("three") + mem.get("b").should be_nil + mem.get_entry("b").not_nil!.alive.should be_false + end + + it "replays a batch followed by a single entry" do + dir = Cinderstore::SpecHelpers.tmp_db_path("wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, false) + writer.append_batch([ + Cinderstore::Entry.new("a", 1_i64, true, "one"), + Cinderstore::Entry.new("b", 2_i64, true, "two"), + ]) + writer.append(Cinderstore::Entry.new("c", 3_i64, true, "three")) + writer.close + + mem = Cinderstore::MemTable.new + seq = Cinderstore::Wal.recover(path, mem, 0_i64) + seq.should eq(3_i64) + mem.get("a").should eq("one") + mem.get("b").should eq("two") + mem.get("c").should eq("three") + end + + it "drops a torn batch whole, without applying any of its entries" do + dir = Cinderstore::SpecHelpers.tmp_db_path("wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, false) + writer.append(Cinderstore::Entry.new("before", 1_i64, true, "ok")) + writer.append_batch([ + Cinderstore::Entry.new("a", 2_i64, true, "one"), + Cinderstore::Entry.new("b", 3_i64, true, "two"), + Cinderstore::Entry.new("c", 4_i64, true, "three"), + ]) + writer.close + + # Cut the batch record short. This simulates a torn write. + Cinderstore::SpecHelpers.truncate(path, File.size(path) - 3) + + mem = Cinderstore::MemTable.new + seq = Cinderstore::Wal.recover(path, mem, 0_i64) + seq.should eq(1_i64) + mem.get("before").should eq("ok") + mem.get("a").should be_nil + mem.get("b").should be_nil + mem.get("c").should be_nil + end + + it "drops a torn batch whole in fast mode" do + dir = Cinderstore::SpecHelpers.tmp_db_path("wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, false, false) + writer.append(Cinderstore::Entry.new("before", 1_i64, true, "ok")) + writer.append_batch([ + Cinderstore::Entry.new("a", 2_i64, true, "one"), + Cinderstore::Entry.new("b", 3_i64, true, "two"), + ]) + writer.close + + Cinderstore::SpecHelpers.truncate(path, File.size(path) - 3) + + mem = Cinderstore::MemTable.new + seq = Cinderstore::Wal.recover(path, mem, 0_i64) + seq.should eq(1_i64) + mem.get("before").should eq("ok") + mem.get("a").should be_nil + mem.get("b").should be_nil + end + + it "round trips a batch in fast mode" do + dir = Cinderstore::SpecHelpers.tmp_db_path("wal") + Dir.mkdir_p(dir) + path = File.join(dir, "000001.wal") + writer = Cinderstore::Wal::Writer.new(path, false, false) + writer.append_batch([ + Cinderstore::Entry.new("a", 1_i64, true, "one"), + Cinderstore::Entry.new("b", 2_i64, false, ""), + ]) + writer.close + + mem = Cinderstore::MemTable.new + seq = Cinderstore::Wal.recover(path, mem, 0_i64) + seq.should eq(2_i64) + mem.get("a").should eq("one") + mem.get("b").should be_nil + end + it "survives a restart through the database" do Cinderstore::SpecHelpers.with_db_path("wal-durable") do |path| db = Cinderstore::DB.new(path) diff --git a/src/cinderstore.cr b/src/cinderstore.cr index e55d3ee..537456b 100644 --- a/src/cinderstore.cr +++ b/src/cinderstore.cr @@ -1,10 +1,13 @@ require "./cinderstore/version" require "./cinderstore/errors" require "./cinderstore/util" +require "./cinderstore/index" require "./cinderstore/entry" require "./cinderstore/skiplist" require "./cinderstore/memtable" require "./cinderstore/wal" +require "./cinderstore/batch" +require "./cinderstore/commit_group" require "./cinderstore/bloom_filter" require "./cinderstore/block_cache" require "./cinderstore/table" @@ -12,5 +15,7 @@ require "./cinderstore/iterator" require "./cinderstore/manifest" require "./cinderstore/db" require "./cinderstore/snapshot" +require "./cinderstore/index_query" +require "./cinderstore/scan_iter" require "./cinderstore/server" require "./cinderstore/demo" diff --git a/src/cinderstore/batch.cr b/src/cinderstore/batch.cr new file mode 100644 index 0000000..3503952 --- /dev/null +++ b/src/cinderstore/batch.cr @@ -0,0 +1,38 @@ +module Cinderstore + # A group of key/value changes written as one atomic operation. + # + # A batch holds puts and deletes. `DB#write` applies the whole batch or + # none of it. The batch becomes a single record in the write ahead log, + # so a crash in the middle of the batch drops the entire batch. + class Batch + @ops = [] of Tuple(String, String, Bool) + + # Queues a live value for `key`. + def put(key : String, value : String) : Nil + @ops << {key, value, true} + end + + # Queues a tombstone for `key`. + def delete(key : String) : Nil + @ops << {key, "", false} + end + + # Returns the number of queued operations. + def size : Int32 + @ops.size + end + + # Returns true when the batch holds no operations. + def empty? : Bool + @ops.empty? + end + + # Yields each queued operation as key, value, and liveness, in the + # order the caller queued it. Deletes yield an empty value. + def each(&block : String, String, Bool ->) : Nil + @ops.each do |key, value, alive| + yield key, value, alive + end + end + end +end diff --git a/src/cinderstore/commit_group.cr b/src/cinderstore/commit_group.cr new file mode 100644 index 0000000..d4fc93d --- /dev/null +++ b/src/cinderstore/commit_group.cr @@ -0,0 +1,87 @@ +require "sync/condition_variable" + +module Cinderstore + # Coalesces durability syncs from concurrent writers. + # + # A writer appends its record to a write ahead log, then registers with + # the group. One committer fiber waits for pending writers, syncs the log + # once, and releases them all. A burst of writes therefore pays for one + # fsync instead of one per write. + class CommitGroup + @mutex = Mutex.new + @cv : Sync::ConditionVariable + @pending : Array(Tuple(Channel(Nil), Wal::Writer)) = [] of Tuple(Channel(Nil), Wal::Writer) + @shutdown : Bool = false + @commits : Int64 = 0_i64 + @done : Channel(Nil) = Channel(Nil).new + @committer : Fiber + + def initialize + @cv = Sync::ConditionVariable.new(@mutex) + @committer = spawn { run } + end + + # Returns how many times the group has synced a log. + getter commits : Int64 + + # Registers the current fiber for a durability sync on `wal`, then + # blocks until the group makes the log durable. + def commit(wal : Wal::Writer) : Nil + gate = Channel(Nil).new + @mutex.synchronize do + if @shutdown + # The committer is gone. Sync directly so the caller still gets + # its durability guarantee. + wal.sync + return + end + @pending << {gate, wal} + @cv.signal + end + gate.receive + end + + # Drains the pending writers, then stops the committer. + def shutdown : Nil + @mutex.synchronize do + @shutdown = true + @cv.broadcast + end + @done.receive + end + + private def run : Nil + loop do + break unless wait_first + # A single yield lets every ready writer register before we drain. + # One sync then covers the whole burst. + Fiber.yield + group = @mutex.synchronize { drain_all } + unless group.empty? + group.map(&.[1]).uniq.each(&.sync) + @mutex.synchronize { @commits += 1 } + group.each { |gate, _wal| gate.send(nil) } + end + end + @done.send(nil) + end + + # Blocks until a writer waits or the group shuts down. Returns true + # when a writer is pending. + private def wait_first : Bool + @mutex.synchronize do + while @pending.empty? && !@shutdown + @cv.wait + end + !@pending.empty? + end + end + + # Removes every pending writer from the list. + private def drain_all : Array(Tuple(Channel(Nil), Wal::Writer)) + group = @pending + @pending = [] of Tuple(Channel(Nil), Wal::Writer) + group + end + end +end diff --git a/src/cinderstore/db.cr b/src/cinderstore/db.cr index 017bf62..47dfb80 100644 --- a/src/cinderstore/db.cr +++ b/src/cinderstore/db.cr @@ -28,6 +28,9 @@ module Cinderstore property cache_blocks : Int32 = 512 # Fsync after every write when true. property sync_writes : Bool = true + # Write CRC32 checksums on new data when true. Set to false for + # higher throughput. Reads still verify files that carry checksums. + property checksums : Bool = true # Number of level-0 tables that trigger compaction. property l0_compact_threshold : Int32 = 4 # Start background compaction after a flush when true. @@ -52,10 +55,13 @@ module Cinderstore getter seq : Int64 getter cache_hits : Int64 getter cache_misses : Int64 + getter writes : Int64 + getter commits : Int64 def initialize(@tables : Int32, @l0 : Int32, @l1 : Int32, @level_counts : Array(Int32), @entries : Int64, @disk_bytes : Int64, @memtable_bytes : Int64, - @wal_bytes : Int64, @seq : Int64, @cache_hits : Int64, @cache_misses : Int64) + @wal_bytes : Int64, @seq : Int64, @cache_hits : Int64, @cache_misses : Int64, + @writes : Int64, @commits : Int64) end def to_h : Hash(String, Int32 | Int64 | Array(Int32)) @@ -71,6 +77,8 @@ module Cinderstore "seq" => @seq, "cache_hits" => @cache_hits, "cache_misses" => @cache_misses, + "writes" => @writes, + "commits" => @commits, } end @@ -86,7 +94,50 @@ module Cinderstore io << "memtable bytes: #{@memtable_bytes}\n" io << "wal bytes: #{@wal_bytes}\n" io << "sequence: #{@seq}\n" - io << "cache hits: #{@cache_hits}, misses: #{@cache_misses}" + io << "cache hits: #{@cache_hits}, misses: #{@cache_misses}\n" + io << "writes: #{@writes}, commits: #{@commits}" + end + end + + # Results from a read-only storage verification. + class VerificationReport + getter tables : Int32 + getter table_entries : Int64 + getter wal_files : Int32 + getter wal_records : Int64 + getter bytes_checked : Int64 + getter errors : Array(String) + + def initialize(@tables : Int32, @table_entries : Int64, @wal_files : Int32, + @wal_records : Int64, @bytes_checked : Int64, @errors : Array(String)) + end + + def valid? : Bool + @errors.empty? + end + + def to_h : Hash(String, Bool | Int32 | Int64 | Array(String)) + { + "valid" => valid?, + "tables" => @tables, + "table_entries" => @table_entries, + "wal_files" => @wal_files, + "wal_records" => @wal_records, + "bytes_checked" => @bytes_checked, + "errors" => @errors, + } + end + + def to_json(io : IO) : Nil + to_h.to_json(io) + end + + def to_s(io : IO) : Nil + io << "verification: #{valid? ? "valid" : "invalid"}\n" + io << "tables: #{@tables}, table entries: #{@table_entries}\n" + io << "wal files: #{@wal_files}, wal records: #{@wal_records}\n" + io << "bytes checked: #{@bytes_checked}" + @errors.each { |error| io << "\nerror: #{error}" } end end @@ -96,13 +147,15 @@ module Cinderstore getter first : String getter last : String getter count : Int64 + getter index_count : Int64 getter path : String @reader : SstableReader? @refs : Int32 = 1 @orphaned : Bool = false - def initialize(@id : Int64, @first : String, @last : String, @count : Int64, @path : String) + def initialize(@id : Int64, @first : String, @last : String, @count : Int64, @path : String, + @index_count : Int64 = 0_i64) end # Returns the cached reader for this table. @@ -168,6 +221,9 @@ module Cinderstore @compacting = false @pending_table_id : Int64 = 0_i64 @manifest : Manifest? = nil + @writes : Int64 = 0_i64 + @commit_group : CommitGroup? = nil + @indexers : Hash(String, Proc(String, String, Array(String))) def initialize(path : String, config : Config = Config.new) @path = path @@ -175,35 +231,249 @@ module Cinderstore @mem = MemTable.new @levels = [[] of TableRef] @block_cache = BlockCache.new(config.cache_blocks) + @indexers = {} of String => Proc(String, String, Array(String)) open + @commit_group = CommitGroup.new if config.sync_writes end def path : String @path end + # Registers a secondary index with a custom extractor. + # + # The block receives the primary key and its value. It returns the + # index keys that identify the value. The store maintains the index + # on every write. Call `query` to find the primary keys that share an + # index key. + def create_index(name : String, &indexer : String, String -> Array(String)) : Nil + validate_index_name(name) + @lock.synchronize do + check_open + raise InvalidIndexError.new("index already exists: #{name}") if @indexers.has_key?(name) + @indexers[name] = indexer + end + end + + # Registers an index that reads one JSON field from every value. + # + # A scalar field yields one index key. An array field yields one key + # per element. A value that is not JSON, or that lacks the field, + # yields no index keys. The indexer never fails a write. + def create_json_index(name : String, field : String) : Nil + raise InvalidIndexError.new("field must not be empty") if field.empty? + create_index(name) do |_key, value| + doc = JSON.parse(value) + Index.json_field_keys(doc, field) + rescue JSON::ParseException + [] of String + end + end + + # Returns true when `name` is a registered index. + def index?(name : String) : Bool + @lock.synchronize { @indexers.has_key?(name) } + end + + # Returns the names of all registered indexes, in creation order. + def indexes : Array(String) + @lock.synchronize { @indexers.keys } + end + + # Returns the primary keys whose value maps to `index_key` in `index`. + # + # The keys come back in primary key order. A negative `limit` means no + # limit. The query reads the materialized index entries, so it works + # whether or not the index is registered in this process. + def query(index : String, index_key : String, limit : Int32 = -1) : Array(String) + validate_index_name(index) + @lock.synchronize do + check_open + prefix = Index.exact_prefix(index, index_key) + keys = [] of String + iter = make_merge_iter(prefix, false) + while entry = iter.next? + break unless entry.key.starts_with?(prefix) + if entry.alive + keys << Index.primary_key_of(entry.key) + break if limit >= 0 && keys.size >= limit + end + end + keys + end + end + + # Returns the primary keys whose index key lies in [start, finish). + # + # The keys come back in index key order, then primary key order. A nil + # `finish_key` means no upper bound. A negative `limit` means no limit. + def query_range(index : String, start_key : String = "", finish_key : String? = nil, + limit : Int32 = -1) : Array(String) + validate_index_name(index) + @lock.synchronize do + check_open + prefix = Index.name_prefix(index) + start = "#{prefix}#{start_key}#{Index::SEP}" + keys = [] of String + iter = make_merge_iter(start, false) + while entry = iter.next? + break unless entry.key.starts_with?(prefix) + index_key = Index.index_key_of(entry.key) + break if finish_key && index_key >= finish_key + if entry.alive + keys << Index.primary_key_of(entry.key) + break if limit >= 0 && keys.size >= limit + end + end + keys + end + end + + # Returns a streaming iterator over the primary keys that map to + # `index_key` in `index`. + # + # The iterator reads a snapshot taken at creation, so it stays + # consistent while the store keeps writing. Keys come back in primary + # key order. The iterator owns its snapshot, so call `close` when you + # are done; that releases the snapshot. + def query_iter(index : String, index_key : String) : IndexIter + validate_index_name(index) + snap = snapshot + begin + iter = snap.query_iter(index, index_key) + iter.take_ownership + iter + rescue ex + snap.release + raise ex + end + end + + # Returns a streaming iterator over the primary keys whose index key + # lies in [start, finish). + # + # The keys come back in index key order, then primary key order. A nil + # `finish_key` means no upper bound. The iterator reads a snapshot + # taken at creation, so it stays consistent while the store keeps + # writing. The iterator owns its snapshot, so call `close` when you are + # done; that releases the snapshot. + def query_range_iter(index : String, start_key : String = "", finish_key : String? = nil) : IndexIter + validate_index_name(index) + snap = snapshot + begin + iter = snap.query_range_iter(index, start_key, finish_key) + iter.take_ownership + iter + rescue ex + snap.release + raise ex + end + end + # Writes a value for `key`. def put(key : String, value : String) : Nil validate_key(key) raise InvalidValueError.new("value exceeds #{MAX_VALUE_BYTES} bytes") if value.bytesize > MAX_VALUE_BYTES + wal : Wal::Writer? = nil @lock.synchronize do check_open @seq += 1 - @wal.not_nil!.append(Entry.new(key, @seq, true, value)) - @mem.put(key, value, @seq) + seq = @seq + entries = maintenance_entries_for(seq, key, value) + main = Entry.new(key, seq, true, value) + wal = @wal.not_nil! + if entries.empty? + wal.append(main) + else + wal.append_batch([main] + entries) + end + @mem.put(key, value, seq) + apply_index_entries(entries) + @writes += 1 end + commit_wal(wal.not_nil!) trigger_flush end # Writes a tombstone for `key`. def delete(key : String) : Nil validate_key(key) + wal : Wal::Writer? = nil @lock.synchronize do check_open @seq += 1 - @wal.not_nil!.append(Entry.new(key, @seq, false, "")) - @mem.delete(key, @seq) + seq = @seq + entries = maintenance_entries_for(seq, key, nil) + main = Entry.new(key, seq, false, "") + wal = @wal.not_nil! + if entries.empty? + wal.append(main) + else + wal.append_batch([main] + entries) + end + @mem.delete(key, seq) + apply_index_entries(entries) + @writes += 1 end + commit_wal(wal.not_nil!) + end + + # Applies `batch` atomically. + # + # Every queued operation becomes one record in the write ahead log. A + # crash before the record is durable drops the whole batch, so the + # store never applies part of it. The sequence numbers of a batch are + # consecutive and assigned in queue order. + def write(batch : Batch) : Nil + return if batch.empty? + wal : Wal::Writer? = nil + @lock.synchronize do + check_open + ops = [] of Tuple(String, String, Bool) + batch.each do |key, value, alive| + validate_key(key) + raise InvalidValueError.new("value exceeds #{MAX_VALUE_BYTES} bytes") if value.bytesize > MAX_VALUE_BYTES + ops << {key, value, alive} + end + return if ops.empty? + entries = [] of Entry + # The index keys a batch op sees start from the previous value the + # batch wrote for the key, or from the store when the batch is the + # first to touch the key. + current_keys = {} of String => Hash(String, Array(String)) + touched = {} of String => Bool + ops.each do |key, value, alive| + @seq += 1 + seq = @seq + unless touched[key]? + touched[key] = true + current_keys[key] = index_keys_for(key, read_live_value(key)) + end + new_keys = alive ? index_keys_for(key, value) : {} of String => Array(String) + entries << Entry.new(key, seq, alive, value) + entries.concat(index_delta_entries(seq, key, current_keys[key], new_keys)) + current_keys[key] = new_keys + end + wal = @wal.not_nil! + wal.append_batch(entries) + entries.each do |entry| + if entry.alive + @mem.put(entry.key, entry.value, entry.seq) + else + @mem.delete(entry.key, entry.seq) + end + end + @writes += 1 + end + commit_wal(wal.not_nil!) + trigger_flush + end + + # Builds a batch in a block, then writes it atomically. + def write(&block : Batch ->) : Nil + batch = Batch.new + yield batch + write(batch) end # Returns the live value for `key`, or nil. @@ -234,6 +504,22 @@ module Cinderstore pairs.each { |key, value| yield key, value } end + # Returns live key/value pairs whose keys start with `prefix`. + # A negative limit means no limit. + def scan_prefix(prefix : String, limit : Int32 = -1) : Array(Tuple(String, String)) + iter = scan_prefix_iter(prefix) + result = [] of Tuple(String, String) + begin + while pair = iter.next? + result << pair + break if limit >= 0 && result.size >= limit + end + result + ensure + iter.close + end + end + # Returns live key/value pairs in key order. The range is [start, finish). # A negative limit means no limit. def scan(start_key : String = "", finish_key : String? = nil, limit : Int32 = -1) : Array(Tuple(String, String)) @@ -243,6 +529,38 @@ module Cinderstore end end + # Returns a streaming iterator over live pairs whose keys start with + # `prefix`. The iterator owns the snapshot it creates. + def scan_prefix_iter(prefix : String) : ScanIter + snap = snapshot + begin + iter = snap.scan_prefix_iter(prefix) + iter.take_ownership + iter + rescue ex + snap.release + raise ex + end + end + + # Returns a streaming iterator over live key/value pairs in key order. + # + # The iterator reads a snapshot taken at creation, so it stays + # consistent while the store keeps writing. The range is [start, finish). + # The iterator owns its snapshot. Call `close` when you are done. + # + def scan_iter(start_key : String = "", finish_key : String? = nil) : ScanIter + snap = snapshot + begin + iter = snap.scan_iter(start_key, finish_key) + iter.take_ownership + iter + rescue ex + snap.release + raise ex + end + end + # Flushes the active memtable to a new sorted table. def flush : Nil @flush_lock.synchronize do @@ -310,7 +628,9 @@ module Cinderstore check_open table_refs = @levels.flatten disk_bytes = table_refs.sum { |ref| file_size(ref.path) } - entries = table_refs.sum(&.count) + @mem.size + (@frozen.try(&.size) || 0_i64) + entries = table_refs.sum { |ref| ref.count - ref.index_count } + + (@mem.size - @mem.index_count) + + (@frozen.try { |m| m.size - m.index_count } || 0_i64) wal_bytes = 0_i64 Dir.children(@path).each do |name| wal_bytes += file_size(File.join(@path, name)) if name.ends_with?(WAL_SUFFIX) @@ -327,10 +647,140 @@ module Cinderstore seq: @seq, cache_hits: @block_cache.hits, cache_misses: @block_cache.misses, + writes: @writes, + commits: @commit_group.try(&.commits) || 0_i64, ) end end + # Verifies a database directory without opening or recovering it. + # + # This entry point is safe for diagnostics because it does not remove + # orphan files, replay logs, or create a new active log. + def self.verify(path : String) : VerificationReport + errors = [] of String + table_entries = 0_i64 + bytes_checked = 0_i64 + table_infos = [] of Manifest::TableInfo + children = begin + Dir.children(path) + rescue ex : Exception + errors << "database directory: #{verification_error(ex)}" + [] of String + end + + manifest_path = File.join(path, MANIFEST_NAME) + bytes_checked += verification_file_size(manifest_path) + if File.exists?(manifest_path) + begin + table_infos = Manifest.load(manifest_path).levels.flatten + rescue ex : Exception + errors << "#{MANIFEST_NAME}: #{verification_error(ex)}" + end + else + errors << "#{MANIFEST_NAME}: file is missing" + end + + known_ids = table_infos.map(&.id) + table_infos.each do |info| + table_path = File.join(path, "#{Util.file_stem(info.id)}#{SST_SUFFIX}") + bytes_checked += verification_file_size(table_path) + reader : SstableReader? = nil + begin + reader = SstableReader.new(table_path, info.id) + actual = reader.not_nil!.verify + table_entries += actual.count + unless actual.id == info.id && actual.first == info.first && + actual.last == info.last && actual.count == info.count && + actual.index_count == info.index_count + errors << "#{File.basename(table_path)}: manifest metadata mismatch" + end + rescue ex : Exception + errors << "#{File.basename(table_path)}: #{verification_error(ex)}" + ensure + reader.try(&.close) + end + end + + children.select { |name| name.ends_with?(SST_SUFFIX) }.sort.each do |name| + id = name.rpartition(".")[0].to_i64 + unless known_ids.includes?(id) + errors << "#{name}: table is not listed in #{MANIFEST_NAME}" + end + end + + wal_files = 0 + wal_records = 0_i64 + children.select { |name| name.ends_with?(WAL_SUFFIX) }.sort.each do |name| + wal_files += 1 + wal_path = File.join(path, name) + bytes_checked += verification_file_size(wal_path) + begin + wal_records += Wal.verify(wal_path) + rescue ex : Exception + errors << "#{name}: #{verification_error(ex)}" + end + end + + VerificationReport.new(table_infos.size.to_i32, table_entries, wal_files, + wal_records, bytes_checked, errors) + end + + # Checks every manifest table and write ahead log without changing data. + # + # The check waits for an active flush, then blocks writes for its duration. + # It verifies table blocks, bloom filters, WAL framing, and manifest + # metadata. Fast-mode files can only be checked for layout, not bit rot. + def verify : VerificationReport + @flush_lock.synchronize do + @lock.synchronize do + check_open + errors = [] of String + table_refs = @levels.flatten + table_entries = 0_i64 + bytes_checked = 0_i64 + bytes_checked += file_size(File.join(@path, MANIFEST_NAME)) + + table_refs.each do |ref| + bytes_checked += file_size(ref.path) + reader : SstableReader? = nil + begin + # Use a fresh reader so verification bypasses caches and does + # not move the file position of a concurrent snapshot reader. + reader = SstableReader.new(ref.path, ref.id) + actual = reader.not_nil!.verify + table_entries += actual.count + unless actual.id == ref.id && actual.first == ref.first && + actual.last == ref.last && actual.count == ref.count && + actual.index_count == ref.index_count + errors << "#{File.basename(ref.path)}: manifest metadata mismatch" + end + rescue ex : Exception + errors << "#{File.basename(ref.path)}: #{verification_error(ex)}" + ensure + reader.try(&.close) + end + end + + wal_files = 0 + wal_records = 0_i64 + Dir.children(@path).select { |name| name.ends_with?(WAL_SUFFIX) }.sort.each do |name| + wal_files += 1 + path = File.join(@path, name) + bytes_checked += file_size(path) + begin + wal_records += Wal.verify(path) + rescue ex : Exception + errors << "#{name}: #{verification_error(ex)}" + end + end + + VerificationReport.new(table_refs.size.to_i32, table_entries, wal_files, + wal_records, bytes_checked, errors) + end + end + end + # Flushes pending work, syncs the manifest, and releases file handles. def close : Nil @flush_lock.synchronize do @@ -343,6 +793,95 @@ module Cinderstore @levels.flatten.each(&.close) end end + @commit_group.try(&.shutdown) + end + + # ------------------------------------------------------------------ + # Secondary indexes + # ------------------------------------------------------------------ + + # Builds the index entries a single write needs. + # + # The previous live value of the key supplies the old index keys. The + # new value supplies the new keys. Removed associations become index + # tombstones. Added associations become live entries. A newer entry + # supersedes an older one for the same association, so stale entries + # drop naturally during compaction. + private def maintenance_entries_for(seq : Int64, key : String, new_value : String?) : Array(Entry) + return [] of Entry if @indexers.empty? + new_keys = new_value ? index_keys_for(key, new_value) : {} of String => Array(String) + old_keys = index_keys_for(key, read_live_value(key)) + index_delta_entries(seq, key, old_keys, new_keys) + end + + # Returns the current live value for `key`, or nil. Callers hold the + # database lock, so the value is consistent with the write order. + private def read_live_value(key : String) : String? + if entry = @mem.get_entry(key) + return entry.alive ? entry.value : nil + end + if entry = @frozen.try { |m| m.get_entry(key) } + return entry.alive ? entry.value : nil + end + iter = LiveIter.new(make_merge_iter(key, true)) + if entry = iter.next? + return entry.value if entry.key == key + end + nil + end + + # Returns the index keys every registered index derives from `value`. + private def index_keys_for(key : String, value : String?) : Hash(String, Array(String)) + result = {} of String => Array(String) + return result if value.nil? + @indexers.each do |name, indexer| + indexer.call(key, value).each do |index_key| + raise InvalidIndexError.new("index key for #{name} contains a NUL byte") if index_key.includes?(Index::SEP) + next if index_key.empty? + (result[name] ||= [] of String) << index_key + end + end + result.each { |name, keys| result[name] = keys.uniq } + result + end + + # Builds the entries that move an association from `old_keys` to + # `new_keys`. Added index keys become live entries. Removed index + # keys become tombstones. Each entry shares the sequence number of the + # primary write, so both land in one write ahead log record. + private def index_delta_entries(seq : Int64, key : String, + old_keys : Hash(String, Array(String)), + new_keys : Hash(String, Array(String))) : Array(Entry) + return [] of Entry if @indexers.empty? + raise InvalidKeyError.new("keys with a NUL byte cannot be indexed") if key.includes?(Index::SEP) + entries = [] of Entry + @indexers.each_key do |name| + added = new_keys[name]? || [] of String + removed = old_keys[name]? || [] of String + (added - removed).each do |index_key| + entries << Entry.new(Index.entry_key(name, index_key, key), seq, true, "") + end + (removed - added).each do |index_key| + entries << Entry.new(Index.entry_key(name, index_key, key), seq, false, "") + end + end + entries + end + + # Applies index entries to the memtable after a write. + private def apply_index_entries(entries : Array(Entry)) : Nil + entries.each do |entry| + if entry.alive + @mem.put(entry.key, "", entry.seq) + else + @mem.delete(entry.key, entry.seq) + end + end + end + + private def validate_index_name(name : String) : Nil + raise InvalidIndexError.new("index name must not be empty") if name.empty? + raise InvalidIndexError.new("index name contains a NUL byte") if name.includes?(Index::SEP) end # ------------------------------------------------------------------ @@ -365,7 +904,7 @@ module Cinderstore private def load_tables(manifest : Manifest) : Nil @levels = manifest.levels.map do |level| level.map do |info| - TableRef.new(info.id, info.first, info.last, info.count, table_path(info.id)) + TableRef.new(info.id, info.first, info.last, info.count, table_path(info.id), info.index_count) end end @levels = [[] of TableRef] if @levels.empty? @@ -404,7 +943,7 @@ module Cinderstore private def create_active_wal : Nil id = @next_id @next_id += 1 - @wal = Wal::Writer.new(wal_path(id), @config.sync_writes) + @wal = Wal::Writer.new(wal_path(id), @config.sync_writes, @config.checksums) end # ------------------------------------------------------------------ @@ -459,7 +998,7 @@ module Cinderstore @next_id += 1 wal_id = @next_id @next_id += 1 - @wal = Wal::Writer.new(wal_path(wal_id), @config.sync_writes) + @wal = Wal::Writer.new(wal_path(wal_id), @config.sync_writes, @config.checksums) end end @@ -468,7 +1007,7 @@ module Cinderstore meta = write_memtable_to_table(table_id, frozen) @lock.synchronize do - @levels[0] << TableRef.new(meta.id, meta.first, meta.last, meta.count, table_path(meta.id)) + @levels[0] << TableRef.new(meta.id, meta.first, meta.last, meta.count, table_path(meta.id), meta.index_count) @frozen = nil if old = @old_wal old.close @@ -489,7 +1028,7 @@ module Cinderstore final_path = table_path(table_id) meta = nil File.open(tmp_path, "w") do |io| - writer = SstableWriter.new(io, table_id, @config.block_size, @config.bloom_fpp) + writer = SstableWriter.new(io, table_id, @config.block_size, @config.bloom_fpp, @config.checksums) mem.each_entry do |entry| writer.add(entry.key, entry.value, entry.seq, entry.alive) end @@ -513,6 +1052,17 @@ module Cinderstore end end + # Makes the appended record durable. + # + # With `sync_writes` enabled the commit group batches this call with + # concurrent writers, so one fsync covers many records. Without it, no + # fsync happens and the call returns immediately. + private def commit_wal(wal : Wal::Writer) : Nil + if group = @commit_group + group.commit(wal) + end + end + private def trigger_compact : Nil return unless @config.compact_on_flush return if @compacting @@ -544,7 +1094,7 @@ module Cinderstore outputs = merge_tables(tables.not_nil!, true) @lock.synchronize do - @levels = [[] of TableRef, outputs.map { |m| TableRef.new(m.id, m.first, m.last, m.count, table_path(m.id)) }] + @levels = [[] of TableRef, outputs.map { |m| TableRef.new(m.id, m.first, m.last, m.count, table_path(m.id), m.index_count) }] save_manifest tables.not_nil!.each do |ref| # Mark the table as gone, then drop the database reference. The @@ -604,7 +1154,7 @@ module Cinderstore @lock.synchronize do return if @closed kept = @levels[target].select { |ref| !overlap.not_nil!.includes?(ref) } - @levels[target] = (kept + outputs.map { |m| TableRef.new(m.id, m.first, m.last, m.count, table_path(m.id)) }).sort_by!(&.first) + @levels[target] = (kept + outputs.map { |m| TableRef.new(m.id, m.first, m.last, m.count, table_path(m.id), m.index_count) }).sort_by!(&.first) @levels[level] = [] of TableRef inputs.each do |ref| ref.orphan @@ -649,7 +1199,7 @@ module Cinderstore id end io = File.open(File.join(@path, "#{Util.file_stem(current_id)}#{TMP_SUFFIX}"), "w") - writer = SstableWriter.new(io.not_nil!, current_id, @config.block_size, @config.bloom_fpp) + writer = SstableWriter.new(io.not_nil!, current_id, @config.block_size, @config.bloom_fpp, @config.checksums) end writer.not_nil!.add(entry.key, entry.value, entry.seq, entry.alive) written += 1 @@ -686,7 +1236,7 @@ module Cinderstore manifest.next_id = @next_id manifest.levels = @levels.map do |level| level.map do |ref| - Manifest::TableInfo.new(ref.id, ref.first, ref.last, ref.count) + Manifest::TableInfo.new(ref.id, ref.first, ref.last, ref.count, ref.index_count) end end manifest.save(File.join(@path, MANIFEST_NAME)) @@ -703,6 +1253,7 @@ module Cinderstore private def validate_key(key : String) : Nil raise InvalidKeyError.new("key must not be empty") if key.empty? raise InvalidKeyError.new("key exceeds #{MAX_KEY_BYTES} bytes") if key.bytesize > MAX_KEY_BYTES + raise InvalidKeyError.new("key starts with the reserved index prefix") if Index.internal_key?(key) end private def check_open : Nil @@ -714,5 +1265,19 @@ module Cinderstore rescue File::NotFoundError 0_i64 end + + private def verification_error(error : Exception) : String + error.message || error.class.to_s + end + + private def self.verification_error(error : Exception) : String + error.message || error.class.to_s + end + + private def self.verification_file_size(path : String) : Int64 + File.size(path) + rescue File::NotFoundError + 0_i64 + end end end diff --git a/src/cinderstore/demo.cr b/src/cinderstore/demo.cr index d2b2fc1..c10a0b8 100644 --- a/src/cinderstore/demo.cr +++ b/src/cinderstore/demo.cr @@ -6,11 +6,11 @@ module Cinderstore # The demo loads a small product catalog, writes it, scans a range, # flushes, compacts, and verifies recovery. Its output is deterministic. class Demo - def self.run(db_path : String? = nil, fixture : String? = nil) : Int32 - new(db_path, fixture).run + def self.run(db_path : String? = nil, fixture : String? = nil, checksums : Bool = true) : Int32 + new(db_path, fixture, checksums).run end - def initialize(@db_path : String? = nil, @fixture : String? = nil) + def initialize(@db_path : String? = nil, @fixture : String? = nil, @checksums : Bool = true) end def run : Int32 @@ -19,21 +19,22 @@ module Cinderstore fixture_path = resolve_fixture raise Error.new("fixture not found: #{fixture_path}") unless File.exists?(fixture_path) - rows = load_fixture(fixture_path) - raise Error.new("fixture is empty") if rows.empty? + catalog = load_fixture(fixture_path) + raise Error.new("fixture is empty") if catalog.empty? config = DB::Config.new config.sync_writes = false config.compact_on_flush = false + config.checksums = @checksums db = DB.new(path, config) puts "== Cinderstore #{VERSION} demo ==" puts "" - puts "Loaded #{rows.size} products from #{fixture_path}" + puts "Loaded #{catalog.size} products from #{fixture_path}" puts "Database directory: #{path}" puts "" - rows.each do |sku, name, price, stock| + catalog.each do |sku, name, price, stock| db.put(sku, %({"name":"#{name}","price":#{price},"stock":#{stock}})) end @@ -110,6 +111,7 @@ module Cinderstore levels_config = DB::Config.new levels_config.sync_writes = false levels_config.compact_on_flush = false + levels_config.checksums = @checksums levels_config.memtable_limit = 2_000 levels_config.l0_compact_threshold = 2 levels_config.level_ratio = 2.0 @@ -128,10 +130,187 @@ module Cinderstore level_db.close puts "" + puts "10. Fast mode skips checksums on new writes" + checked_path = File.join(Dir.tempdir, "cinderstore-checked-demo") + fast_path = File.join(Dir.tempdir, "cinderstore-fast-demo") + FileUtils.rm_rf(checked_path) if File.exists?(checked_path) + FileUtils.rm_rf(fast_path) if File.exists?(fast_path) + checked = write_mode_demo(checked_path, catalog, true) + fast = write_mode_demo(fast_path, catalog, false) + puts " checksummed: wal #{checked[0]} bytes, disk #{checked[1]} bytes" + puts " fast mode: wal #{fast[0]} bytes, disk #{fast[1]} bytes" + reopened = DB.new(fast_path) + puts " all #{reopened.scan.size} rows readable after a fast-mode restart" + reopened.close + puts "" + + puts "11. Batch writes and group commit" + batch_path = File.join(Dir.tempdir, "cinderstore-batch-demo") + FileUtils.rm_rf(batch_path) if File.exists?(batch_path) + batch_config = DB::Config.new + batch_config.sync_writes = false + batch_config.compact_on_flush = false + batch_db = DB.new(batch_path, batch_config) + batch_db.write do |b| + catalog.each do |sku, name, price, stock| + b.put(sku, %({"name":"#{name}","price":#{price},"stock":#{stock}})) + end + end + puts " one batch wrote #{batch_db.scan.size} rows as 1 write operation" + puts " sequence after the batch: #{batch_db.stats.seq}" + batch_db.close + reopened = DB.new(batch_path) + puts " all #{reopened.scan.size} rows recovered after a restart" + reopened.close + puts "" + + group_path = File.join(Dir.tempdir, "cinderstore-group-demo") + FileUtils.rm_rf(group_path) if File.exists?(group_path) + group_config = DB::Config.new + group_config.sync_writes = true + group_config.compact_on_flush = false + group_config.memtable_limit = 1_i64 << 30 + group_db = DB.new(group_path, group_config) + done = Channel(Nil).new + 8.times do |i| + spawn do + 25.times do |j| + group_db.put("grow-%d-%02d" % {i, j}, "value-%d" % j) + end + done.send(nil) + end + end + 8.times { done.receive } + puts " concurrent writes: #{group_db.stats.writes}, durability commits: #{group_db.stats.commits}" + group_db.close + reopened = DB.new(group_path) + puts " all #{reopened.scan.size} rows recovered after a restart" + reopened.close + puts "" + + puts "12. Secondary indexes track derived views" + index_path = File.join(Dir.tempdir, "cinderstore-index-demo") + FileUtils.rm_rf(index_path) if File.exists?(index_path) + index_config = DB::Config.new + index_config.sync_writes = false + index_config.compact_on_flush = false + index_db = DB.new(index_path, index_config) + index_db.create_json_index("by-name", "name") + index_db.create_json_index("by-price", "price") + index_db.create_json_index("by-stock", "stock") + catalog.each do |sku, name, price, stock| + index_db.put(sku, %({"name":"#{name}","price":#{price},"stock":#{stock}})) + end + index_db.flush + puts " created indexes on name, price, and stock" + puts " query by-name \"Ash Rake Forged\" => #{format_keys(index_db.query("by-name", "Ash Rake Forged"))}" + puts " query by-price \"14.0\" => #{format_keys(index_db.query("by-price", "14.0"))}" + puts " query by-stock \"31\" => #{format_keys(index_db.query("by-stock", "31"))}" + puts " stock range 20 to 30 => #{format_keys(index_db.query_range("by-stock", "20", "30"))}" + index_db.put("SKU-0010", %({"name":"Ash Rake Forged","price":15.50,"stock":31})) + index_db.flush + index_db.compact + puts " update SKU-0010 price to 15.50, then flush and compact" + puts " query by-price \"14.0\" => #{format_keys(index_db.query("by-price", "14.0"))}" + puts " query by-price \"15.5\" => #{format_keys(index_db.query("by-price", "15.5"))}" + index_db.delete("SKU-0011") + index_db.flush + index_db.compact + puts " delete SKU-0011, then flush and compact" + puts " query by-name \"Coal Shovel Small\" => #{format_keys(index_db.query("by-name", "Coal Shovel Small"))}" + puts "" + + puts "13. Streaming iterators for secondary index queries" + streamed = [] of String + iter = index_db.query_iter("by-price", "15.5") + while key = iter.next? + streamed << key + end + iter.close + puts " streamed by-price \"15.5\" => #{format_keys(streamed)}" + taken = [] of String + qiter = index_db.query_range_iter("by-stock", "20", "30") + while key = qiter.next? + taken << key + break if taken.size >= 3 + end + qiter.close + puts " stock range 20 to 30, first 3 => #{format_keys(taken)}" + snap = index_db.snapshot + index_db.delete("SKU-0012") + siter = snap.query_iter("by-name", "Ember Tray Brass") + puts " snapshot stream \"Ember Tray Brass\" => #{format_keys(siter.to_a)}" + siter.close + puts " live query after the delete => #{format_keys(index_db.query("by-name", "Ember Tray Brass"))}" + snap.release + index_db.close + reopened = DB.new(index_path) + puts " reopen and query by-stock \"31\" => #{format_keys(reopened.query("by-stock", "31"))}" + puts "14. Streaming primary-key range scans" + range_keys = [] of String + range_iter = reopened.scan_iter("SKU-0010", "SKU-0016") + range_iter.each { |key, _value| range_keys << key } + range_iter.close + puts " primary range SKU-0010 to SKU-0016 => #{format_keys(range_keys)}" + range_snapshot = reopened.snapshot + reopened.delete("SKU-0010") + snapshot_range_keys = [] of String + snapshot_range_iter = range_snapshot.scan_iter("SKU-0010", "SKU-0016") + snapshot_range_iter.each { |key, _value| snapshot_range_keys << key } + snapshot_range_iter.close + range_snapshot.release + puts " snapshot still sees SKU-0010 => #{format_keys(snapshot_range_keys)}" + puts "15. Prefix scans group keys" + prefix_keys = [] of String + prefix_iter = reopened.scan_prefix_iter("SKU-001") + prefix_iter.each { |key, _value| prefix_keys << key } + prefix_iter.close + puts " live prefix SKU-001 => #{format_keys(prefix_keys)}" + puts "" + + puts "16. Verify persistent files without changing them" + report = reopened.verify + puts " valid: #{report.valid?}" + puts " tables checked: #{report.tables}, WAL files checked: #{report.wal_files}" + puts " table entries: #{report.table_entries}, WAL records: #{report.wal_records}" + puts " bytes checked: #{report.bytes_checked}" + reopened.close + puts "" + puts "Demo complete." 0 end + # Formats a query result for the demo output. + private def format_keys(keys : Array(String)) : String + keys.empty? ? "(none)" : keys.join(", ") + end + + # Writes the catalog into a fresh database and reports the file sizes. + # + # The catalog is written three times so the checksums produce a clear + # byte difference. The return value is the WAL size before the flush + # and the table size after it. + private def write_mode_demo(db_path : String, rows : Array(Tuple(String, String, String, String)), + checksums : Bool) : Tuple(Int64, Int64) + config = DB::Config.new + config.sync_writes = false + config.compact_on_flush = false + config.block_size = 128 + config.checksums = checksums + db = DB.new(db_path, config) + 3.times do |round| + rows.each do |sku, name, price, stock| + db.put("%d-%s" % {round, sku}, %({"name":"#{name}","price":#{price},"stock":#{stock}})) + end + end + wal_bytes = db.stats.wal_bytes + db.flush + disk_bytes = db.stats.disk_bytes + db.close + {wal_bytes, disk_bytes} + end + private def print_stats(db : DB) : Nil stats = db.stats puts " tables: #{stats.tables} (l0: #{stats.l0}, l1: #{stats.l1}), entries: #{stats.entries}" diff --git a/src/cinderstore/errors.cr b/src/cinderstore/errors.cr index 880ad91..978d85e 100644 --- a/src/cinderstore/errors.cr +++ b/src/cinderstore/errors.cr @@ -26,4 +26,8 @@ module Cinderstore # Raised when an operation targets a released snapshot. class SnapshotReleasedError < Error end + + # Raised when a secondary index definition or index key is invalid. + class InvalidIndexError < Error + end end diff --git a/src/cinderstore/index.cr b/src/cinderstore/index.cr new file mode 100644 index 0000000..e4b5456 --- /dev/null +++ b/src/cinderstore/index.cr @@ -0,0 +1,90 @@ +require "json" + +module Cinderstore + # Reserved namespace and key layout for secondary indexes. + # + # An index entry is an ordinary entry whose key starts with NAMESPACE. + # The key has this layout: + # + # NAMESPACE + name + SEP + index_key + SEP + primary_key + # + # The entries share the main log structured merge tree with the primary + # data. They ride the same write ahead log, sorted tables, compaction, + # and recovery. A newer tombstone supersedes a stale entry, exactly as + # it does for primary keys. + module Index + # Reserved prefix for every internal index entry. + NAMESPACE = "\u{0000}idx\u{0000}" + # Byte that separates the parts of an index entry key. + SEP = '\u{0000}' + + # Returns the stored key for one index association. + def self.entry_key(name : String, index_key : String, primary_key : String) : String + "#{NAMESPACE}#{name}#{SEP}#{index_key}#{SEP}#{primary_key}" + end + + # Returns the key prefix that covers every entry of one index. + def self.name_prefix(name : String) : String + "#{NAMESPACE}#{name}#{SEP}" + end + + # Returns the key prefix that covers every entry of one index key. + def self.exact_prefix(name : String, index_key : String) : String + "#{name_prefix(name)}#{index_key}#{SEP}" + end + + # Returns true when `key` belongs to the internal index namespace. + def self.internal_key?(key : String) : Bool + key.starts_with?(NAMESPACE) + end + + # Splits an index entry key into its name, index key, and primary key. + def self.split_entry(key : String) : Tuple(String, String, String) + raise CorruptDataError.new("not an index entry key") unless key.starts_with?(NAMESPACE) + rest = key[NAMESPACE.size..] + first = rest.index(SEP) + raise CorruptDataError.new("malformed index entry key") unless first + name = rest[0, first] + rest = rest[first + 1..] + second = rest.index(SEP) + raise CorruptDataError.new("malformed index entry key") unless second + index_key = rest[0, second] + primary_key = rest[second + 1..] + {name, index_key, primary_key} + end + + # Returns the index key part of an index entry key. + def self.index_key_of(key : String) : String + split_entry(key)[1] + end + + # Returns the primary key part of an index entry key. + def self.primary_key_of(key : String) : String + split_entry(key)[2] + end + + # Extracts the string keys for a JSON value field. + # + # A scalar field yields one key. An array field yields one key per + # element. A missing field or a non-scalar value yields no keys. A + # value that is not JSON yields no keys, so it never breaks a write. + def self.json_field_keys(doc : JSON::Any, field : String) : Array(String) + return [] of String if field.empty? + value = doc[field]? + return [] of String unless value + keys = [] of String + case raw = value.raw + when String, Int64, Float64, Bool + keys << raw.to_s + when Array + raw.each do |item| + case item.raw + when String, Int64, Float64, Bool + keys << item.raw.to_s + end + end + end + keys + end + end +end diff --git a/src/cinderstore/index_query.cr b/src/cinderstore/index_query.cr new file mode 100644 index 0000000..053dd8f --- /dev/null +++ b/src/cinderstore/index_query.cr @@ -0,0 +1,57 @@ +module Cinderstore + # Streams the primary keys of one secondary index query. + # + # The iterator reads from a snapshot, so it stays consistent while the + # database keeps writing. Each `next?` returns the primary key of the + # next live index association, in index key order. + # + # A database-level iterator owns the snapshot it reads. Close it when you + # are done; that releases the snapshot. A snapshot-level iterator shares + # the snapshot you created, so closing it never releases that snapshot. + class IndexIter + @own_snapshot : Bool + + def initialize(@snapshot : Snapshot, @inner : SnapshotIter, @prefix : String, + @finish_key : String?, @own_snapshot : Bool = false) + end + + # Returns the next primary key, or nil at the end of the query. + def next? : String? + finish = @finish_key + while entry = @inner.next? + break unless entry.key.starts_with?(@prefix) + index_key = Index.index_key_of(entry.key) + break if finish && index_key >= finish + return Index.primary_key_of(entry.key) if entry.alive + end + nil + end + + # Yields every primary key in order. + def each(&block : String ->) : Nil + while key = next? + yield key + end + end + + # Returns every primary key in order. + def to_a : Array(String) + keys = [] of String + each { |key| keys << key } + keys + end + + # Marks this iterator as the owner of its snapshot. Closing the + # iterator then releases the snapshot. The database uses this for the + # snapshot it creates internally. + def take_ownership : Nil + @own_snapshot = true + end + + # Releases the snapshot this iterator owns, if any. + def close : Nil + @inner.close + @snapshot.release if @own_snapshot + end + end +end diff --git a/src/cinderstore/iterator.cr b/src/cinderstore/iterator.cr index 8c5120d..eb8959e 100644 --- a/src/cinderstore/iterator.cr +++ b/src/cinderstore/iterator.cr @@ -202,7 +202,8 @@ module Cinderstore end end - # Wraps an iterator and hides tombstone entries. + # Wraps an iterator and hides tombstone entries and internal index + # entries. User-facing reads never see the secondary index namespace. class LiveIter < Store::Iter def initialize(@inner : Store::Iter) end @@ -213,6 +214,7 @@ module Cinderstore def next? : Entry? while entry = @inner.next? + next if Index.internal_key?(entry.key) return entry if entry.alive end nil diff --git a/src/cinderstore/manifest.cr b/src/cinderstore/manifest.cr index 64f8cca..ae0f3e2 100644 --- a/src/cinderstore/manifest.cr +++ b/src/cinderstore/manifest.cr @@ -14,11 +14,13 @@ module Cinderstore property first : String = "" property last : String = "" property count : Int64 = 0_i64 + property index_count : Int64 = 0_i64 def initialize end - def initialize(@id : Int64, @first : String, @last : String, @count : Int64) + def initialize(@id : Int64, @first : String, @last : String, @count : Int64, + @index_count : Int64 = 0_i64) end end diff --git a/src/cinderstore/memtable.cr b/src/cinderstore/memtable.cr index 8d67b94..40d4e29 100644 --- a/src/cinderstore/memtable.cr +++ b/src/cinderstore/memtable.cr @@ -10,6 +10,7 @@ module Cinderstore @table : SkipList @bytes : Int64 = 0_i64 @count : Int64 = 0_i64 + @index_count : Int64 = 0_i64 def initialize(rng : Random = Random.new) @table = SkipList.new(rng) @@ -27,6 +28,11 @@ module Cinderstore @bytes end + # Returns how many held entries belong to the internal index namespace. + def index_count : Int64 + @index_count + end + # Writes a live value for `key`. Overwrites any previous entry. def put(key : String, value : String, seq : Int64) : Nil prev = @table.get(key) @@ -35,6 +41,7 @@ module Cinderstore else @bytes += key.bytesize + value.bytesize + OVERHEAD @count += 1 + @index_count += 1 if Index.internal_key?(key) end @table.insert(key, seq, true, value) end @@ -47,6 +54,7 @@ module Cinderstore else @bytes += key.bytesize + OVERHEAD @count += 1 + @index_count += 1 if Index.internal_key?(key) end @table.insert(key, seq, false, "") end diff --git a/src/cinderstore/scan_iter.cr b/src/cinderstore/scan_iter.cr new file mode 100644 index 0000000..3888433 --- /dev/null +++ b/src/cinderstore/scan_iter.cr @@ -0,0 +1,57 @@ +module Cinderstore + # Streams live key/value pairs over a primary key range or prefix. + # + # The iterator reads from a snapshot, so it stays consistent while the + # database keeps writing. Each `next?` returns the next live pair in + # ascending key order. The range is [start, finish). + # + # A database-level iterator owns the snapshot it reads. Close it when you + # are done; that releases the snapshot. A snapshot-level iterator shares + # the snapshot you created, so closing it never releases that snapshot. + class ScanIter + @own_snapshot : Bool + + def initialize(@snapshot : Snapshot, @inner : Store::Iter, @finish_key : String?, + @prefix : String? = nil, @own_snapshot : Bool = false) + end + + # Returns the next key/value pair, or nil at the end of the range. + def next? : Tuple(String, String)? + finish = @finish_key + prefix = @prefix + while entry = @inner.next? + break if finish && entry.key >= finish + next if prefix && !entry.key.starts_with?(prefix) + return {entry.key, entry.value} + end + nil + end + + # Yields every key/value pair in key order. + def each(&block : String, String ->) : Nil + while pair = next? + yield pair[0], pair[1] + end + end + + # Returns every key/value pair in key order. + def to_a : Array(Tuple(String, String)) + pairs = [] of Tuple(String, String) + each { |key, value| pairs << {key, value} } + pairs + end + + # Marks this iterator as the owner of its snapshot. Closing the + # iterator then releases the snapshot. The database uses this for the + # snapshot it creates internally. + def take_ownership : Nil + @own_snapshot = true + end + + # Releases the snapshot this iterator owns, if any. + def close : Nil + @inner.close + @snapshot.release if @own_snapshot + end + end +end diff --git a/src/cinderstore/server.cr b/src/cinderstore/server.cr index c5a7b0f..8a67725 100644 --- a/src/cinderstore/server.cr +++ b/src/cinderstore/server.cr @@ -1,7 +1,7 @@ require "socket" module Cinderstore - # A local TCP server that exposes get, put, and delete. + # A local TCP server that exposes reads, writes, and maintenance commands. # # Each client connection runs in its own fiber. Commands are newline # terminated. See the README for the full protocol reference. @@ -59,10 +59,16 @@ module Cinderstore next if line.empty? begin command = parse_command(line) - response = execute(command) shutdown = true if command.op == "SHUTDOWN" - client << response << "\n" - client.flush + if command.op == "SCAN" || command.op == "SCANPREFIX" + stream_scan(client, command) + elsif command.op == "QUERY" || command.op == "QUERYRANGE" + stream_index_query(client, command) + else + response = execute(command) + client << response << "\n" + client.flush + end rescue ex : Error client << "ERR #{ex.message}\n" client.flush @@ -89,10 +95,24 @@ module Cinderstore Command.new(op, parts[1], nil, "", nil, -1) when "SCAN" tokens = line.split(" ") + raise ProtocolError.new("too many arguments") if tokens.size > 4 start = tokens[1]? || "" finish = tokens[2]? - limit = tokens[3]?.try(&.to_i?) || -1 + limit = parse_limit(tokens[3]?) Command.new(op, "", nil, start, finish, limit) + when "SCANPREFIX" + tokens = line.split(" ") + raise ProtocolError.new("missing prefix") if tokens.size < 2 + raise ProtocolError.new("too many arguments") if tokens.size > 3 + Command.new(op, "", nil, tokens[1], nil, parse_limit(tokens[2]?)) + when "QUERY" + raise ProtocolError.new("missing index") if parts.size < 2 + raise ProtocolError.new("missing index key") if parts.size < 3 + Command.new(op, parts[1], parts[2], "", nil, -1) + when "QUERYRANGE" + tokens = line.split(" ") + raise ProtocolError.new("missing index") if tokens.size < 2 + Command.new(op, tokens[1], nil, tokens[2]? || "", tokens[3]?, -1) when "STATS", "PING", "FLUSH", "COMPACT", "SHUTDOWN" Command.new(op, "", nil, "", nil, -1) else @@ -114,18 +134,6 @@ module Cinderstore when "DEL", "DELETE" @db.delete(command.key) "OK" - when "SCAN" - rows = @db.scan(command.start, command.finish, command.limit) - if rows.empty? - "END" - else - String.build do |s| - rows.each do |key, value| - s << "ROW #{key} #{value}\n" - end - s << "END" - end - end when "STATS" "STATS #{@db.stats.to_json}" when "PING" @@ -143,5 +151,59 @@ module Cinderstore raise ProtocolError.new("unknown command") end end + + # Streams a primary-key scan as one ROW line per live pair, then END. + # The iterator owns a snapshot, so writes after the command starts do not + # change the result. A limit stops the iterator before the next row. + private def stream_scan(client : TCPSocket, command : Command) : Nil + iter = if command.op == "SCANPREFIX" + @db.scan_prefix_iter(command.start) + else + @db.scan_iter(command.start, command.finish) + end + count = 0 + begin + while command.limit < 0 || count < command.limit + pair = iter.next? + break unless pair + client << "ROW #{pair[0]} #{pair[1]}\n" + client.flush + count += 1 + end + client << "END\n" + client.flush + ensure + iter.close + end + end + + # Streams a query result as one ROW line per primary key, then END. + # + # The server never materializes the result. Each row is flushed as the + # iterator produces it, so a large result keeps a small memory footprint. + private def stream_index_query(client : TCPSocket, command : Command) : Nil + iter = if command.op == "QUERY" + @db.query_iter(command.key, command.value.not_nil!) + else + @db.query_range_iter(command.key, command.start, command.finish) + end + begin + while key = iter.next? + client << "ROW #{key}\n" + client.flush + end + client << "END\n" + ensure + iter.close + end + end + + private def parse_limit(value : String?) : Int32 + return -1 unless value + limit = value.to_i? + raise ProtocolError.new("limit must be an integer") unless limit + raise ProtocolError.new("limit must not be less than -1") if limit < -1 + limit + end end end diff --git a/src/cinderstore/snapshot.cr b/src/cinderstore/snapshot.cr index 40fa508..fd38f48 100644 --- a/src/cinderstore/snapshot.cr +++ b/src/cinderstore/snapshot.cr @@ -43,6 +43,22 @@ module Cinderstore end end + # Returns live key/value pairs whose keys start with `prefix`. + # A negative limit means no limit. + def scan_prefix(prefix : String, limit : Int32 = -1) : Array(Tuple(String, String)) + iter = scan_prefix_iter(prefix) + result = [] of Tuple(String, String) + begin + while pair = iter.next? + result << pair + break if limit >= 0 && result.size >= limit + end + result + ensure + iter.close + end + end + # Returns live key/value pairs in key order. The range is [start, finish). # A negative limit means no limit. def scan(start_key : String = "", finish_key : String? = nil, limit : Int32 = -1) : Array(Tuple(String, String)) @@ -61,11 +77,81 @@ module Cinderstore end end + # Returns a streaming iterator over live pairs whose keys start with + # `prefix`. The iterator borrows this snapshot. + def scan_prefix_iter(prefix : String) : ScanIter + @lock.synchronize do + check_released + finish = Util.prefix_end(prefix) + inner = SnapshotIter.new(self, LiveIter.new(make_merge_iter(prefix, false))) + ScanIter.new(self, inner, finish, prefix) + end + end + + # Returns a streaming iterator over live key/value pairs in key order. + # + # The iterator shares this snapshot and reads the range [start, finish). + # Closing it never releases the snapshot. Release the snapshot when done. + def scan_iter(start_key : String = "", finish_key : String? = nil) : ScanIter + @lock.synchronize do + check_released + ScanIter.new(self, SnapshotIter.new(self, LiveIter.new(make_merge_iter(start_key, false))), finish_key) + end + end + # Yields live key/value pairs in key order. The range is [start, finish). def each(start_key : String = "", finish_key : String? = nil, &block : String, String ->) : Nil scan(start_key, finish_key).each { |key, value| yield key, value } end + # Returns the primary keys whose value maps to `index_key` in `index`. + # + # The result reflects the state the snapshot holds. A negative `limit` + # means no limit. The keys come back in primary key order. + def query(index : String, index_key : String, limit : Int32 = -1) : Array(String) + validate_index_name(index) + @lock.synchronize do + check_released + prefix = Index.exact_prefix(index, index_key) + keys = [] of String + iter = make_merge_iter(prefix, false) + while entry = iter.next? + break unless entry.key.starts_with?(prefix) + if entry.alive + keys << Index.primary_key_of(entry.key) + break if limit >= 0 && keys.size >= limit + end + end + keys + end + end + + # Returns the primary keys whose index key lies in [start, finish). + # + # The keys come back in index key order, then primary key order. A nil + # `finish_key` means no upper bound. A negative `limit` means no limit. + def query_range(index : String, start_key : String = "", finish_key : String? = nil, + limit : Int32 = -1) : Array(String) + validate_index_name(index) + @lock.synchronize do + check_released + prefix = Index.name_prefix(index) + start = "#{prefix}#{start_key}#{Index::SEP}" + keys = [] of String + iter = make_merge_iter(start, false) + while entry = iter.next? + break unless entry.key.starts_with?(prefix) + index_key = Index.index_key_of(entry.key) + break if finish_key && index_key >= finish_key + if entry.alive + keys << Index.primary_key_of(entry.key) + break if limit >= 0 && keys.size >= limit + end + end + keys + end + end + # Returns a live iterator over the snapshot, starting at `start_key`. # # The iterator stays consistent while the database keeps writing. Close @@ -77,6 +163,40 @@ module Cinderstore end end + # Returns a streaming iterator over the primary keys that map to + # `index_key` in `index`. + # + # The result reflects the state the snapshot holds. The iterator shares + # the snapshot, so it stays consistent while the database keeps writing. + # Keys come back in primary key order. Close the iterator before you + # release the snapshot. Closing the iterator never releases the snapshot. + def query_iter(index : String, index_key : String) : IndexIter + validate_index_name(index) + @lock.synchronize do + check_released + prefix = Index.exact_prefix(index, index_key) + IndexIter.new(self, SnapshotIter.new(self, make_merge_iter(prefix, false)), prefix, nil) + end + end + + # Returns a streaming iterator over the primary keys whose index key + # lies in [start, finish). + # + # The keys come back in index key order, then primary key order. A nil + # `finish_key` means no upper bound. The iterator shares the snapshot, + # so it stays consistent while the database keeps writing. Close the + # iterator before you release the snapshot. Closing the iterator never + # releases the snapshot. + def query_range_iter(index : String, start_key : String = "", finish_key : String? = nil) : IndexIter + validate_index_name(index) + @lock.synchronize do + check_released + prefix = Index.name_prefix(index) + start = "#{prefix}#{start_key}#{Index::SEP}" + IndexIter.new(self, SnapshotIter.new(self, make_merge_iter(start, false)), prefix, finish_key) + end + end + # Returns the number of live entries the snapshot sees. The count never # changes after creation. def count : Int64 @@ -137,6 +257,12 @@ module Cinderstore private def validate_key(key : String) : Nil raise InvalidKeyError.new("key must not be empty") if key.empty? raise InvalidKeyError.new("key exceeds #{DB::MAX_KEY_BYTES} bytes") if key.bytesize > DB::MAX_KEY_BYTES + raise InvalidKeyError.new("key starts with the reserved index prefix") if Index.internal_key?(key) + end + + private def validate_index_name(name : String) : Nil + raise InvalidIndexError.new("index name must not be empty") if name.empty? + raise InvalidIndexError.new("index name contains a NUL byte") if name.includes?(Index::SEP) end private def check_released : Nil diff --git a/src/cinderstore/table.cr b/src/cinderstore/table.cr index bcd8bd3..416e034 100644 --- a/src/cinderstore/table.cr +++ b/src/cinderstore/table.cr @@ -5,8 +5,10 @@ module Cinderstore getter first : String getter last : String getter count : Int64 + getter index_count : Int64 - def initialize(@id : Int64, @first : String, @last : String, @count : Int64) + def initialize(@id : Int64, @first : String, @last : String, @count : Int64, + @index_count : Int64 = 0_i64) end end @@ -30,28 +32,39 @@ module Cinderstore # [footer] # # A data block holds the block length, a batch of serialized entries, - # and a CRC32. The block index maps the first key of each block to its - # file offset and total length. The bloom filter lets a reader skip a - # table that cannot contain a key. The footer stores offsets and a CRC32 - # over the entire footer. + # and an optional CRC32. The block index maps the first key of each + # block to its file offset and total length. The bloom filter lets a + # reader skip a table that cannot contain a key. + # + # The version-2 footer stores offsets, a version, a flag byte, and a + # CRC32. The flag byte says whether the data blocks carry checksums. + # Version-1 footers have no flag byte. Their blocks always carry + # checksums. The reader reads the version and the flag from the file, + # so one process can open tables written in either mode. class SstableWriter - MAGIC = 0x43494E4445525F31_u64 - VERSION = 1_u32 - FOOTER_SIZE = 48 + MAGIC = 0x43494E4445525F31_u64 + VERSION = 2_u32 + LEGACY_VERSION = 1_u32 + FOOTER_SIZE = 52 + LEGACY_FOOTER_SIZE = 48 + FLAG_CHECKSUM = 1_u8 @io : IO @id : Int64 @block_size : Int32 @fpp : Float64 + @checksums : Bool @pending : IO::Memory @pending_keys : Array(String) @keys : Array(String) @index : Array(BlockIndexEntry) @count : Int64 = 0_i64 + @index_count : Int64 = 0_i64 @first : String? @last : String? - def initialize(io : IO, @id : Int64, @block_size : Int32 = 4096, @fpp : Float64 = 0.01) + def initialize(io : IO, @id : Int64, @block_size : Int32 = 4096, + @fpp : Float64 = 0.01, @checksums : Bool = true) @io = io @pending = IO::Memory.new @pending_keys = [] of String @@ -71,6 +84,7 @@ module Cinderstore @first = key if @first.nil? @last = key @count += 1 + @index_count += 1 if Index.internal_key?(key) flush_block if @pending.size >= @block_size end @@ -101,12 +115,14 @@ module Cinderstore footer.write_bytes(bloom_offset.to_u64, IO::ByteFormat::LittleEndian) footer.write_bytes(bloom_length.to_u64, IO::ByteFormat::LittleEndian) footer.write_bytes(VERSION, IO::ByteFormat::LittleEndian) + footer.write_byte(@checksums ? FLAG_CHECKSUM : 0_u8) + 3.times { footer.write_byte(0_u8) } footer_bytes = footer.to_slice @io.write(footer_bytes) @io.write_bytes(Util.crc32(footer_bytes), IO::ByteFormat::LittleEndian) @io.flush - TableMeta.new(@id, @first.not_nil!, @last.not_nil!, @count) + TableMeta.new(@id, @first.not_nil!, @last.not_nil!, @count, @index_count) end private def flush_block : Nil @@ -114,8 +130,10 @@ module Cinderstore offset = @io.pos Util.write_varint(@io, data.size.to_u64) @io.write(data) - @io.write_bytes(Util.crc32(data), IO::ByteFormat::LittleEndian) - total = Util.varint_len(data.size.to_u64) + data.size + 4 + if @checksums + @io.write_bytes(Util.crc32(data), IO::ByteFormat::LittleEndian) + end + total = Util.varint_len(data.size.to_u64) + data.size + (@checksums ? 4 : 0) @index << BlockIndexEntry.new(@pending_keys.first, offset.to_u64, total.to_u64) @pending = IO::Memory.new @pending_keys = [] of String @@ -136,6 +154,7 @@ module Cinderstore @index_length : UInt64 @bloom_offset : UInt64 @bloom_length : UInt64 + @checksums : Bool def initialize(@path : String, @file_id : Int64, @cache : BlockCache? = nil) @file = File.open(@path, "r") @@ -147,6 +166,7 @@ module Cinderstore @index_length = 0_u64 @bloom_offset = 0_u64 @bloom_length = 0_u64 + @checksums = true read_footer read_index read_bloom @@ -179,6 +199,40 @@ module Cinderstore Cinderstore.decode_entries(read_block(idx)) end + # Verifies every block and returns the metadata found in the file. + def verify : TableMeta + raise CorruptDataError.new if @index.empty? + previous_key : String? = nil + first_key : String? = nil + last_key : String? = nil + count = 0_i64 + index_count = 0_i64 + + @index.each_with_index do |block, index| + raise CorruptDataError.new if block.length == 0 + raise CorruptDataError.new if block.offset >= @index_offset + raise CorruptDataError.new if block.length > @index_offset - block.offset + raise CorruptDataError.new if index > 0 && block.first_key <= @index[index - 1].first_key + # Bypass the block cache. Verification must inspect the bytes on + # disk, even when a normal read already cached this block. + entries = Cinderstore.decode_entries(read_block_raw(block)) + raise CorruptDataError.new if entries.empty? + raise CorruptDataError.new unless entries.first.key == block.first_key + + entries.each do |entry| + raise CorruptDataError.new if previous_key && entry.key <= previous_key.not_nil! + raise CorruptDataError.new unless bloom_may_contain?(entry.key) + previous_key = entry.key + first_key ||= entry.key + last_key = entry.key + count += 1 + index_count += 1 if Index.internal_key?(entry.key) + end + end + + TableMeta.new(@file_id, first_key.not_nil!, last_key.not_nil!, count, index_count) + end + # Returns the index of the block that may contain `key`. # # Returns the last block whose first key is <= `key`, or block zero. @@ -215,27 +269,57 @@ module Cinderstore private def read_block_raw(entry : BlockIndexEntry) : Bytes @file.pos = entry.offset data_len = Util.read_varint(@file).to_i - if data_len.to_u64 + Util.varint_len(data_len.to_u64) + 4 != entry.length + crc_len = @checksums ? 4 : 0 + if data_len.to_u64 + Util.varint_len(data_len.to_u64) + crc_len != entry.length raise CorruptDataError.new("block length mismatch in #{@path}") end data = Bytes.new(data_len) @file.read_fully(data) - stored_crc = @file.read_bytes(UInt32, IO::ByteFormat::LittleEndian) - actual_crc = Util.crc32(data) - raise CorruptDataError.new("block checksum mismatch in #{@path}") unless stored_crc == actual_crc + if @checksums + stored_crc = @file.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + actual_crc = Util.crc32(data) + raise CorruptDataError.new("block checksum mismatch in #{@path}") unless stored_crc == actual_crc + end data end private def read_footer : Nil - raise CorruptDataError.new("file too small: #{@path}") if @size < SstableWriter::FOOTER_SIZE - @file.pos = @size - SstableWriter::FOOTER_SIZE - footer = Bytes.new(SstableWriter::FOOTER_SIZE) + raise CorruptDataError.new("file too small: #{@path}") if @size < SstableWriter::LEGACY_FOOTER_SIZE + if @size >= SstableWriter::FOOTER_SIZE + @file.pos = @size - SstableWriter::FOOTER_SIZE + footer = Bytes.new(SstableWriter::FOOTER_SIZE) + @file.read_fully(footer) + return if read_footer_v2(footer) + end + # The footer is version 1, so its layout is smaller. + @file.pos = @size - SstableWriter::LEGACY_FOOTER_SIZE + footer = Bytes.new(SstableWriter::LEGACY_FOOTER_SIZE) @file.read_fully(footer) + read_footer_v1(footer) + end - crc_bytes = footer[SstableWriter::FOOTER_SIZE - 4, 4] - stored_crc = IO::Memory.new(crc_bytes).read_bytes(UInt32, IO::ByteFormat::LittleEndian) - actual_crc = Util.crc32(footer[0, SstableWriter::FOOTER_SIZE - 4]) - raise CorruptDataError.new("footer checksum mismatch in #{@path}") unless stored_crc == actual_crc + # Reads a version-2 footer. Returns false when the magic does not match, + # which means the file predates the version-2 layout. + private def read_footer_v2(footer : Bytes) : Bool + io = IO::Memory.new(footer) + magic = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) + return false unless magic == SstableWriter::MAGIC + + verify_footer_crc(footer, SstableWriter::FOOTER_SIZE) + + @index_offset = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) + @index_length = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) + @bloom_offset = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) + @bloom_length = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) + version = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + raise CorruptDataError.new("unsupported version #{version} in #{@path}") unless version == SstableWriter::VERSION + flags = io.read_byte.not_nil! + @checksums = (flags & SstableWriter::FLAG_CHECKSUM) != 0 + true + end + + private def read_footer_v1(footer : Bytes) : Nil + verify_footer_crc(footer, SstableWriter::LEGACY_FOOTER_SIZE) io = IO::Memory.new(footer) magic = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) @@ -245,7 +329,16 @@ module Cinderstore @bloom_offset = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) @bloom_length = io.read_bytes(UInt64, IO::ByteFormat::LittleEndian) version = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) - raise CorruptDataError.new("unsupported version #{version} in #{@path}") unless version == SstableWriter::VERSION + raise CorruptDataError.new("unsupported version #{version} in #{@path}") unless version == SstableWriter::LEGACY_VERSION + @checksums = true + end + + # Checks the CRC32 that guards the fixed part of a footer. + private def verify_footer_crc(footer : Bytes, size : Int32) : Nil + crc_bytes = footer[size - 4, 4] + stored_crc = IO::Memory.new(crc_bytes).read_bytes(UInt32, IO::ByteFormat::LittleEndian) + actual_crc = Util.crc32(footer[0, size - 4]) + raise CorruptDataError.new("footer checksum mismatch in #{@path}") unless stored_crc == actual_crc end private def read_index : Nil diff --git a/src/cinderstore/util.cr b/src/cinderstore/util.cr index 7171f04..882a748 100644 --- a/src/cinderstore/util.cr +++ b/src/cinderstore/util.cr @@ -65,5 +65,21 @@ module Cinderstore def self.file_stem(id : Int64) : String id.to_s.rjust(6, '0') end + + # Returns the smallest byte string greater than every string with + # `prefix` as its prefix. Returns nil when no such bound exists. + def self.prefix_end(prefix : String) : String? + bytes = prefix.to_slice + index = bytes.size - 1 + while index >= 0 && bytes[index] == UInt8::MAX + index -= 1 + end + return nil if index < 0 + + String.build(index + 1) do |io| + io.write(bytes[0, index]) + io.write_byte((bytes[index].to_i + 1).to_u8) + end + end end end diff --git a/src/cinderstore/version.cr b/src/cinderstore/version.cr index 3b31a8c..613f7dc 100644 --- a/src/cinderstore/version.cr +++ b/src/cinderstore/version.cr @@ -1,3 +1,3 @@ module Cinderstore - VERSION = "0.4.0" + VERSION = "0.12.0" end diff --git a/src/cinderstore/wal.cr b/src/cinderstore/wal.cr index 04735b6..095b646 100644 --- a/src/cinderstore/wal.cr +++ b/src/cinderstore/wal.cr @@ -1,19 +1,89 @@ module Cinderstore # The write ahead log (WAL). # - # The WAL frames each entry with a length, a payload, and a CRC32. We - # fsync after every append when durability is enabled. A torn write at - # the tail is truncated during recovery. + # A version-2 log starts with a small header: a magic value and one flag + # byte. The flag byte says whether the records carry CRC32 checksums and + # whether each record starts with a kind byte. + # + # A version-3 record starts with a kind byte. Kind zero is a single + # entry. Kind one is a batch of entries. Both share one CRC32, so a torn + # batch is dropped whole during recovery. Version-1 and version-2 logs + # have no kind byte. Their records always hold one entry and always carry + # checksums. Recovery reads the header to learn how to parse the file. + # + # With checksums enabled the database fsyncs once per commit group. A + # torn write at the tail is truncated during recovery. class Wal - # Serializes an entry into a framed record. - def self.encode(entry : Entry) : Bytes + # First bytes of a version-2 log. + HEADER_MAGIC = 0x31535743_u32 + # Total header bytes: magic value plus flag byte. + HEADER_SIZE = 5_i64 + # Flag that says the records carry CRC32 checksums. + FLAG_CHECKSUMS = 1_u8 + # Flag that says each record starts with a kind byte. + FLAG_KIND = 2_u8 + + # Record kind for a single entry. + KIND_ENTRY = 0_u8 + # Record kind for a batch of entries. + KIND_BATCH = 1_u8 + + # Describes how the records in one log are framed. + private record Format, checksums : Bool, kind : Bool + + # Serializes an entry into a version-1 framed record. + # + # A version-1 record holds a varint length, the entry body, and a CRC32 + # over the body. This method exists so tests can build legacy files. + def self.encode(entry : Entry, checksums : Bool = true) : Bytes body = IO::Memory.new Cinderstore.write_entry(body, entry) payload = body.to_slice framed = IO::Memory.new Util.write_varint(framed, payload.size.to_u64) framed.write(payload) - framed.write_bytes(Util.crc32(payload), IO::ByteFormat::LittleEndian) + framed.write_bytes(Util.crc32(payload), IO::ByteFormat::LittleEndian) if checksums + framed.to_slice + end + + # Serializes a single entry into a version-3 record. + def self.encode_entry(entry : Entry, checksums : Bool = true) : Bytes + body = IO::Memory.new + Cinderstore.write_entry(body, entry) + encode_payload(KIND_ENTRY, body.to_slice, checksums) + end + + # Serializes a batch of entries into one version-3 record. + # + # The body holds a varint count followed by that many serialized + # entries. The whole record carries one CRC32, so recovery either + # applies every entry or drops the record. + def self.encode_batch(entries : Array(Entry), checksums : Bool = true) : Bytes + body = IO::Memory.new + Util.write_varint(body, entries.size.to_u64) + entries.each { |entry| Cinderstore.write_entry(body, entry) } + encode_payload(KIND_BATCH, body.to_slice, checksums) + end + + # Writes the version-2 header to `io`. + def self.write_header(io : IO, checksums : Bool, kind : Bool = true) : Nil + io.write_bytes(HEADER_MAGIC, IO::ByteFormat::LittleEndian) + flags = (checksums ? FLAG_CHECKSUMS : 0_u8) | (kind ? FLAG_KIND : 0_u8) + io.write_byte(flags) + end + + # Frames a kind byte, a payload, and an optional CRC32 over both. + private def self.encode_payload(kind : UInt8, payload : Bytes, checksums : Bool) : Bytes + framed = IO::Memory.new + Util.write_varint(framed, (payload.size + 1).to_u64) + framed.write_byte(kind) + framed.write(payload) + if checksums + crc_source = IO::Memory.new + crc_source.write_byte(kind) + crc_source.write(payload) + framed.write_bytes(Util.crc32(crc_source.to_slice), IO::ByteFormat::LittleEndian) + end framed.to_slice end @@ -27,22 +97,27 @@ module Cinderstore return max_seq unless File.exists?(path) File.open(path, "r") do |io| + format = read_format(io) loop do break if io.pos >= io.size begin body_len = Util.read_varint(io).to_i - body = Bytes.new(body_len) - io.read_fully(body) - stored_crc = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) - actual_crc = Util.crc32(body) - raise CorruptDataError.new("wal checksum mismatch") unless stored_crc == actual_crc - entry = Cinderstore.read_entry(IO::Memory.new(body)) - if entry.alive - mem.put(entry.key, entry.value, entry.seq) + if format.kind + kind = io.read_byte.not_nil! + body = Bytes.new(body_len - 1) + io.read_fully(body) + verify_checksum(io, kind, body) if format.checksums + max_seq = apply_kind(kind, body, mem, max_seq) else - mem.delete(entry.key, entry.seq) + body = Bytes.new(body_len) + io.read_fully(body) + if format.checksums + stored_crc = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + actual_crc = Util.crc32(body) + raise CorruptDataError.new("wal checksum mismatch") unless stored_crc == actual_crc + end + max_seq = apply_entry(Cinderstore.read_entry(IO::Memory.new(body)), mem, max_seq) end - max_seq = entry.seq if entry.seq > max_seq rescue ex : IO::EOFError break rescue ex : Error @@ -53,25 +128,166 @@ module Cinderstore max_seq end + # Verifies every complete record in `path`. + def self.verify(path : String) : Int64 + return 0_i64 unless File.exists?(path) + File.open(path, "r") do |io| + format = read_format(io) + records = 0_i64 + while io.pos < io.size + body_len = Util.read_varint(io).to_i + if format.kind + raise CorruptDataError.new("empty wal record") if body_len < 1 + kind = io.read_byte.not_nil! + body = Bytes.new(body_len - 1) + io.read_fully(body) + verify_checksum(io, kind, body) if format.checksums + verify_kind(kind, body) + else + body = Bytes.new(body_len) + io.read_fully(body) + verify_checksum(io, body) if format.checksums + verify_entry_body(body) + end + records += 1 + end + records + end + end + + private def self.verify_kind(kind : UInt8, body : Bytes) : Nil + case kind + when KIND_ENTRY + verify_entry_body(body) + when KIND_BATCH + io = IO::Memory.new(body) + count = Util.read_varint(io).to_i + raise CorruptDataError.new if count == 0 + count.times { Cinderstore.read_entry(io) } + raise CorruptDataError.new unless io.pos == io.size + else + raise CorruptDataError.new + end + end + + private def self.verify_entry_body(body : Bytes) : Nil + io = IO::Memory.new(body) + Cinderstore.read_entry(io) + raise CorruptDataError.new unless io.pos == io.size + end + + private def self.verify_checksum(io : IO, body : Bytes) : Nil + stored_crc = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + actual_crc = Util.crc32(body) + raise CorruptDataError.new("wal checksum mismatch") unless stored_crc == actual_crc + end + + # Detects the record framing of a log and leaves `io` at the first + # record. A version-2 header sets the mode from its flag byte. A legacy + # version-1 log has no header, so its records are entries with + # checksums and no kind byte. + private def self.read_format(io : IO) : Format + return Format.new(true, false) if io.size < HEADER_SIZE + magic = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + if magic == HEADER_MAGIC + if flags = io.read_byte + return Format.new((flags & FLAG_CHECKSUMS) != 0, (flags & FLAG_KIND) != 0) + end + # The header is torn after the magic. Nothing is recoverable. + io.pos = io.size + return Format.new(true, false) + end + io.pos = 0 + Format.new(true, false) + end + + # Verifies the CRC32 over a kind byte and its body. + private def self.verify_checksum(io : IO, kind : UInt8, body : Bytes) : Nil + crc_source = IO::Memory.new + crc_source.write_byte(kind) + crc_source.write(body) + stored_crc = io.read_bytes(UInt32, IO::ByteFormat::LittleEndian) + actual_crc = Util.crc32(crc_source.to_slice) + raise CorruptDataError.new("wal checksum mismatch") unless stored_crc == actual_crc + end + + # Applies one entry to `mem` and returns the higher sequence number. + private def self.apply_entry(entry : Entry, mem : MemTable, max_seq : Int64) : Int64 + if entry.alive + mem.put(entry.key, entry.value, entry.seq) + else + mem.delete(entry.key, entry.seq) + end + entry.seq > max_seq ? entry.seq : max_seq + end + + # Applies a kind-byte record to `mem` and returns the higher sequence + # number. A batch applies every entry it holds. + private def self.apply_kind(kind : UInt8, body : Bytes, mem : MemTable, max_seq : Int64) : Int64 + case kind + when KIND_ENTRY + apply_entry(Cinderstore.read_entry(IO::Memory.new(body)), mem, max_seq) + when KIND_BATCH + io = IO::Memory.new(body) + count = Util.read_varint(io).to_i + seq = max_seq + count.times do + seq = apply_entry(Cinderstore.read_entry(io), mem, seq) + end + seq + else + raise CorruptDataError.new("unknown wal record kind") + end + end + # Appends framed records to a WAL file. class Writer getter path : String getter size : Int64 - def initialize(@path : String, @sync_each_write : Bool = true) + # Opens the log and writes a version-2 header on a fresh file. + def initialize(@path : String, @sync_each_write : Bool = true, @checksums : Bool = true, + @kind : Bool = true) @file = File.open(@path, "a+") + @closed = false @size = @file.size + if @size == 0 + Wal.write_header(@file, @checksums, @kind) + @file.flush + @file.fsync if @sync_each_write + @size = HEADER_SIZE + end end + # Appends one entry to the file and flushes the userspace buffer. + # The database decides when the buffer becomes durable. def append(entry : Entry) : Nil - record = Wal.encode(entry) + record = Wal.encode_entry(entry, @checksums) + @file.write(record) + @file.flush + @size += record.size + end + + # Appends a batch of entries as one record. + def append_batch(entries : Array(Entry)) : Nil + record = Wal.encode_batch(entries, @checksums) @file.write(record) @file.flush - @file.fsync if @sync_each_write @size += record.size end + # Forces the buffered records to the disk. Called once per commit + # group, and on close. + def sync : Nil + return if @closed + @file.flush + @file.fsync + end + def close : Nil + return if @closed + @closed = true + @file.flush @file.fsync @file.close end diff --git a/src/cli.cr b/src/cli.cr index 8a91070..8daa0a5 100644 --- a/src/cli.cr +++ b/src/cli.cr @@ -21,9 +21,11 @@ module Cinderstore when "get" then run_read(args[1..]) when "del", "delete" then run_delete(args[1..]) when "scan" then run_scan(args[1..]) + when "query" then run_query(args[1..]) when "stats" then run_simple("stats", args[1..]) when "flush" then run_simple("flush", args[1..]) when "compact" then run_simple("compact", args[1..]) + when "verify" then run_verify(args[1..]) when "demo" then run_demo(args[1..]) when "help", "--help", "-h" print_help @@ -39,24 +41,35 @@ module Cinderstore db_path = "cinderstore-data" host = "127.0.0.1" port = 7654 + config = DB::Config.new + index_specs = [] of Tuple(String, String) help = false parser = OptionParser.new parser.on("--db PATH", "Database directory") { |v| db_path = v } parser.on("--host HOST", "Bind address") { |v| host = v } parser.on("--port PORT", "Listen on this port") { |v| port = v.to_i } + parser.on("--no-checksums", "Skip checksums on new writes") { config.checksums = false } + parser.on("--index SPEC", "JSON field index as name:field") { |v| index_specs << parse_index_spec(v) } parser.on("-h", "--help", "Show this help") { help = true } parser.parse(rest) if help puts parser return 0 end - db = DB.new(db_path) + db = DB.new(db_path, config) + index_specs.each { |name, field| db.create_json_index(name, field) } server = Server.new(db, host, port) server.run db.close 0 end + private def parse_index_spec(spec : String) : Tuple(String, String) + name, field = spec.split(":", 2) + raise Error.new("--index must be name:field") if field.nil? || name.empty? || field.empty? + {name, field} + end + private def run_write(rest : Array(String)) : Int32 db_path = "cinderstore-data" key = "" @@ -139,6 +152,7 @@ module Cinderstore db_path = "cinderstore-data" start_key = "" finish_key = "" + prefix : String? = nil limit = -1 help = false parser = OptionParser.new @@ -147,17 +161,74 @@ module Cinderstore parser.on("--finish KEY", "Last key (exclusive)") { |v| finish_key = v } parser.on("--limit N", "Maximum rows") { |v| limit = v.to_i } parser.on("-h", "--help", "Show this help") { help = true } + parser.on("--prefix PREFIX", "Keys that start with prefix") { |v| prefix = v } parser.parse(rest) if help puts parser return 0 end + if prefix && (!start_key.empty? || !finish_key.empty?) + raise Error.new("--prefix cannot be combined with --start or --finish") + end db = DB.new(db_path) + iter : ScanIter? = nil begin - rows = db.scan(start_key, finish_key.empty? ? nil : finish_key, limit) - rows.each do |key, value| - puts "#{key}\t#{value}" + iter = if prefix + db.scan_prefix_iter(prefix.not_nil!) + else + db.scan_iter(start_key, finish_key.empty? ? nil : finish_key) + end + stream = iter.not_nil! + + count = 0 + while pair = stream.next? + puts "#{pair[0]}\t#{pair[1]}" + count += 1 + break if limit >= 0 && count >= limit end + ensure + iter.try { |value| value.close } + db.close + end + 0 + end + + private def run_query(rest : Array(String)) : Int32 + db_path = "cinderstore-data" + index = "" + key = "" + start_key = "" + finish_key = "" + limit = -1 + help = false + parser = OptionParser.new + parser.on("--db PATH", "Database directory") { |v| db_path = v } + parser.on("--index NAME", "Index to query") { |v| index = v } + parser.on("--key KEY", "Exact index key") { |v| key = v } + parser.on("--start KEY", "First index key (inclusive)") { |v| start_key = v } + parser.on("--finish KEY", "Last index key (exclusive)") { |v| finish_key = v } + parser.on("--limit N", "Maximum rows") { |v| limit = v.to_i } + parser.on("-h", "--help", "Show this help") { help = true } + parser.parse(rest) + if help + puts parser + return 0 + end + raise Error.new("missing --index") if index.empty? + db = DB.new(db_path) + begin + iter = if key.empty? + db.query_range_iter(index, start_key, finish_key.empty? ? nil : finish_key) + else + db.query_iter(index, key) + end + count = 0 + while pk = iter.next? + puts pk + count += 1 + break if limit >= 0 && count >= limit + end + iter.close ensure db.close end @@ -193,20 +264,38 @@ module Cinderstore 0 end + private def run_verify(rest : Array(String)) : Int32 + db_path = "cinderstore-data" + help = false + parser = OptionParser.new + parser.on("--db PATH", "Database directory") { |v| db_path = v } + parser.on("-h", "--help", "Show this help") { help = true } + parser.parse(rest) + if help + puts parser + return 0 + end + report = DB.verify(db_path) + puts report + report.valid? ? 0 : 1 + end + private def run_demo(rest : Array(String)) : Int32 db_path = nil fixture = nil + checksums = true help = false parser = OptionParser.new parser.on("--db PATH", "Database directory") { |v| db_path = v } parser.on("--fixture PATH", "CSV fixture file") { |v| fixture = v } + parser.on("--no-checksums", "Skip checksums on new writes") { checksums = false } parser.on("-h", "--help", "Show this help") { help = true } parser.parse(rest) if help puts parser return 0 end - Demo.run(db_path, fixture) + Demo.run(db_path, fixture, checksums) end private def print_help : Nil @@ -221,9 +310,11 @@ module Cinderstore get Read a value by key del Delete a key scan List keys in a range + query Find primary keys through a secondary index stats Show database counters flush Flush the memtable to a table compact Merge tables and drop stale data + verify Check table and log integrity demo Run a self-contained walkthrough help Show this help