diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 1f4522c4..00000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,52 +0,0 @@ -# Summary - -Describe the change in one or two sentences. - -## Why are you making this contribution? - -Explain how you encountered the problem or need, who or what it affects, and why -this repository and scope are appropriate. - -## Related Issue - -Link the accepted issue for substantial work. For a small self-contained change, -explain why a prior issue was unnecessary. - -## What Changed - -- - -## Validation - -- - -## Domain Boundary Review - -Complete this section whenever the change touches domain semantics, persistence, recovery, or cross-domain composition. - -- Does the change preserve the domain boundary it touches? -- Does it avoid implying durability, replay, ownership continuity, or recovery that the implementation does not provide? -- If semantics changed, were the relevant docs updated in the same change? - -## Notes - -- - -## Tool Assistance Disclosure - -Select exactly one: - -- [ ] No AI or other generative tool materially assisted this contribution. -- [ ] AI or another generative tool materially assisted this contribution. - -If assisted, identify the kind of tool used, what it assisted, and how you -reviewed and validated the resulting work. Do not include private prompts, -credentials, or confidential information. - -## Contributor Responsibility - -- [ ] I understand the complete change and can explain or revise it. -- [ ] I reviewed the complete diff. -- [ ] I reported validation accurately and did not claim checks I did not run. -- [ ] I disclosed material generated assistance. -- [ ] I have the right to submit this work under the repository's license. diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index 908dd976..c50647fa 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -20,6 +20,14 @@ jobs: backend: runs-on: ubuntu-latest timeout-minutes: 20 + env: + RUST_LOG: "off" + # This workspace links ~24 separate integration-test binaries against a + # heavy dependency tree (tokio, hyper, rustls, opentelemetry, + # aws-lc-sys, ...) -- lld is meaningfully faster than the default bfd + # linker for that many large links, with no behavior change. Scoped to + # CI only; local dev toolchains are untouched. + RUSTFLAGS: "-C link-arg=-fuse-ld=lld" steps: - name: Checkout repository uses: actions/checkout@v7 @@ -27,6 +35,9 @@ jobs: - name: Update Rust toolchain run: rustup update stable + - name: Install lld linker + run: sudo apt-get update && sudo apt-get install -y lld + - name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: @@ -47,18 +58,17 @@ jobs: cntryl-tools validate-benchmarks --config .cntryl/repository.toml cntryl-tools check-module-sizes --config .cntryl/repository.toml - - name: Cargo build - run: cargo build --locked + - name: Cargo check + run: cargo check --locked - name: Cargo clippy pedantic run: cargo clippy --locked --workspace --all-targets --all-features -- -D warnings -D clippy::pedantic - - name: Cargo test + - name: Build tests + run: cargo test --no-run --locked --workspace + + - name: Run tests env: - # Shared CI runners have been observed running the e2e suites roughly - # two orders of magnitude slower than a developer machine, tripping - # the fixed per-frame deadlines the tests pass. Scale them here rather - # than loosening the deadlines for everyone. FITZ_TEST_TIMEOUT_MULTIPLIER: "10" run: cargo test --locked --workspace diff --git a/.github/workflows/dependency-drift.yml b/.github/workflows/dependency-drift.yml deleted file mode 100644 index ec2bef85..00000000 --- a/.github/workflows/dependency-drift.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Dependency drift - -on: - schedule: - - cron: "0 9 * * 1" - workflow_dispatch: - -permissions: - contents: read - issues: write - -jobs: - git-pins: - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Report stale git pins - uses: actions/github-script@v8 - with: - script: | - const fs = require("node:fs"); - const cargo = fs.readFileSync("Cargo.toml", "utf8"); - const dependencies = [ - { name: "cntryl-lexkey", owner: "cntryl", repo: "lexkey-rs" }, - { name: "cntryl-midge", owner: "cntryl", repo: "midge" }, - { name: "cntryl-stress", owner: "cntryl", repo: "stress" }, - ]; - const stale = []; - - for (const dependency of dependencies) { - const escaped = dependency.name.replaceAll("-", "\\-"); - const declaration = cargo.match( - new RegExp(`^${escaped}\\s*=.*$`, "m"), - ); - if (!declaration?.[0].includes('branch = "main"')) { - core.setFailed(`${dependency.name} must track branch = "main"`); - return; - } - - const lock = fs.readFileSync("Cargo.lock", "utf8"); - const packageBlock = lock.match( - new RegExp(`\\[\\[package\\]\\]\\nname = "${escaped}"[\\s\\S]*?(?=\\n\\[\\[package\\]\\]|$)`), - ); - const source = packageBlock?.[0].match(/source = ".*\\?branch=main#([0-9a-f]+)"/); - if (!source) { - core.setFailed(`Could not find a main-branch lock for ${dependency.name}`); - return; - } - - const pinned = source[1]; - const repository = await github.rest.repos.get({ - owner: dependency.owner, - repo: dependency.repo, - }); - const branch = repository.data.default_branch; - const comparison = await github.rest.repos.compareCommitsWithBasehead({ - owner: dependency.owner, - repo: dependency.repo, - basehead: `${pinned}...${branch}`, - }); - if (comparison.data.ahead_by > 0) { - stale.push( - `- ${dependency.name}: ${pinned} is ${comparison.data.ahead_by} commit(s) behind ${branch}`, - ); - } - } - - const title = "[Deps] Git dependency pins have drifted"; - const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`; - const existing = await github.rest.search.issuesAndPullRequests({ q: query }); - if (stale.length === 0) { - core.summary.addHeading("Git dependency pins are current").write(); - return; - } - - const body = [ - "The weekly dependency check found stale Git branch locks:", - "", - ...stale, - "", - `Detected by workflow run ${context.runId}. Evaluate each update on its own merits.`, - ].join("\n"); - core.summary.addHeading("Stale git dependency pins").addRaw(body).write(); - - if (existing.data.total_count === 0) { - await github.rest.issues.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title, - body, - labels: ["dependencies"], - }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: existing.data.items[0].number, - body, - }); - } diff --git a/Cargo.lock b/Cargo.lock index f56e3a74..d7e61f45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,7 +89,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -238,9 +238,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.2" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" dependencies = [ "find-msvc-tools", "jobserver", @@ -313,7 +313,7 @@ dependencies = [ [[package]] name = "cntryl-midge" version = "0.1.0" -source = "git+https://github.com/cntryl/midge?branch=main#49442e6bcda490641581052fdd8db97e3729d7b5" +source = "git+https://github.com/cntryl/midge?branch=main#e04ecb4f108ae1dcbfd0421e1c919144bc957afd" dependencies = [ "arc-swap", "base64 0.23.1", @@ -357,7 +357,7 @@ dependencies = [ [[package]] name = "cntryl-stress" version = "0.3.0" -source = "git+https://github.com/cntryl/stress?branch=main#590d96dd7c37a222550758ba0d0ac542d39ce9fd" +source = "git+https://github.com/cntryl/stress?branch=main#7078e18fb05f2f96f5c6c18caa26c112e56fb025" dependencies = [ "cntryl-stress-macros", "fs2", @@ -369,7 +369,7 @@ dependencies = [ [[package]] name = "cntryl-stress-macros" version = "0.3.0" -source = "git+https://github.com/cntryl/stress?branch=main#590d96dd7c37a222550758ba0d0ac542d39ce9fd" +source = "git+https://github.com/cntryl/stress?branch=main#7078e18fb05f2f96f5c6c18caa26c112e56fb025" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -379,9 +379,9 @@ dependencies = [ [[package]] name = "combine" -version = "4.6.7" +version = "4.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" dependencies = [ "bytes", "memchr", @@ -438,9 +438,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" dependencies = [ "cfg-if", ] @@ -596,7 +596,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -607,9 +607,9 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "either" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "equivalent" @@ -635,9 +635,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "fitz" @@ -792,7 +792,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -875,9 +875,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -1118,9 +1118,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -1132,9 +1132,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -1145,9 +1145,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1159,16 +1159,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -1179,15 +1180,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -1367,7 +1368,7 @@ checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -1378,9 +1379,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -1393,9 +1394,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru-slab" @@ -1493,9 +1494,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -1743,9 +1744,9 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "portable-atomic" @@ -1755,9 +1756,9 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -1874,9 +1875,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ "aws-lc-rs", "bytes", @@ -2175,9 +2176,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -2283,7 +2284,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -2321,7 +2322,7 @@ checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -2461,9 +2462,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" dependencies = [ "proc-macro2", "quote", @@ -2535,7 +2536,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -2579,9 +2580,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -2627,7 +2628,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.4", ] [[package]] @@ -2978,9 +2979,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.24.1" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" +checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -3367,9 +3368,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "xxhash-rust" @@ -3463,9 +3464,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -3474,9 +3475,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -3485,13 +3486,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 379ff40c..a7d5bce6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,6 +83,7 @@ uuid = { version = "1", features = ["v4", "serde"] } cntryl-stress = { git = "https://github.com/cntryl/stress", branch = "main" } proptest = "1" serial_test = "4.0.1" +tokio = { version = "1", features = ["test-util"] } # Tier 1: Hot Path (Infrastructure - Pure sync internals, stress micro mode) [[bench]] diff --git a/benches/tier2_subsystem_schedule_churn.rs b/benches/tier2_subsystem_schedule_churn.rs index aba3d2f3..162d88e3 100644 --- a/benches/tier2_subsystem_schedule_churn.rs +++ b/benches/tier2_subsystem_schedule_churn.rs @@ -76,7 +76,7 @@ fn delete_then_full_list_shared_cache(ctx: &mut StressContext, name: &str, count let mut total = Duration::ZERO; for _ in 0..DELETE_CHURN_OPERATION_COUNT { - let (cached, cached_total) = actor.list_entries(0, 0); + let (cached, cached_total) = actor.list_entries(0, 0).expect("list entries"); assert_eq!( cached_total, count_u64, "delete churn setup should restore route" @@ -88,7 +88,7 @@ fn delete_then_full_list_shared_cache(ctx: &mut StressContext, name: &str, count route: route.clone(), }); assert!(matches!(response, ScheduleResponse::Ok)); - let (entries, total_count) = actor.list_entries(0, 0); + let (entries, total_count) = actor.list_entries(0, 0).expect("list entries"); total += started.elapsed(); assert_eq!( total_count, @@ -130,7 +130,7 @@ fn upsert_then_full_list_shared_cache(ctx: &mut StressContext, name: &str, count ]; let mut actor = create_test_actor(); populate_actor(&mut actor, &routes, &crons, &payloads, count); - let (cached, _) = actor.list_entries(0, 0); + let (cached, _) = actor.list_entries(0, 0).expect("list entries"); black_box(cached.len()); tier2_stress::measure_once(ctx, name, UPSERT_CHURN_OPERATION_COUNT, || { @@ -144,7 +144,7 @@ fn upsert_then_full_list_shared_cache(ctx: &mut StressContext, name: &str, count }); replacement_index = (replacement_index + 1) % replacement_crons.len(); assert!(matches!(response, ScheduleResponse::Ok)); - let (entries, total_count) = actor.list_entries(0, 0); + let (entries, total_count) = actor.list_entries(0, 0).expect("list entries"); black_box((entries.len(), total_count)); } }); diff --git a/benches/tier4_stream_direct.rs b/benches/tier4_stream_direct.rs index 4a183028..92818f5f 100644 --- a/benches/tier4_stream_direct.rs +++ b/benches/tier4_stream_direct.rs @@ -110,7 +110,6 @@ fn direct_write_dimensions( write_mode: match write_mode { StreamWriteMode::Buffered => "buffered", StreamWriteMode::Sync => "sync", - StreamWriteMode::CloudStrict => "cloud_strict", }, write_operation: "begin_append_commit", payload_size, diff --git a/docs/clients/acceptance/schedule-errors-performance.md b/docs/clients/acceptance/schedule-errors-performance.md index 3a205881..61924ff7 100644 --- a/docs/clients/acceptance/schedule-errors-performance.md +++ b/docs/clients/acceptance/schedule-errors-performance.md @@ -137,6 +137,7 @@ Clients **MUST** interpret error codes using this mapping. - `1001` (Transaction Not Found) → Fatal, do NOT retry - `6001` (ERR_RPC_TIMEOUT; worker accepted but did not reply before timeout) → Retryable with backoff - `6004` (ERR_ROUTE_NOT_REGISTERED; no workers registered for route) → Retryable with backoff +- `7010` (Schedule ERR_BACKEND_ERROR; backend unavailable or saturated) → Retryable with backoff when the operation is safe to replay - `1011` (KV Unauthorized) → Fatal, do NOT retry - `2009` (Stream Unauthorized) → Fatal, do NOT retry - `4009` (Queue Unauthorized) → Fatal, do NOT retry @@ -284,6 +285,7 @@ Error codes follow the format `XXYY` where: | 2010 | ERR_INVALID_SUBSCRIPTION_PATTERN | Subscription pattern syntax invalid | No | | 2011 | ERR_SUBSCRIPTION_LIMIT | Session exceeded 128 wildcard Stream registrations | No | | 2012 | ERR_BACKEND_ERROR | Stream storage backend error | Yes (with backoff) | +| 2013 | ERR_READ_RESPONSE_TOO_LARGE | A single record's wire-encoded size alone exceeds the maximum response frame size and can never be returned by any READ | No | ### Notice Domain (3000-3999) @@ -357,6 +359,7 @@ Error codes follow the format `XXYY` where: | 7007 | ERR_SUBSCRIPTION_LIMIT | Session exceeded 128 wildcard Schedule registrations | No | | 7008 | ERR_INVALID_DELIVERY_MODE | Delivery mode is not broadcast (0) or single (1) | No | | 7009 | ERR_UNAUTHORIZED | Permission denied for schedule operation | No | +| 7010 | ERR_BACKEND_ERROR | Schedule backend unavailable or saturated; not a cron or payload parse failure | Yes, when the operation is safe to replay | ### Error Handling Guidelines diff --git a/docs/clients/client-requirements.md b/docs/clients/client-requirements.md index c8e19929..81365627 100644 --- a/docs/clients/client-requirements.md +++ b/docs/clients/client-requirements.md @@ -100,7 +100,7 @@ The acceptance criteria in `client-acceptance-criteria.md` are the normative sou **REQ-PROTO-011 (T1)** The client MUST correctly handle all error code ranges and map each code to the right domain (AC-ERROR-002). Error code `XXYY` where `XX` identifies the domain and `YY` the specific error MUST NOT be confused across domains. -**REQ-PROTO-012 (T1)** The client MUST correctly categorize retryable vs. fatal error codes per the table in `client-acceptance-criteria.md` (AC-ERROR-003). Retryable codes: 1004, 4005, 5001, 6001, 6002, 6003, 6004. All Unauthorized codes and non-retryable codes MUST be treated as fatal (no retry). +**REQ-PROTO-012 (T1)** The client MUST correctly categorize retryable vs. fatal error codes per the table in `client-acceptance-criteria.md` (AC-ERROR-003). Retryable codes: 1004, 1014, 2014, 3006, 4005, 5001, 5007, 6001, 6002, 6003, 6004, 7010. All Unauthorized codes and non-retryable codes MUST be treated as fatal (no retry). Schedule 7010 retries remain subject to the operation's normal replay-safety rules. **REQ-PROTO-013 (T1)** Frame size MUST be respected. Default server limit is 1 MB (configurable). Clients SHOULD expose this as a configurable option. Individual TLV values MUST NOT exceed 65535 bytes regardless of frame size setting. @@ -281,7 +281,7 @@ Reconnect rebuild behavior is domain-specific: **REQ-ERR-001 (T0)** Every server error response (status byte = 1) MUST be surfaced to the caller as a non-nil error. Silent discard of server errors is a critical defect. -**REQ-ERR-002 (T0)** Every error MUST carry the numeric error code and the human-readable message from the server response payload. +**REQ-ERR-002 (T0)** Every coded error MUST carry the numeric error code and the human-readable message from the server response payload. Stream `READ` uses a coded error envelope; other Stream operations use their protocol-defined plain message envelope and MUST NOT be decoded as if a numeric code were present. **REQ-ERR-003 (T0)** `context.Context` cancellation and deadline expiry MUST be correctly propagated: if the calling context is cancelled before a response arrives, the operation MUST return `ctx.Err()` (or a wrapping error), and the pending response MUST be cleaned up. @@ -291,7 +291,7 @@ Reconnect rebuild behavior is domain-specific: **REQ-ERR-005 (T1)** Domain error codes MUST be exported as named constants (e.g., `ErrKvKeyExists`, `ErrLeaseHeld`, `ErrRpcRouteNotRegistered`) so callers can write `errors.Is(err, fitz.ErrKvKeyExists)` without hard-coding integers. -**REQ-ERR-006 (T1)** Retryable errors (codes 1004, 4005, 5001, 6001, 6002, 6003, 6004) MUST be distinguishable from fatal errors via a type assertion or helper (`fitz.IsRetryable(err) bool`). Callers MUST NOT be required to know the numeric ranges. +**REQ-ERR-006 (T1)** Retryable errors (codes 1004, 1014, 2014, 3006, 4005, 5001, 5007, 6001, 6002, 6003, 6004, 7010) MUST be distinguishable from fatal errors via a type assertion or helper (`fitz.IsRetryable(err) bool`). Callers MUST NOT be required to know the numeric ranges. **REQ-ERR-007 (T1)** Server error messages MUST be included in the `Error()` string. `fmt.Errorf("kv get: %w", err)` wrapping MUST preserve the code through the chain. diff --git a/docs/clients/four-client-parity-ledger.csv b/docs/clients/four-client-parity-ledger.csv index 8e497969..3f73c835 100644 --- a/docs/clients/four-client-parity-ledger.csv +++ b/docs/clients/four-client-parity-ledger.csv @@ -8,4 +8,5 @@ heartbeat,pass,pass,pass,pass,pass,"WebSocket ping-pong watchdog; TCP keepalive; observability,pass,pass,pass,pass,pass,"shared lifecycle vocabulary; request error state and subscription telemetry" error_ergonomics,pass,pass,pass,pass,pass,"language-native typed errors or sentinels with wrapping support" documentation_truth,pass,pass,pass,pass,pass,"default public APIs in examples; breaking migration notes" +exclusive_scan_resume,todo,todo,todo,todo,todo,"encode trailing start_exclusive byte on SCAN; paginate a byte-bounded scan to completion in both directions; forward-only fallback documented" reviewed_head,8c88483e948e466c289c658479087a83e845d214,4453e808c3e4b0e2ec2213fd4b515db0fd60dc60,fe877bbeac4843591b5a0115c8e4a04743febcce,bfba39ead69795448ee20985b1281543467ffad8,82de7f605d836cfcd55ace1f7a36398ad2ad3cc8,"immutable SDK inputs; hosted CI verified by exact commit" diff --git a/docs/clients/implementation/components-testing-performance.md b/docs/clients/implementation/components-testing-performance.md index d12cd84f..2e4aa235 100644 --- a/docs/clients/implementation/components-testing-performance.md +++ b/docs/clients/implementation/components-testing-performance.md @@ -120,6 +120,7 @@ KvClient 7000-7999: Schedule 7002: Invalid cron → Retryable: No 7009: Unauthorized → Retryable: No + 7010: Backend unavailable or saturated → Retryable: Yes when replay-safe See the client-acceptance-criteria.md appendix for the complete error code reference. ``` diff --git a/docs/clients/spec/lease-schedule.md b/docs/clients/spec/lease-schedule.md index 269ec4f5..9bc49c8a 100644 --- a/docs/clients/spec/lease-schedule.md +++ b/docs/clients/spec/lease-schedule.md @@ -483,7 +483,7 @@ elif response.type == "Fenced": - 5004 = ERR_LEASE_NOT_FOUND (route never acquired) - 5005 = ERR_INVALID_TOKEN (lease token invalid or wrong) - 5006 = ERR_TIMEOUT (pending acquire timed out) -- 5007 = ERR_QUEUE_FULL (too many pending waiters) +- 5007 = ERR_QUEUE_FULL (retryable; too many pending waiters, or the lease mailbox was full and the request was never accepted) - 5008 = ERR_BAD_REQUEST (malformed Lease operation request) - 5009 = ERR_UNAUTHORIZED - 5010 = ERR_INVALID_SUBSCRIPTION_ROUTE @@ -613,8 +613,18 @@ Response (error): **Semantics:** - Omitting the payload defaults to `offset=0, limit=100` -- `limit=0` means "all remaining entries from offset" -- LIST is scoped to the current route family and returns a single response payload, not a multi-frame stream +- `limit=0` requests all remaining entries from `offset`, but the response is + still one TLV value bounded by the wire frame limit and MAY return fewer + entries than exist, regardless of what `limit` requested. There is no + `has_more` flag on this response: detect truncation by comparing the + returned entry count to `total_count`. If `offset + entries_returned < + total_count`, more entries remain; continue by re-issuing LIST with + `offset += entries_returned` (same `limit`) until the count is exhausted. + Every entry sits at a stable index for the duration of an unchanging + definition set, so this offset advance is safe. +- LIST is scoped to the current route family and each call returns exactly one + response payload (never a multi-frame stream), but that payload may be a + partial page per the truncation rule above #### Broker Extensions @@ -764,6 +774,21 @@ or wildcard Schedule patterns via `SCHEDULE_SUBSCRIBE` and receiving - 7006 = ERR_INVALID_SUBSCRIPTION_PATTERN - 7007 = ERR_SUBSCRIPTION_LIMIT - 7008 = ERR_INVALID_DELIVERY_MODE +- 7010 = ERR_BACKEND_ERROR +- 7011 = ERR_TIMEOUT + +`ERR_BACKEND_ERROR` reports transient broker backend unavailability or +saturation. It is distinct from `ERR_PARSE_ERROR`: clients must not tell callers +that their cron or payload is malformed when the broker could not service an +otherwise valid request. Clients may classify 7010 as retryable, subject to the +operation's normal replay-safety rules. + +`ERR_TIMEOUT` reports that the broker accepted the command but did not finish it +before its deadline. The outcome is unknown: the command may still apply. It is +deliberately NOT retryable, because 7010 means the request was declined and is +safe to re-send, whereas re-sending after 7011 can apply the same create or +cancel twice. A client that knows its operation is idempotent may still retry +deliberately; an automatic `IsRetryable` retry must not. #### Acceptance Tests diff --git a/docs/clients/spec/notice-stream.md b/docs/clients/spec/notice-stream.md index 81f9ec57..b0e6fec6 100644 --- a/docs/clients/spec/notice-stream.md +++ b/docs/clients/spec/notice-stream.md @@ -360,6 +360,9 @@ CLIENT → SERVER (second unsubscribe, last handler removed): - 3003 = ERR_SUBSCRIPTION_LIMIT - 3004 = ERR_TRANSPORT_CLOSED - 3005 = ERR_BACKEND_ERROR +- 3006 = ERR_BUSY (retryable) + +`ERR_BUSY` means the domain mailbox was full and the request was never accepted. Nothing applied, so re-sending after a backoff is safe; this is distinct from `ERR_BACKEND_ERROR`, which is fatal and says nothing about whether the request took effect. - 3009 = ERR_UNAUTHORIZED #### Acceptance Tests @@ -450,6 +453,13 @@ Response (status=1): **Optional discriminator:** Clients MAY include an immutable discriminator string on APPEND. The broker stores it as a replay sidecar and uses it only for filtered reads. Clients that do not need filtered replay SHOULD omit it. +**Event size:** `body_len + metadata_len` MUST NOT exceed 61,247 bytes. This +limit reserves enough room in the `u16`-length READ response TLV for the +largest valid 4,096-byte route and the encoded record/response overhead, so +every accepted event can be replayed. The discriminator does not count toward +this event-payload limit, but it still counts toward the configured append +batch and ingress-frame limits. + **Design Note:** The `data` field in Stream responses carries broker-defined metadata (e.g., current watermark, stream info). Clients MUST parse past it (read `data_len` bytes) but SHOULD NOT interpret its contents unless broker documentation specifies a schema. **Design Note:** `session_id` is `u64` (not string), returned from BEGIN response. @@ -749,6 +759,13 @@ class StreamSession: #### Error Codes (2xxx) +Stream uses operation-specific error envelopes. `READ` errors are +`[status=1][u32 error_code][string message]` and preserve the numeric 2xxx +code. Every other Stream operation uses the plain +`[status=1][string message]` envelope. Clients must select the decoder from the +request message type; they must not consume the first four bytes of a +non-`READ` message as an error code. + - 2001 = ERR_CONCURRENCY_CONFLICT (expected_offset mismatch) - 2002 = ERR_SESSION_ALREADY_ACTIVE - 2003 = ERR_SESSION_NOT_FOUND @@ -760,6 +777,10 @@ class StreamSession: - 2010 = ERR_INVALID_SUBSCRIPTION_PATTERN - 2011 = ERR_SUBSCRIPTION_LIMIT - 2012 = ERR_BACKEND_ERROR +- 2013 = ERR_READ_RESPONSE_TOO_LARGE (a single record's wire-encoded size alone exceeds the maximum broker response frame size and can never be returned by any READ call at that offset; this is distinct from `max_bytes` pagination, which stops a page early instead of failing) +- 2014 = ERR_BUSY (retryable) + +`ERR_BUSY` means the domain mailbox was full and the request was never accepted. Nothing applied, so re-sending after a backoff is safe; this is distinct from `ERR_BACKEND_ERROR`, which is fatal and says nothing about whether the request took effect. #### Acceptance Tests diff --git a/docs/clients/spec/queue-rpc-kv.md b/docs/clients/spec/queue-rpc-kv.md index 5e83dcd3..03f50a2e 100644 --- a/docs/clients/spec/queue-rpc-kv.md +++ b/docs/clients/spec/queue-rpc-kv.md @@ -73,6 +73,14 @@ contains a whole-segment wildcard, decode `concrete_route` before each item and use that route for EXTEND and COMPLETE. Wildcard reservation supports `*` and `**` as complete segments, including unknown realm, area, or resource segments. +The broker bounds each RESERVE response to one TLV value. If a queued message +cannot fit in an otherwise empty response using the requested concrete or +wildcard item encoding, the broker moves it to dead-letter state with reason +`reserve_response_too_large` under the configured Queue write policy and +continues reserving later work. A message that fits an empty response but not +the remaining bytes of a partial +batch stays ready for the next RESERVE response. + #### EXTEND Request ``` @@ -666,6 +674,7 @@ Response (error): [u8] has_limit (0 or 1) [u32 BE] limit (if present) [u8] reverse (0 or 1) +[u8] start_exclusive (0 or 1, optional; absent means 0) Response (success): [u8] 0 (status: success) [u32 BE] item_count @@ -809,6 +818,13 @@ Only `0` and `1` are valid durability values. Other values are rejected. ##### SCAN Semantics +**`start_exclusive` flag:** + +- `start_exclusive=0` (default, and the value assumed when the byte is absent): + `start_key` is inclusive, as described below +- `start_exclusive=1`: the scan begins strictly after `start_key` in the + selected direction. Required for continuation; see the pagination rules below + **`reverse` flag:** - `reverse=0` (forward): Scan keys in ascending lexicographic order @@ -820,9 +836,38 @@ Only `0` and `1` are valid durability values. Other values are rejected. the lower side unbounded - Equal bounds, or bounds inverted for the selected direction, return an empty successful result -- `limit` applies regardless of direction. `has_more=1` means at least one - additional matching key exists beyond the returned page; an omitted limit is - unlimited and returns `has_more=0` +- `limit` applies regardless of direction and bounds a page from above. The + broker bounds every page by two further limits: at most 1024 items, and at + most what fits in one response frame (a scan response is carried as a single + TLV value with a `u16` length, so a page of large values reaches the byte + bound well before the item cap). A page ends at whichever limit is reached + first. +- **An omitted `limit` therefore does not mean unlimited.** It means "as much as + fits in one page". A scan of 300 keys with 1 KiB values returns a partial page + with `has_more=1` even though no limit was supplied. +- `has_more=1` means at least one additional matching key exists beyond the + returned page. Clients MUST honour it whether or not they supplied a `limit`; + treating an omitted limit as complete silently leaves data unread. +- To continue, re-issue the scan with `start_key` set to the last key returned + by the previous page, `start_exclusive=1`, and the same `reverse` value and + opposite bound. Repeat until `has_more=0`. +- `start_exclusive` is a trailing optional byte on the SCAN request, defaulting + to `0` when absent. Resuming with an inclusive `start_key` does not + terminate: a page bounded by the byte budget can hold a single pair, and the + next request then returns that same pair forever, so later keys are never + reached. + +**Mixed-version behaviour.** The byte is additive and older brokers reject +trailing data, so clients MUST NOT send it unless the broker advertises support. +Clients that cannot yet encode it can still paginate **forward** with no wire +change: re-issue with `start_key` set to the last returned key followed by a +single `0x00` byte, which is that key's immediate successor and therefore an +exclusive resume. **Reverse** continuation has no such equivalent - the +symmetric operation is a byte-string predecessor, which is not expressible - so +a client that cannot send `start_exclusive` must not paginate reverse scans +across a byte-bounded page. Until a client ships the byte, keep reverse scans +within a single page by supplying a `limit` small enough that the page is not +byte-bounded. #### Usage Example @@ -874,6 +919,9 @@ still maintains live session-scoped transaction state keyed by `tx_id`. - 1011 = ERR_UNAUTHORIZED - 1012 = ERR_INVALID_SUBSCRIPTION_PATTERN - 1013 = ERR_SUBSCRIPTION_LIMIT +- 1014 = ERR_BUSY (retryable) + +`ERR_BUSY` means the domain mailbox was full and the request was never accepted. Nothing applied, so re-sending after a backoff is safe; this is distinct from `ERR_BACKEND_ERROR`, which is fatal and says nothing about whether the request took effect. #### Acceptance Tests diff --git a/docs/development/routing-design.md b/docs/development/routing-design.md index c1d08927..a8640b47 100644 --- a/docs/development/routing-design.md +++ b/docs/development/routing-design.md @@ -30,6 +30,12 @@ The design makes these decisions explicitly: dimensions. 6. Global Stream selectors use one real, contiguous, family-global offset space. Sorted traversal of independent realms is not global order. + +Stream request completion is ordered within one resource, not across +independent resources. Different resources in the same `RouteFamily` may be +executed concurrently, so their client responses may arrive in either order. +Durable resource, area, realm, and family-global offsets plus captured +watermarks are the only cross-resource ordering authority. 7. One family-keyed ordering coordinator serializes only exact global-range assignment. Area and realm offsets are assigned by the resource data transaction and their counters commit atomically with the records. Resource diff --git a/docs/operations/auth-browser-deployment.md b/docs/operations/auth-browser-deployment.md index d4adada4..567f5abb 100644 --- a/docs/operations/auth-browser-deployment.md +++ b/docs/operations/auth-browser-deployment.md @@ -37,6 +37,15 @@ Configuration constraints: - Do not use `FITZ_JWT_HMAC_SECRET` outside testing or local prototyping. - Issue short-lived tokens and reconnect with a fresh token on expiry. +When Fitz rejects a runtime JWT, the CONNECT failure log includes bounded +diagnostics for the algorithm, key ID, issuer, audience, time bounds, configured +permission-source names, and values found in those permission sources. The log +uses a short SHA-256 token fingerprint for correlation. It never includes the +compact JWT or signature. The added diagnostic fields exclude the subject, +identity values, and unrelated claims. Treat the reported header and payload +fields as untrusted troubleshooting input; they are decoded only after the +normal verification path has rejected the token. + ## Admin and Browser Perimeter Set at least these environment values: diff --git a/docs/operations/migration-guide.md b/docs/operations/migration-guide.md index 051bdd7e..12b8a379 100644 --- a/docs/operations/migration-guide.md +++ b/docs/operations/migration-guide.md @@ -69,6 +69,12 @@ Upgrade every Schedule client decoder before routing traffic to the new broker. Rollback requires restoring the prior broker and prior client codec together; mixed versions cannot safely decode 705 frames. +Schedule backend unavailability and saturation now use the dedicated +`ERR_BACKEND_ERROR` (`7010`) wire code. Upgrade clients to preserve and classify +that code as transient, subject to operation replay safety. Do not map these +failures to `ERR_PARSE_ERROR` (`7004`), which incorrectly tells callers that +their cron or payload is malformed. + ### Subscription registration contract KV, Queue, Notice, Stream, RPC, and Schedule now share strict whole-segment @@ -149,6 +155,20 @@ mismatch counter. actor already gets its own processing thread, so the argument had no effect; call `Scheduler::new()`. +## Removed KV Authorization And Metrics Facades + +The public `fitz::domains::kv::SessionActor` authorization helper has been +removed. Send KV frames through runtime ingress, which authorizes BEGIN against +the exact `kv://{realm}/{area}/{resource}` route and keeps subsequent +transaction operations session-owned. Direct state-machine tests may continue +to use `fitz::domains::kv::KvActor`, but application authorization must not be +reimplemented around it. + +The public `fitz::domains::kv::KvMetrics` path has also been removed. Configure +KV metrics through `KvDomainSink::with_metrics` before registering the sink with +the router. The consuming configuration method rebuilds the sink's private +actor and returns the configured sink. + ## Breaking: Single-Generation Storage Formats **This upgrade cannot read any store written by an earlier broker.** Every @@ -184,6 +204,22 @@ Existing drop-counter names are unchanged, including `fitz_notice_delivery_drops_total`, which keeps its `delivery` spelling rather than the `notify` spelling used by the other domains. +## Stream Rust API Cleanup + +New construction code should call `StreamDomainSink::try_new` and handle +`StreamSinkInitError`. `StreamDomainSink::new` remains as a compatibility +wrapper and retains its historical panic-on-initialization behavior. + +The client-facing `StreamWriteMode` now contains only `Buffered` and `Sync`. +Cloud provider acknowledgement remains a broker storage-policy choice for +`Sync`; callers must replace `StreamWriteMode::CloudStrict` with `Sync` and +configure cloud-strict write options when constructing the sink. + +The unused `StreamEvent`, `parse_stream_route`, and public `StreamMetrics` +paths were removed. Use protocol `StreamMessage` values, the typed +three-segment Stream selector grammar, and `StreamDomainSink::with_metrics`, +respectively. + ## Pre-Upgrade Checklist 1. Back up durability-sensitive state. diff --git a/src/api/admin/list/resource_inventory.rs b/src/api/admin/list/resource_inventory.rs index 3d0ebd65..7749f1fc 100644 --- a/src/api/admin/list/resource_inventory.rs +++ b/src/api/admin/list/resource_inventory.rs @@ -634,19 +634,20 @@ pub fn kv_prefix_scan_for_resource( prefix, limit, ) { - Ok((items, has_more)) => Ok(crate::api::admin::json_response(KvPrefixScanResponse { + Ok(result) => Ok(crate::api::admin::json_response(KvPrefixScanResponse { route_family: family, realm: path.realm.to_string(), area: path.area.to_string(), resource: path.resource.to_string(), prefix: kv_byte_value(prefix), limit, - has_more, - items: items + has_more: result.has_more, + items: result + .items .into_iter() - .map(|(key, value)| KvCommittedPair { - key: kv_byte_value(&key), - value: kv_byte_value(&value), + .map(|item| KvCommittedPair { + key: kv_byte_value(&item.key), + value: kv_byte_value(&item.value), }) .collect(), })), @@ -682,26 +683,26 @@ pub fn kv_rows_for_resource( cursor, limit, }) { - Ok((items, next_cursor, has_more)) => { - Ok(crate::api::admin::json_response(KvRowsResponse { - route_family: family, - realm: path.realm.to_string(), - area: path.area.to_string(), - resource: path.resource.to_string(), - starts_with: kv_byte_value(starts_with), - limit, - next_cursor: next_cursor - .map(|cursor| base64::engine::general_purpose::STANDARD.encode(cursor)), - has_more, - items: items - .into_iter() - .map(|(key, value)| KvCommittedPair { - key: kv_byte_value(&key), - value: kv_byte_value(&value), - }) - .collect(), - })) - } + Ok(result) => Ok(crate::api::admin::json_response(KvRowsResponse { + route_family: family, + realm: path.realm.to_string(), + area: path.area.to_string(), + resource: path.resource.to_string(), + starts_with: kv_byte_value(starts_with), + limit, + next_cursor: result + .next_cursor + .map(|cursor| base64::engine::general_purpose::STANDARD.encode(cursor)), + has_more: result.has_more, + items: result + .items + .into_iter() + .map(|item| KvCommittedPair { + key: kv_byte_value(&item.key), + value: kv_byte_value(&item.value), + }) + .collect(), + })), Err(error) => Ok(kv_storage_error_response(&error)), } } diff --git a/src/api/admin/troubleshooting/analysis_queue_rpc.rs b/src/api/admin/troubleshooting/analysis_queue_rpc.rs index 9e4794d3..d298ab7c 100644 --- a/src/api/admin/troubleshooting/analysis_queue_rpc.rs +++ b/src/api/admin/troubleshooting/analysis_queue_rpc.rs @@ -9,6 +9,14 @@ use super::{ ScoredHotspot, }; +/// Explanation for RPC entries labelled `DataLossRisk`. +/// +/// RPC holds no durable state: a lost response is in-flight work dropped +/// under transport backpressure. Saying "durability gap" here sends an +/// operator hunting for storage corruption that cannot exist. +pub(super) const RPC_RESPONSE_LOSS_HINT: &str = + "Ephemeral RPC response loss caused by transport backpressure; no durable state is affected"; + fn i64_from_u64(value: u64) -> i64 { i64::try_from(value).unwrap_or(i64::MAX) } @@ -463,6 +471,14 @@ pub(crate) fn analyze_rpc( DiagnosticSeverity::High, Some("correlation mismatch".to_string()), ) + } else if transport_pressure > 0 { + // The backlog is not the problem; the path to the client is. + ( + DiagnosisLabel::TransportBackpressure, + DiagnosticTrend::Growing, + DiagnosticSeverity::High, + Some("transport backpressure".to_string()), + ) } else if pending_count > worker_count && (age_seconds.unwrap_or(0) >= 30 || pending_count >= worker_count.saturating_mul(2)) { @@ -509,16 +525,18 @@ pub(crate) fn analyze_rpc( None, ) }; - let failure_count = if matches!(label, DiagnosisLabel::DataLossRisk) { - if late_response_pressure > 0 { - late_response_pressure - } else if correlation_pressure > 0 { - correlation_pressure - } else { - 0 + let failure_count = match label { + DiagnosisLabel::DataLossRisk => { + if late_response_pressure > 0 { + late_response_pressure + } else { + correlation_pressure + } } - } else { - 0 + // Shed work is failed work: counting it as zero is what let a + // saturated broker report success totals with no failures. + DiagnosisLabel::TransportBackpressure => transport_pressure, + _ => 0, }; let mut hints = vec![]; if pending_count > 0 { @@ -548,6 +566,9 @@ pub(crate) fn analyze_rpc( if late_response_pressure > 0 { hints.push(format!("{late_response_pressure} late response drop(s)")); } + if matches!(label, DiagnosisLabel::DataLossRisk) { + hints.push(RPC_RESPONSE_LOSS_HINT.to_string()); + } if transport_pressure > 0 { hints.push(format!( "{transport_pressure} timeout/backpressure rejection(s)" @@ -619,18 +640,16 @@ pub(crate) fn analyze_rpc( let label = if late_response_pressure > 0 || correlation_pressure > 0 { DiagnosisLabel::DataLossRisk } else { - DiagnosisLabel::Throughput + // Not throughput: nothing is flowing slowly, work is being + // shed because the transport cannot carry it. + DiagnosisLabel::TransportBackpressure }; let trend = if late_response_pressure > 0 || correlation_pressure > 0 { DiagnosticTrend::Stalled } else { DiagnosticTrend::Growing }; - let severity = if late_response_pressure > 0 || correlation_pressure > 0 { - DiagnosticSeverity::High - } else { - DiagnosticSeverity::Medium - }; + let severity = DiagnosticSeverity::High; let mut hints = vec![]; if request_timeouts_total > 0 { hints.push(format!("{request_timeouts_total} request timeout(s)")); @@ -694,7 +713,11 @@ pub(crate) fn analyze_rpc( .unwrap_or(0), ), recent_transition_count: data_loss_pressure, - failure_count: data_loss_pressure, + failure_count: if data_loss_pressure > 0 { + data_loss_pressure + } else { + transport_pressure + }, contention_count: correlation_pressure, waiter_count: pending.len(), explanation_hints: hints, diff --git a/src/api/admin/troubleshooting/model.rs b/src/api/admin/troubleshooting/model.rs index 221c50af..33bae4e8 100644 --- a/src/api/admin/troubleshooting/model.rs +++ b/src/api/admin/troubleshooting/model.rs @@ -45,6 +45,13 @@ pub enum DiagnosisLabel { DeadLetterPressure, WorkerStarvation, DataLossRisk, + /// The domain is shedding work because the transport cannot carry it. + /// + /// Distinct from `BacklogGrowth`: the backlog is not the problem, the + /// path to the client is. Timeouts and backpressure rejections were + /// previously computed and then discarded into a hint string, so this + /// condition reported as healthy. + TransportBackpressure, } impl DiagnosisLabel { @@ -58,6 +65,7 @@ impl DiagnosisLabel { Self::DeadLetterPressure => "dead_letter_pressure", Self::WorkerStarvation => "worker_starvation", Self::DataLossRisk => "data_loss_risk", + Self::TransportBackpressure => "transport_backpressure", } } @@ -71,6 +79,7 @@ impl DiagnosisLabel { Self::DeadLetterPressure => "dead-letter pressure", Self::WorkerStarvation => "worker starvation", Self::DataLossRisk => "data-loss risk", + Self::TransportBackpressure => "transport backpressure", } } @@ -83,7 +92,10 @@ impl DiagnosisLabel { Self::StaleHandoff => "A durable handoff is overdue", Self::DeadLetterPressure => "Dead letters are accumulating", Self::WorkerStarvation => "Work is waiting for workers or owners", - Self::DataLossRisk => "The control plane sees a durability gap", + Self::DataLossRisk => "The control plane sees accepted work that was never delivered", + Self::TransportBackpressure => { + "Requests are being shed because the client transport is saturated" + } } } @@ -98,7 +110,14 @@ impl DiagnosisLabel { Self::StaleHandoff => Some("Durable ownership or schedule state with live lateness"), Self::DeadLetterPressure => Some("Durable failure state plus live retry pressure"), Self::WorkerStarvation => Some("Mostly live capacity pressure"), - Self::DataLossRisk => Some("Potential durable-state loss; treat this as critical"), + // Deliberately does not assert *durable* loss: this label is also + // raised for ephemeral domains such as RPC, where the loss is + // in-flight work rather than stored state. Each analysis adds the + // domain-accurate detail (see `RPC_RESPONSE_LOSS_HINT`). + Self::DataLossRisk => Some("Accepted work was lost; treat this as critical"), + Self::TransportBackpressure => { + Some("Live delivery pressure; durable state is not implicated") + } } } @@ -112,6 +131,7 @@ impl DiagnosisLabel { "dead_letter_pressure" => Self::DeadLetterPressure, "worker_starvation" => Self::WorkerStarvation, "data_loss_risk" => Self::DataLossRisk, + "transport_backpressure" => Self::TransportBackpressure, _ => return None, }) } @@ -461,7 +481,9 @@ fn primary_signal_for_stage( signals.contention_signal() || signals.age_signal(), 2, ), - DiagnosisLabel::DeadLetterPressure | DiagnosisLabel::DataLossRisk => { + DiagnosisLabel::DeadLetterPressure + | DiagnosisLabel::DataLossRisk + | DiagnosisLabel::TransportBackpressure => { ("failure_signal_present", signals.failure_signal(), 2) } DiagnosisLabel::StaleHandoff => ( diff --git a/src/api/admin/troubleshooting/tests.rs b/src/api/admin/troubleshooting/tests.rs index 4cf60024..2987bcdd 100644 --- a/src/api/admin/troubleshooting/tests.rs +++ b/src/api/admin/troubleshooting/tests.rs @@ -693,3 +693,61 @@ fn should_summarize_incident_given_broker_hotspot() { "inspect /api/v1/all/metrics" ); } + +#[test] +fn should_report_rpc_response_loss_as_ephemeral_not_durable() { + // Arrange + // RPC holds no durable state: a lost response is in-flight work dropped + // under transport backpressure. Calling it a "durability gap" and warning + // about "durable-state loss" sends an operator hunting for storage + // corruption that cannot exist. + let label = DiagnosisLabel::DataLossRisk; + + // Act + let hints = crate::api::admin::troubleshooting::model::canonical_explanation_hints( + label, + vec![super::analysis_queue_rpc::RPC_RESPONSE_LOSS_HINT.to_string()], + ); + + // Assert + let joined = hints.join(" | "); + assert!( + hints + .iter() + .any(|hint| hint.contains("Ephemeral RPC response loss")), + "expected an ephemeral RPC explanation, got {joined}" + ); + assert!( + !joined.contains("durable-state loss"), + "must not claim durable-state loss for an ephemeral domain: {joined}" + ); +} + +#[test] +fn should_classify_rpc_transport_backpressure() { + // Arrange + // request_timeouts_total + backpressure_rejects_total was computed as + // `transport_pressure` and then used only in a hint string, so a broker + // shedding load under transport saturation still reported healthy with a + // zero failure count. + let now = Utc::now(); + + // Act + let analysis = analyze_rpc(&[], &[], 4, 7, 0, 0, 0, 0, now); + let hotspot = analysis.hotspots.first().expect("rpc hotspot"); + + // Assert + assert_eq!( + hotspot.hotspot.snapshot.diagnosis_label(), + DiagnosisLabel::TransportBackpressure, + "transport pressure must drive the diagnosis, not just a hint" + ); + assert_eq!( + hotspot.hotspot.snapshot.failure_count, 11, + "timeouts and backpressure rejections must both be counted" + ); + assert_ne!( + hotspot.hotspot.snapshot.severity, + DiagnosticSeverity::Informational + ); +} diff --git a/src/api/handlers/websocket.rs b/src/api/handlers/websocket.rs index 2695a4bc..e5c6159c 100644 --- a/src/api/handlers/websocket.rs +++ b/src/api/handlers/websocket.rs @@ -51,6 +51,13 @@ fn bounded_websocket_config(max_frame_size: usize) -> WebSocketConfig { .max_frame_size(Some(max_frame_size)) } +fn cache_websocket_authentication(authenticated: &mut bool, lookup: impl FnOnce() -> bool) -> bool { + if !*authenticated { + *authenticated = lookup(); + } + *authenticated +} + #[allow(clippy::too_many_arguments)] pub(super) async fn handle_websocket( req: Request, @@ -430,8 +437,15 @@ where { use futures_util::StreamExt; use hyper_tungstenite::tungstenite::Message; + let mut authenticated = false; loop { - let next_message = if context.session.info().authenticated { + let authentication_complete = cache_websocket_authentication(&mut authenticated, || { + context + .ingress + .get_session_info(context.session_id) + .is_some_and(|session| session.authenticated) + }); + let next_message = if authentication_complete { ws_receiver.next().await } else { let remaining = context @@ -546,8 +560,9 @@ where #[cfg(test)] mod tests { use super::{ - bounded_websocket_config, is_normal_websocket_disconnect, send_websocket_batch, - websocket_close_reason, websocket_origin_allowed, websocket_session_frame_error_reason, + bounded_websocket_config, cache_websocket_authentication, is_normal_websocket_disconnect, + send_websocket_batch, websocket_close_reason, websocket_origin_allowed, + websocket_session_frame_error_reason, }; use crate::protocol::frame::ChannelId; use crate::session::{CloseReason, SessionError}; @@ -557,6 +572,24 @@ mod tests { use hyper_tungstenite::tungstenite::Error as WsError; use hyper_tungstenite::tungstenite::Message; + #[test] + fn should_stop_authentication_lookups_after_websocket_authenticates() { + // Arrange + let lookups = std::cell::Cell::new(0); + let mut authenticated = false; + + // Act + for _ in 0..2 { + assert!(cache_websocket_authentication(&mut authenticated, || { + lookups.set(lookups.get() + 1); + true + })); + } + + // Assert + assert_eq!(lookups.get(), 1); + } + #[test] fn should_treat_websocket_backpressure_as_terminal_session_error() { // Arrange diff --git a/src/api/outbound.rs b/src/api/outbound.rs index 2feb859c..711f401c 100644 --- a/src/api/outbound.rs +++ b/src/api/outbound.rs @@ -6,7 +6,7 @@ use crate::runtime::router::MailboxSink; use crate::runtime::EncodedClientFrame; use bytes::{BufMut, Bytes, BytesMut}; -use std::time::Instant; +use std::time::{Duration, Instant}; use tokio::sync::mpsc; use tracing::{debug, trace, warn}; @@ -147,7 +147,7 @@ impl SessionOutboundSink { "Outbound sink: encoding TLV response for session" ); let encode_start = Self::encode_latency_start(); - let bytes = encode_single_tlv_frame(ctx.msg_type, &ctx.payload); + let bytes = encode_single_tlv_frame(ctx.msg_type, &ctx.payload)?; Self::observe_encode_latency(encode_start); self.send_encoded_frame(ctx.session_id, &bytes) } @@ -167,7 +167,7 @@ impl SessionOutboundSink { let bytes = encode_single_tlv_frame( crate::protocol::tlv::MessageType::new(frame.meta.message_type), &frame.payload, - ); + )?; Self::observe_encode_latency(encode_start); self.send_encoded_frame(frame.meta.session_id, &bytes) } @@ -248,7 +248,7 @@ impl SessionOutboundSink { let bytes = encode_single_tlv_frame( crate::protocol::tlv::MessageType::new(response.meta.message_type), &payload, - ); + )?; Self::observe_encode_latency(encode_start); self.send_encoded_frame(response.meta.session_id, &bytes) } @@ -271,7 +271,7 @@ impl SessionOutboundSink { let bytes = encode_single_tlv_frame( crate::protocol::tlv::MessageType::new(crate::protocol::kv::msg_type::NOTIFY), &payload, - ); + )?; Self::observe_encode_latency(encode_start); self.send_encoded_frame(notification.session_id, &bytes) } @@ -291,7 +291,7 @@ impl SessionOutboundSink { let bytes = encode_single_tlv_frame( crate::protocol::tlv::MessageType::new(response.meta.message_type), &payload, - ); + )?; Self::observe_encode_latency(encode_start); self.send_encoded_frame(response.meta.session_id, &bytes) } @@ -314,7 +314,7 @@ impl SessionOutboundSink { let bytes = encode_single_tlv_frame( crate::protocol::tlv::MessageType::new(crate::protocol::lease_codec::msg_type::NOTIFY), &payload, - ); + )?; Self::observe_encode_latency(encode_start); self.send_encoded_frame(notification.session_id, &bytes) } @@ -334,7 +334,7 @@ impl SessionOutboundSink { let bytes = encode_single_tlv_frame( crate::protocol::tlv::MessageType::new(response.meta.message_type), &payload, - ); + )?; Self::observe_encode_latency(encode_start); self.send_encoded_frame(response.meta.session_id, &bytes) } @@ -354,7 +354,7 @@ impl SessionOutboundSink { ¬ification.route, ¬ification.payload, ); - let bytes = encode_single_tlv_frame(crate::protocol::tlv::MessageType::new(504), &payload); + let bytes = encode_single_tlv_frame(crate::protocol::tlv::MessageType::new(504), &payload)?; Self::observe_encode_latency(encode_start); self.send_encoded_frame(notification.session_id, &bytes) } @@ -377,7 +377,7 @@ impl SessionOutboundSink { let bytes = encode_single_tlv_frame( crate::protocol::tlv::MessageType::new(response.meta.message_type), &payload, - ); + )?; Self::observe_encode_latency(encode_start); self.send_encoded_frame(response.meta.session_id, &bytes) } @@ -396,7 +396,7 @@ impl SessionOutboundSink { ¬ification.route, ¬ification.payload, ); - let bytes = encode_single_tlv_frame(crate::protocol::tlv::MessageType::new(705), &payload); + let bytes = encode_single_tlv_frame(crate::protocol::tlv::MessageType::new(705), &payload)?; Self::observe_encode_latency(encode_start); self.send_encoded_frame(notification.session_id, &bytes) } @@ -419,7 +419,7 @@ impl SessionOutboundSink { let bytes = encode_single_tlv_frame( crate::protocol::tlv::MessageType::new(response.meta.message_type), &payload, - ); + )?; Self::observe_encode_latency(encode_start); self.send_encoded_frame(response.meta.session_id, &bytes) } @@ -439,7 +439,7 @@ impl SessionOutboundSink { ¬ification.route, ¬ification.payload, ); - let bytes = encode_single_tlv_frame(crate::protocol::tlv::MessageType::new(609), &payload); + let bytes = encode_single_tlv_frame(crate::protocol::tlv::MessageType::new(609), &payload)?; Self::observe_encode_latency(encode_start); self.send_encoded_frame(notification.session_id, &bytes) } @@ -462,7 +462,7 @@ impl SessionOutboundSink { let bytes = encode_single_tlv_frame( crate::protocol::tlv::MessageType::new(response.meta.message_type), &payload, - ); + )?; Self::observe_encode_latency(encode_start); self.send_encoded_frame(response.meta.session_id, &bytes) } @@ -485,13 +485,41 @@ impl SessionOutboundSink { let bytes = encode_single_tlv_frame( crate::protocol::tlv::MessageType::new(crate::protocol::queue_codec::msg_type::NOTIFY), &payload, - ); + )?; Self::observe_encode_latency(encode_start); - self.send_encoded_frame(notification.session_id, &bytes) + // Best-effort: this is delivered synchronously from the Queue domain + // actor thread, serially per watcher, BEFORE that actor replies to the + // client whose write just committed. The default budget can block up + // to ~177ms per saturated consumer; a handful of saturated watchers + // would alone exceed the actor's reply deadline for a request that + // already succeeded. A missed ready-notification is not data loss - + // the watcher's own next poll or RESERVE observes current state - so + // this gives up in microseconds rather than blocking the actor. + self.send_encoded_frame_with_budget( + notification.session_id, + &bytes, + OUTBOUND_BEST_EFFORT_RETRIES, + ) } + // Every `deliver_*` caller of this reaches `SessionOutboundSink::deliver` + // synchronously from whatever domain actor thread produced the response - + // that thread is shared by every session routed to the same actor/key. + // A budget that can sleep (previously up to ~177ms across 100 attempts) + // lets one session's saturated outbound channel stall every other + // session queued behind it on that actor. Use the same yield-only budget + // already required for the Queue ready-notification path below, for the + // same reason: give up in microseconds rather than block the actor. fn send_encoded_frame(&self, session_id: u64, bytes: &Bytes) -> Result<(), DeliveryError> { - const MAX_OUTBOUND_SEND_RETRIES: usize = 100; + self.send_encoded_frame_with_budget(session_id, bytes, OUTBOUND_BEST_EFFORT_RETRIES) + } + + fn send_encoded_frame_with_budget( + &self, + session_id: u64, + bytes: &Bytes, + max_retries: usize, + ) -> Result<(), DeliveryError> { let metrics_enabled = obs::hot_path_metrics_enabled(); trace!( @@ -530,10 +558,11 @@ impl SessionOutboundSink { ); } attempt += 1; - if attempt >= MAX_OUTBOUND_SEND_RETRIES { + if attempt >= max_retries { warn!( session_id = session_id, capacity = capacity, + attempts = attempt, "Outbound sink: transport channel full" ); return Err(DeliveryError::MailboxFull { @@ -541,7 +570,12 @@ impl SessionOutboundSink { current_len: capacity, }); } - std::thread::yield_now(); + // A best-effort budget never leaves the yield-only range, + // so it can never reach the sleeping tail of the backoff. + match outbound_retry_backoff(attempt) { + Some(delay) => std::thread::sleep(delay), + None => std::thread::yield_now(), + } } Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => { warn!( @@ -555,13 +589,57 @@ impl SessionOutboundSink { } } -fn encode_single_tlv_frame(msg_type: crate::protocol::tlv::MessageType, payload: &[u8]) -> Bytes { - assert!( - u16::try_from(payload.len()).is_ok(), - "TLV value too large: {} bytes (max {})", - payload.len(), - u16::MAX - ); +/// Sample size used only to exercise `outbound_retry_backoff`'s general +/// escalation shape in tests; no live caller requests this many attempts +/// since every `deliver_*` path now uses the yield-only best-effort budget. +#[cfg(test)] +const MAX_OUTBOUND_SEND_RETRIES: usize = 100; +/// Attempts for a best-effort delivery made synchronously from a domain +/// actor thread with its own reply deadline (e.g. Queue ready-notifications). +/// Bounded to the yield-only range so this can never sleep - see +/// `outbound_retry_backoff`. +const OUTBOUND_BEST_EFFORT_RETRIES: usize = OUTBOUND_YIELD_ATTEMPTS; +/// Attempts served by a cheap yield before real waiting begins. +const OUTBOUND_YIELD_ATTEMPTS: usize = 8; +/// Ceiling on any single wait between send attempts. +const OUTBOUND_MAX_RETRY_BACKOFF: Duration = Duration::from_millis(2); + +/// How long to wait before outbound send attempt `attempt`. +/// +/// `None` means yield instead of sleeping. The first few attempts stay on a +/// yield because a transport channel that is momentarily full usually drains +/// within a scheduling quantum. Past that, spinning is not waiting: a hundred +/// `yield_now` calls elapse in microseconds, so a frame would be abandoned +/// before a briefly-saturated consumer could possibly catch up. The remaining +/// attempts escalate to a bounded sleep so a real burst gets real time. +fn outbound_retry_backoff(attempt: usize) -> Option { + if attempt < OUTBOUND_YIELD_ATTEMPTS { + return None; + } + let step = attempt - OUTBOUND_YIELD_ATTEMPTS; + let micros = 100_u64.saturating_mul(1_u64 << step.min(5)); + Some(Duration::from_micros(micros).min(OUTBOUND_MAX_RETRY_BACKOFF)) +} + +/// Frame one TLV value for the wire. +/// +/// # Errors +/// +/// Returns `DeliveryError::InvalidPayload` when the payload exceeds the `u16` +/// length a TLV value can carry. This used to be an assertion, which turned +/// any aggregate-overflow bug in any domain - a schedule listing, a large read +/// page - into a broker panic. Framing must fail the one delivery, never the +/// process; the real fix always lives at the source, which must paginate. +fn encode_single_tlv_frame( + msg_type: crate::protocol::tlv::MessageType, + payload: &[u8], +) -> Result { + if u16::try_from(payload.len()).is_err() { + return Err(DeliveryError::InvalidPayload { + len: payload.len(), + max: usize::from(u16::MAX), + }); + } let header_len = msg_type.encoded_type_len() + 2; let mut out = BytesMut::with_capacity(header_len + payload.len()); @@ -576,7 +654,7 @@ fn encode_single_tlv_frame(msg_type: crate::protocol::tlv::MessageType, payload: let payload_len = u16::try_from(payload.len()).unwrap_or(u16::MAX); out.extend_from_slice(&payload_len.to_be_bytes()); out.extend_from_slice(payload); - out.freeze() + Ok(out.freeze()) } #[cfg(test)] @@ -615,6 +693,51 @@ mod tests { assert_eq!(frame.as_ref(), &[101, 0, 2, b'o', b'k']); } + #[tokio::test] + async fn should_give_up_quickly_on_a_saturated_queue_watcher() { + // Arrange + // Queue delivers ready-notifications to every watcher SERIALLY, on the + // actor thread, before it replies to the client whose SEND just + // committed - see `mailbox_sink_impl.rs`'s notify loop ahead of + // `route_queue_response`. The default retry budget blocks up to ~177ms + // per saturated consumer (8 yields then escalating sleeps to 100 + // attempts); six saturated watchers alone would exceed + // QUEUE_ACTOR_REPLY_TIMEOUT (1s) even though the write already + // succeeded. A best-effort notification must give up fast instead. + let (tx, _rx) = mpsc::channel(1); + // Fill the channel so every attempt is met with Full. `_rx` is kept + // alive (never read) so the channel stays Full rather than Closed. + tx.try_send(Bytes::from_static(b"occupied")) + .expect("prime the channel to capacity"); + let sink = SessionOutboundSink::new(tx); + let notification = crate::domains::queue::QueueClientNotification::new( + 1, + RouteFamily::new(1), + 7, + Route::new("queue://acme/jobs/watched"), + crate::domains::queue::QueueNotification { + ready_messages: 1, + delayed_messages: 0, + inflight_messages: 0, + }, + ); + + // Act + let started = Instant::now(); + let result = sink.deliver(Envelope::new( + RouteAddress::new(RouteFamily::new(1), Route::new("inbox://session/1")), + notification, + )); + let elapsed = started.elapsed(); + + // Assert + assert!(result.is_err(), "a permanently full channel must fail"); + assert!( + elapsed < Duration::from_millis(20), + "best-effort notification delivery took {elapsed:?}, blocking the queue actor thread far past what six saturated watchers can afford" + ); + } + #[tokio::test] async fn should_return_backpressure_given_full_outbound_channel() { // Arrange @@ -670,4 +793,62 @@ mod tests { let occupied = handle.join().expect("thread joined"); assert_eq!(occupied, Bytes::from_static(b"occupied")); } + + #[test] + fn should_wait_meaningfully_before_giving_up_on_a_full_outbound_channel() { + // Arrange + // Spinning on `yield_now` for every attempt takes microseconds, so a + // frame is abandoned long before a briefly-saturated consumer has any + // chance to drain. The schedule must yield for the first few attempts + // (the genuinely transient case) and then wait in escalating steps. + let schedule = (0..MAX_OUTBOUND_SEND_RETRIES) + .map(outbound_retry_backoff) + .collect::>(); + + // Act + let total_wait: Duration = schedule.iter().flatten().copied().sum(); + let yielded_attempts = schedule.iter().filter(|delay| delay.is_none()).count(); + + // Assert + assert!( + yielded_attempts >= 4, + "the first attempts should stay on a cheap yield, got {yielded_attempts}" + ); + assert!( + total_wait >= Duration::from_millis(50), + "a frame must not be dropped after only {total_wait:?} of waiting" + ); + assert!( + total_wait <= Duration::from_millis(500), + "the wait must stay bounded, got {total_wait:?}" + ); + assert!( + schedule.windows(2).all(|pair| { + pair[0].unwrap_or(Duration::ZERO) <= pair[1].unwrap_or(Duration::ZERO) + }), + "the backoff must be monotonically non-decreasing" + ); + } + + #[test] + fn should_reject_oversized_tlv_frame_instead_of_panicking() { + // Arrange + // A TLV value carries a u16 length. Asserting on that turns any + // aggregate-overflow bug in any domain into a broker panic; the + // schedule LIST response reached 270KB this way. Framing must fail the + // one delivery, not the process. + let payload = vec![0x5a; usize::from(u16::MAX) + 1]; + + // Act + let result = encode_single_tlv_frame(crate::protocol::tlv::MessageType::new(701), &payload); + + // Assert + let Err(error) = result else { + panic!("oversized payload must not be framed"); + }; + assert!( + matches!(error, DeliveryError::InvalidPayload { .. }), + "unexpected error: {error:?}" + ); + } } diff --git a/src/api/runtime_ingress/builder_and_sessions.rs b/src/api/runtime_ingress/builder_and_sessions.rs index efdb21f7..1f2db229 100644 --- a/src/api/runtime_ingress/builder_and_sessions.rs +++ b/src/api/runtime_ingress/builder_and_sessions.rs @@ -24,6 +24,7 @@ impl RuntimeIngress { auth_config: None, auth_claims_config: crate::auth::AuthClaimsConfig::default(), route_family_resolver: crate::auth::RouteFamilyResolverConfig::default(), + connect_diagnostics_budget: Arc::default(), } } diff --git a/src/api/runtime_ingress/domain_frame_dispatcher.rs b/src/api/runtime_ingress/domain_frame_dispatcher.rs index d8aa8575..817ed73e 100644 --- a/src/api/runtime_ingress/domain_frame_dispatcher.rs +++ b/src/api/runtime_ingress/domain_frame_dispatcher.rs @@ -117,6 +117,20 @@ impl DomainFrameDispatcher<'_> { .unauthorized_error_code } + fn backpressure_error_code(domain: DispatchDomain) -> u16 { + crate::api::runtime_ingress::domain_registry::IngressDomainRegistry::descriptor_for_domain( + domain, + ) + .backpressure_error_code + } + + fn indeterminate_error_code(domain: DispatchDomain) -> u16 { + crate::api::runtime_ingress::domain_registry::IngressDomainRegistry::descriptor_for_domain( + domain, + ) + .indeterminate_error_code + } + fn encode_domain_error_body(code: u16, message: &str) -> Bytes { let body = crate::protocol::error_codes::encode_error_body(code, message); Bytes::from(body) @@ -136,7 +150,7 @@ impl DomainFrameDispatcher<'_> { warn!( session_id = session_id, domain = domain.as_str(), - "Ingress: unauthorized response backpressure" + "Ingress: domain error response backpressure" ); IngressDecision::Backpressure } @@ -145,9 +159,9 @@ impl DomainFrameDispatcher<'_> { session_id = session_id, domain = domain.as_str(), error = %error, - "Ingress: unauthorized response delivery failed" + "Ingress: domain error response delivery failed" ); - IngressDecision::Close(format!("unauthorized response delivery failed: {error}")) + IngressDecision::Close(format!("domain error response delivery failed: {error}")) } } } @@ -217,41 +231,80 @@ impl DomainFrameDispatcher<'_> { &self, dispatch: &DomainDispatchRequest<'_>, request_payload: &[u8], + ) -> Result<(), IngressDecision> { + self.send_domain_error_response( + dispatch, + request_payload, + Self::unauthorized_error_code(dispatch.domain), + crate::protocol::error_codes::rpc::ERR_UNAUTHORIZED, + "unauthorized: permission denied", + ) + } + + /// Answer one frame with a domain error, leaving the session intact. + fn send_domain_error_response( + &self, + dispatch: &DomainDispatchRequest<'_>, + request_payload: &[u8], + domain_code: u16, + rpc_submit_code: u16, + message: &'static str, ) -> Result<(), IngressDecision> { if dispatch.domain == DispatchDomain::Rpc && dispatch.msg_type.as_u16() == 302 { return self.send_rpc_submit_error_response( dispatch, request_payload, - crate::protocol::error_codes::rpc::ERR_UNAUTHORIZED, - "unauthorized: permission denied", + rpc_submit_code, + message, ); } - let payload = Self::encode_domain_error_body( - Self::unauthorized_error_code(dispatch.domain), - "unauthorized: permission denied", - ); - let response_ctx = crate::protocol::frame_context::FrameContext::new( + self.send_domain_error_frame( dispatch.session_id, dispatch.channel_id, dispatch.msg_type, - payload, dispatch.route_family, + dispatch.domain, + dispatch.router, + domain_code, + message, + ) + } + + #[allow(clippy::too_many_arguments)] + fn send_domain_error_frame( + &self, + session_id: u64, + channel_id: crate::protocol::frame::ChannelId, + msg_type: crate::protocol::tlv::MessageType, + route_family: crate::runtime::routing::RouteFamily, + domain: DispatchDomain, + router: &crate::runtime::Router, + domain_code: u16, + message: &'static str, + ) -> Result<(), IngressDecision> { + let payload = Self::encode_domain_error_body(domain_code, message); + let response_ctx = crate::protocol::frame_context::FrameContext::new( + session_id, + channel_id, + msg_type, + payload, + route_family, ); let source = crate::runtime::routing::RouteAddress::new( - dispatch.route_family, - dispatch.domain.inbound_route().clone(), + route_family, + domain.inbound_route().clone(), ); let destination = crate::runtime::routing::RouteAddress::new( - dispatch.route_family, - self.cached_session_inbox_route(dispatch.session_id), + route_family, + self.cached_session_inbox_route(session_id), ); let envelope = crate::runtime::envelope::Envelope::from_route(source, destination, response_ctx); - dispatch.router.route(envelope).map_err(|error| { - Self::route_error_response_delivery_failure(dispatch.session_id, dispatch.domain, error) - }) + router + .route(envelope) + .map_err(|error| Self::route_error_response_delivery_failure(session_id, domain, error)) } fn derive_auth_route_for_frame( @@ -346,6 +399,18 @@ impl DomainFrameDispatcher<'_> { ) } + /// Whether the destination was alive but did not answer before its + /// deadline - meaning the command was accepted and may still run. + fn domain_dispatch_timed_out(error: &crate::runtime::router::RouteError) -> bool { + matches!( + error, + crate::runtime::router::RouteError::DeliveryFailed( + _, + crate::runtime::router::DeliveryError::Timeout, + ) + ) + } + fn record_backpressure_retry() { obs::counter_inc(obs::METRIC_INGRESS_DOMAIN_BACKPRESSURE_RETRIES); } @@ -370,23 +435,141 @@ impl DomainFrameDispatcher<'_> { ); } - async fn dispatch_domain_frame( + /// Answer a frame whose domain could not reply in time. + /// + /// The actor is alive but did not answer. That is the client's problem for + /// this one request, not grounds to destroy a multiplexed session along + /// with every other domain's in-flight work on it. + /// + /// The command was already enqueued and may still execute, so this reports + /// an indeterminate outcome rather than a retryable rejection. Closing the + /// session would not make this at-most-once either - the command keeps + /// running and the client reconnects and retries with the same uncertainty + /// - it would only add collateral damage. + #[allow(clippy::too_many_arguments)] + fn answer_indeterminate_dispatch( &self, - dispatch: DomainDispatchRequest<'_>, - ) -> Result<(), IngressDecision> { - let DomainDispatchRequest { + session_id: u64, + channel_id: crate::protocol::frame::ChannelId, + msg_type: crate::protocol::tlv::MessageType, + route_family: crate::runtime::routing::RouteFamily, + domain: DispatchDomain, + router: &crate::runtime::Router, + error: &crate::runtime::router::RouteError, + ) -> IngressDecision { + obs::counter_inc(obs::METRIC_INGRESS_DOMAIN_DISPATCH_TIMEOUTS); + warn!( + session_id = session_id, + domain = domain.as_str(), + error = %error, + "Ingress: domain dispatch timed out; answering with a retryable error" + ); + self.send_domain_error_frame( + session_id, + channel_id, + msg_type, + route_family, + domain, router, + Self::indeterminate_error_code(domain), + "domain timeout: request outcome unknown, do not blindly retry", + ) + .map_or_else(|decision| decision, |()| IngressDecision::Accept) + } + + /// Reject a frame whose domain mailbox stayed full past the retry budget. + /// + /// The command was never enqueued, which makes this the one failure a + /// client can safely re-send. Answering with a rejection frame preserves + /// that: returning `IngressDecision::Backpressure` instead closes the + /// connection at the transport, which turns a clean retryable rejection + /// into an unknown outcome the caller dare not retry. + #[allow(clippy::too_many_arguments)] + fn answer_exhausted_backpressure( + &self, + session_id: u64, + channel_id: crate::protocol::frame::ChannelId, + msg_type: crate::protocol::tlv::MessageType, + route_family: crate::runtime::routing::RouteFamily, + domain: DispatchDomain, + router: &crate::runtime::Router, + retries: u64, + backpressure_started_at: Instant, + ) -> IngressDecision { + Self::record_backpressure_exhausted(backpressure_started_at); + warn!( + session_id = session_id, + domain = domain.as_str(), + retries = retries, + waited_us = Self::elapsed_micros_u64(backpressure_started_at), + "Ingress: domain dispatch backpressure" + ); + self.send_domain_error_frame( session_id, channel_id, + msg_type, route_family, domain, - policy: _, + router, + Self::backpressure_error_code(domain), + "domain at capacity: request was not accepted, retry with backoff", + ) + .map_or_else(|decision| decision, |()| IngressDecision::Accept) + } + + /// Answer a frame whose domain could not be reached at all. + /// + /// A dead actor, a panicked sink, an unroutable domain or a response that + /// cannot be framed are all failures of THIS request. None is a client + /// protocol violation, so none justifies destroying a multiplexed session + /// and every other domain's in-flight work on it. Reported with a + /// non-retryable code, since the command may have partially applied (the + /// actor died holding it) or can never succeed. + #[allow(clippy::too_many_arguments)] + fn answer_unavailable_dispatch( + &self, + session_id: u64, + channel_id: crate::protocol::frame::ChannelId, + msg_type: crate::protocol::tlv::MessageType, + route_family: crate::runtime::routing::RouteFamily, + domain: DispatchDomain, + router: &crate::runtime::Router, + error: &crate::runtime::router::RouteError, + ) -> IngressDecision { + error!( + session_id = session_id, + domain = domain.as_str(), + error = %error, + "Ingress: router.route failed for domain dispatch" + ); + self.send_domain_error_frame( + session_id, + channel_id, msg_type, - payload, - } = dispatch; - let route = domain.inbound_route().clone(); - let addr = crate::runtime::routing::RouteAddress::new(route_family, route); - let dispatch_payload = payload.into_dispatch_bytes(); + route_family, + domain, + router, + Self::indeterminate_error_code(domain), + "domain unavailable: request could not be completed", + ) + .map_or_else(|decision| decision, |()| IngressDecision::Accept) + } + + /// Resolve the destination, reply source, and descriptor for one dispatch. + fn dispatch_addressing( + &self, + session_id: u64, + route_family: crate::runtime::routing::RouteFamily, + domain: DispatchDomain, + ) -> ( + crate::runtime::routing::RouteAddress, + crate::runtime::routing::RouteAddress, + &'static crate::api::runtime_ingress::domain_registry::IngressDomainDescriptor, + ) { + let addr = crate::runtime::routing::RouteAddress::new( + route_family, + domain.inbound_route().clone(), + ); let source = crate::runtime::routing::RouteAddress::new( route_family, self.cached_session_inbox_route(session_id), @@ -395,6 +578,25 @@ impl DomainFrameDispatcher<'_> { crate::api::runtime_ingress::domain_registry::IngressDomainRegistry::descriptor_for_domain( domain, ); + (addr, source, descriptor) + } + + async fn dispatch_domain_frame( + &self, + dispatch: DomainDispatchRequest<'_>, + ) -> Result<(), IngressDecision> { + let DomainDispatchRequest { + router, + session_id, + channel_id, + route_family, + domain, + policy: _, + msg_type, + payload, + } = dispatch; + let (addr, source, descriptor) = self.dispatch_addressing(session_id, route_family, domain); + let dispatch_payload = payload.into_dispatch_bytes(); let backpressure_started_at = Instant::now(); let mut retries = 0_u64; @@ -444,26 +646,38 @@ impl DomainFrameDispatcher<'_> { policy.wait_before_retry().await; } Err(error) if Self::domain_dispatch_backpressured(&error) => { - Self::record_backpressure_exhausted(backpressure_started_at); - warn!( - session_id = session_id, - domain = domain.as_str(), - retries = retries, - waited_us = Self::elapsed_micros_u64(backpressure_started_at), - "Ingress: domain dispatch backpressure" - ); - return Err(IngressDecision::Backpressure); + return Err(self.answer_exhausted_backpressure( + session_id, + channel_id, + msg_type, + route_family, + domain, + router, + retries, + backpressure_started_at, + )); + } + Err(error) if Self::domain_dispatch_timed_out(&error) => { + return Err(self.answer_indeterminate_dispatch( + session_id, + channel_id, + msg_type, + route_family, + domain, + router, + &error, + )); } Err(error) => { - error!( - session_id = session_id, - domain = domain.as_str(), - error = %error, - "Ingress: router.route failed for domain dispatch" - ); - return Err(IngressDecision::Close(format!( - "route delivery failed: {error}" - ))); + return Err(self.answer_unavailable_dispatch( + session_id, + channel_id, + msg_type, + route_family, + domain, + router, + &error, + )); } } } diff --git a/src/api/runtime_ingress/domain_registry.rs b/src/api/runtime_ingress/domain_registry.rs index 645c6989..e856083c 100644 --- a/src/api/runtime_ingress/domain_registry.rs +++ b/src/api/runtime_ingress/domain_registry.rs @@ -13,6 +13,30 @@ pub(crate) use crate::dispatch::DomainEnvelopeBuildRequest; pub(crate) struct IngressDomainDescriptor { pub(super) manifest: &'static crate::runtime::DomainDescriptor, pub(super) unauthorized_error_code: u16, + /// Code returned when the domain did not answer before its deadline. + /// + /// Deliberately NOT a backpressure/"queue full" code. Those mean the + /// request was rejected without being accepted, so a client may safely + /// retry. A deadline expiry means the opposite: the command was already + /// enqueued and may still execute, so the outcome is unknown. Only queue + /// ACK is deduplicated, so an automatic retry of a SEND would enqueue the + /// message twice. + /// + /// Every value here must sit outside `REQ-PROTO-012`'s retryable set + /// (1004, 4005, 5001, 6001, 6002, 6003, 6004, 7010). Those codes tell a + /// compliant SDK the request was never accepted, which is the opposite of + /// what a deadline expiry means. Notably RPC uses its backend code rather + /// than `ERR_RPC_TIMEOUT`, which is documented retryable. + pub(super) indeterminate_error_code: u16, + /// Code returned when the domain mailbox stayed full and the command was + /// never enqueued. + /// + /// The opposite of `indeterminate_error_code`: nothing was accepted, so the + /// client may safely re-send. Every value here must be inside + /// `REQ-PROTO-012`'s retryable set, or a compliant client gives up on a + /// request it could have retried - the response message says "retry with + /// backoff", and a fatal code contradicts it. + pub(super) backpressure_error_code: u16, extract_auth_route: AuthRouteExtractor, build_request_envelope: RequestEnvelopeBuilder, } @@ -105,42 +129,56 @@ static INGRESS_DOMAIN_DESCRIPTORS: [IngressDomainDescriptor; 7] = [ IngressDomainDescriptor { manifest: crate::runtime::DomainKind::Kv.descriptor(), unauthorized_error_code: crate::protocol::error_codes::kv::ERR_UNAUTHORIZED, + backpressure_error_code: crate::protocol::error_codes::kv::ERR_BUSY, + indeterminate_error_code: crate::protocol::error_codes::kv::ERR_BACKEND_ERROR, extract_auth_route: crate::protocol::kv_codec::extract_auth_route, build_request_envelope: crate::dispatch::build_request_envelope, }, IngressDomainDescriptor { manifest: crate::runtime::DomainKind::Queue.descriptor(), unauthorized_error_code: crate::protocol::error_codes::queue::ERR_UNAUTHORIZED, + backpressure_error_code: crate::protocol::error_codes::queue::ERR_QUEUE_FULL, + indeterminate_error_code: crate::protocol::error_codes::queue::ERR_BACKEND_ERROR, extract_auth_route: crate::protocol::queue_codec::extract_auth_route, build_request_envelope: crate::dispatch::build_request_envelope, }, IngressDomainDescriptor { manifest: crate::runtime::DomainKind::Notice.descriptor(), unauthorized_error_code: crate::protocol::error_codes::notice::ERR_UNAUTHORIZED, + backpressure_error_code: crate::protocol::error_codes::notice::ERR_BUSY, + indeterminate_error_code: crate::protocol::error_codes::notice::ERR_BACKEND_ERROR, extract_auth_route: crate::protocol::notice_codec::extract_auth_route, build_request_envelope: crate::dispatch::build_request_envelope, }, IngressDomainDescriptor { manifest: crate::runtime::DomainKind::Stream.descriptor(), unauthorized_error_code: crate::protocol::error_codes::stream::ERR_UNAUTHORIZED, + backpressure_error_code: crate::protocol::error_codes::stream::ERR_BUSY, + indeterminate_error_code: crate::protocol::error_codes::stream::ERR_BACKEND_ERROR, extract_auth_route: crate::protocol::stream_codec::extract_auth_route, build_request_envelope: crate::dispatch::build_request_envelope, }, IngressDomainDescriptor { manifest: crate::runtime::DomainKind::Rpc.descriptor(), unauthorized_error_code: crate::protocol::error_codes::rpc::ERR_UNAUTHORIZED, + backpressure_error_code: crate::protocol::error_codes::rpc::ERR_RPC_BACKPRESSURE, + indeterminate_error_code: crate::protocol::error_codes::rpc::ERR_BACKEND_ERROR, extract_auth_route: crate::protocol::rpc_codec::extract_auth_route, build_request_envelope: crate::dispatch::build_request_envelope, }, IngressDomainDescriptor { manifest: crate::runtime::DomainKind::Lease.descriptor(), unauthorized_error_code: crate::protocol::error_codes::lease::ERR_UNAUTHORIZED, + backpressure_error_code: crate::protocol::error_codes::lease::ERR_QUEUE_FULL, + indeterminate_error_code: crate::protocol::error_codes::lease::ERR_TIMEOUT, extract_auth_route: crate::protocol::lease_codec::extract_auth_route, build_request_envelope: crate::dispatch::build_request_envelope, }, IngressDomainDescriptor { manifest: crate::runtime::DomainKind::Schedule.descriptor(), unauthorized_error_code: crate::protocol::error_codes::schedule::ERR_UNAUTHORIZED, + backpressure_error_code: crate::protocol::error_codes::schedule::ERR_BACKEND_ERROR, + indeterminate_error_code: crate::protocol::error_codes::schedule::ERR_TIMEOUT, extract_auth_route: crate::protocol::schedule_codec::extract_auth_route, build_request_envelope: crate::dispatch::build_request_envelope, }, diff --git a/src/api/runtime_ingress/session_authenticator.rs b/src/api/runtime_ingress/session_authenticator.rs index eabdec15..7000c2e4 100644 --- a/src/api/runtime_ingress/session_authenticator.rs +++ b/src/api/runtime_ingress/session_authenticator.rs @@ -1,6 +1,7 @@ use super::{debug, warn, Bytes, ChannelId, IngressDecision, RuntimeIngress, SessionFrame}; use crate::session::{SessionInfo, SessionPermissions}; use std::sync::Arc; +use std::time::Duration; use tracing::error; pub(super) struct SessionAuthenticator<'a> { @@ -13,7 +14,143 @@ impl RuntimeIngress { } } +/// How long one CONNECT-failure diagnostics budget window lasts. +const CONNECT_DIAGNOSTICS_WINDOW: Duration = Duration::from_secs(1); +/// Full diagnostics emitted per window before the rest are summarized. +const MAX_FULL_CONNECT_DIAGNOSTICS_PER_WINDOW: u32 = 5; + +/// Whether one CONNECT failure may log full diagnostics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ConnectDiagnosticsGrant { + /// Emit the full diagnostic record. + Full, + /// Emit only a terse line; `suppressed_in_window` failures have been + /// summarized this window so far, including this one. + Suppressed { suppressed_in_window: u64 }, +} + +/// Rate limiter for CONNECT-failure diagnostics. +/// +/// The full record costs a SHA-256, a base64 decode, and a `serde_json` parse +/// of an attacker-controlled payload, and expands to several kilobytes of +/// ERROR-level output - all for a peer that has not authenticated. Without a +/// bound, a peer looping CONNECT with a JWT-shaped payload of long claim +/// values can flood the log pipeline. Windowed rather than per-session, since +/// the attacker chooses the session count. +#[derive(Debug, Default)] +struct ConnectDiagnosticsWindow { + started_at_millis: u64, + opened: bool, + emitted: u32, + suppressed: u64, +} + +#[derive(Debug)] +pub(crate) struct ConnectDiagnosticsBudget { + baseline: std::time::Instant, + // The window marker and its counters must move together: publishing a new + // window before resetting its counters lets a concurrent caller increment + // the outgoing counter and then have that increment erased, granting more + // full diagnostics than the bound allows. One lock keeps the whole + // decision atomic, and this is a failure path that is already about to + // log, so the contention cost is irrelevant. + window: std::sync::Mutex, +} + +impl Default for ConnectDiagnosticsBudget { + fn default() -> Self { + Self { + baseline: std::time::Instant::now(), + window: std::sync::Mutex::new(ConnectDiagnosticsWindow::default()), + } + } +} + +impl ConnectDiagnosticsBudget { + /// Take one grant for a failure observed now. + pub(crate) fn acquire_now(&self) -> ConnectDiagnosticsGrant { + self.acquire(u64::try_from(self.baseline.elapsed().as_millis()).unwrap_or(u64::MAX)) + } + + /// Take one grant for a failure observed at `now_millis` (monotonic). + pub(crate) fn acquire(&self, now_millis: u64) -> ConnectDiagnosticsGrant { + let window_millis = u64::try_from(CONNECT_DIAGNOSTICS_WINDOW.as_millis()).unwrap_or(1_000); + let mut window = self + .window + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + + if !window.opened + || now_millis < window.started_at_millis + || now_millis.saturating_sub(window.started_at_millis) >= window_millis + { + *window = ConnectDiagnosticsWindow { + started_at_millis: now_millis, + opened: true, + emitted: 1, + suppressed: 0, + }; + return ConnectDiagnosticsGrant::Full; + } + + if window.emitted < MAX_FULL_CONNECT_DIAGNOSTICS_PER_WINDOW { + window.emitted = window.emitted.saturating_add(1); + return ConnectDiagnosticsGrant::Full; + } + window.suppressed = window.suppressed.saturating_add(1); + ConnectDiagnosticsGrant::Suppressed { + suppressed_in_window: window.suppressed, + } + } +} + impl SessionAuthenticator<'_> { + fn log_connect_failure(&self, session_id: u64, compact: &str, stage: &str, error: &str) { + const MAX_LOGGED_ERROR_CHARS: usize = 512; + + if let ConnectDiagnosticsGrant::Suppressed { + suppressed_in_window, + } = self.ingress.connect_diagnostics_budget.acquire_now() + { + // Terse and cheap: no token hashing, decoding, or claim parsing. + warn!( + session_id, + stage, + suppressed_in_window, + "Ingress: CONNECT authentication failed (diagnostics suppressed)" + ); + return; + } + + let diagnostics = + crate::auth::jwt_failure_diagnostics(compact, &self.ingress.auth_claims_config); + let mut error_characters = error.chars(); + let mut bounded_error = error_characters + .by_ref() + .take(MAX_LOGGED_ERROR_CHARS) + .collect::(); + if error_characters.next().is_some() { + bounded_error.push_str("..."); + } + + error!( + session_id, + stage, + error = ?bounded_error, + jwt_fingerprint = %diagnostics.token_fingerprint, + jwt_algorithm = ?diagnostics.algorithm, + jwt_key_id = ?diagnostics.key_id, + jwt_payload_status = %diagnostics.payload_status, + jwt_issuer = ?diagnostics.issuer, + jwt_audience = ?diagnostics.audience, + jwt_exp = ?diagnostics.expires_at, + jwt_nbf = ?diagnostics.not_before, + jwt_expected_permission_sources = ?diagnostics.expected_permission_sources, + jwt_presented_permission_sources = ?diagnostics.presented_permission_sources, + "Ingress: CONNECT authentication failed" + ); + } + pub(super) async fn authenticate_frame( &self, session_id: u64, @@ -137,10 +274,11 @@ impl SessionAuthenticator<'_> { match self.resolve_authenticated_route_family(&verified.raw_claims) { Ok(route_family) => route_family, Err(error) => { - error!( - session_id = session_id, - error = %error, - "Ingress: CONNECT failed (route family resolution)" + self.log_connect_failure( + session_id, + &compact, + "route_family_resolution", + &error, ); return Err(IngressDecision::Close(format!("connect failed: {error}"))); } @@ -148,11 +286,7 @@ impl SessionAuthenticator<'_> { Ok((verified.permissions, verified.claims, route_family)) } Err(error) => { - error!( - session_id = session_id, - error = %error, - "Ingress: CONNECT failed (verification)" - ); + self.log_connect_failure(session_id, &compact, "jwt_verification", &error); Err(IngressDecision::Close(format!("connect failed: {error}"))) } } diff --git a/src/api/runtime_ingress/session_cleanup_coordinator.rs b/src/api/runtime_ingress/session_cleanup_coordinator.rs index 2f3e5412..e551e2e2 100644 --- a/src/api/runtime_ingress/session_cleanup_coordinator.rs +++ b/src/api/runtime_ingress/session_cleanup_coordinator.rs @@ -7,6 +7,20 @@ use std::time::Duration; const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(10); const MAX_RETRY_DELAY: Duration = Duration::from_secs(1); +/// Give up on a cleanup ticket once it has been pending this long instead of +/// retrying forever. A domain actor that has permanently failed (see +/// `ManagedActor`'s fail-closed supervision) can never accept a cleanup +/// command again, so retrying indefinitely would leave the pending-cleanup +/// gauge and oldest-age metric growing without bound instead of surfacing a +/// terminal failure an operator can act on. +/// +/// This is measured per ticket against its own age, not as an attempt count. +/// The backoff above is worker-global and is reset to its 10ms floor whenever +/// any *other* ticket in the batch succeeds, so under normal session churn an +/// attempt counter does not track elapsed time at all - eight attempts can +/// burn in under 100ms, abandoning the subscriptions and inflight leases of a +/// session whose actor was merely busy. +const MAX_CLEANUP_RETRY_WINDOW: Duration = Duration::from_millis(2_300); pub(super) struct SessionCleanupCoordinator<'a> { ingress: &'a RuntimeIngress, @@ -195,9 +209,26 @@ async fn run_cleanup_worker( crate::observability::counter_inc(obs::METRIC_SESSION_CLEANUP_SUCCESSES); made_progress = true; } else if let Some(mut current) = pending.get_mut(&session_id) { - current.pending_domains = failed_domains; - current.attempts = ticket.attempts.saturating_add(1); - crate::observability::counter_inc(obs::METRIC_SESSION_CLEANUP_RETRIES); + let attempts = ticket.attempts.saturating_add(1); + if ticket.created_at.elapsed() >= MAX_CLEANUP_RETRY_WINDOW { + drop(current); + pending.remove(&session_id); + crate::observability::counter_inc( + obs::METRIC_SESSION_CLEANUP_PERMANENT_FAILURES, + ); + tracing::error!( + session_id = session_id, + attempts, + pending_ms = ticket.created_at.elapsed().as_millis(), + pending_domains = ?failed_domains, + "Ingress: session cleanup permanently failed after exhausting \ + its retry window" + ); + } else { + current.pending_domains = failed_domains; + current.attempts = attempts; + crate::observability::counter_inc(obs::METRIC_SESSION_CLEANUP_RETRIES); + } } } update_cleanup_gauges(&pending); diff --git a/src/api/runtime_ingress/tests/authorization_routes.rs b/src/api/runtime_ingress/tests/authorization_routes.rs index 7beadeda..7e7c8e0e 100644 --- a/src/api/runtime_ingress/tests/authorization_routes.rs +++ b/src/api/runtime_ingress/tests/authorization_routes.rs @@ -683,6 +683,30 @@ async fn should_authorize_kv_begin_by_mode_while_keeping_tx_ops_session_owned_at .await; let put_dispatch = receive_frame(&domain_mailbox, "kv put dispatch"); + let commit_frame = crate::benchkit::build_kv_commit(7, route); + let (_, commit_payload) = crate::benchkit::extract_single_tlv_field(&commit_frame); + let commit_decision = ingress + .on_frame( + session_id, + ChannelId::Pub, + MessageType::new(101), + commit_payload, + ) + .await; + let commit_dispatch = receive_frame(&domain_mailbox, "kv commit dispatch"); + + let rollback_frame = crate::benchkit::build_kv_rollback(7, route); + let (_, rollback_payload) = crate::benchkit::extract_single_tlv_field(&rollback_frame); + let rollback_decision = ingress + .on_frame( + session_id, + ChannelId::Pub, + MessageType::new(102), + rollback_payload, + ) + .await; + let rollback_dispatch = receive_frame(&domain_mailbox, "kv rollback dispatch"); + // Assert assert_eq!(read_only_decision, IngressDecision::Accept); assert_eq!(read_only_frame.msg_type, MessageType::new(100)); @@ -693,6 +717,73 @@ async fn should_authorize_kv_begin_by_mode_while_keeping_tx_ops_session_owned_at ); assert_eq!(put_decision, IngressDecision::Accept); assert_eq!(put_dispatch.msg_type, MessageType::new(104)); + assert_eq!(commit_decision, IngressDecision::Accept); + assert_eq!(commit_dispatch.msg_type, MessageType::new(101)); + assert_eq!(rollback_decision, IngressDecision::Accept); + assert_eq!(rollback_dispatch.msg_type, MessageType::new(102)); +} + +#[tokio::test] +async fn should_require_exact_kv_realm_area_and_resource_at_ingress() { + // Arrange + let family = RouteFamily::new(1); + let session_id = 622; + let exact_route = "kv://acme/app/users"; + let router = Arc::new(crate::runtime::Router::new()); + let domain_mailbox = Arc::new(Mailbox::new(8)); + let inbox_mailbox = Arc::new(Mailbox::new(8)); + router.register_domain_pattern("kv", domain_mailbox.clone()); + router.register( + RouteAddress::new(family, Route::new("inbox://session/622")), + inbox_mailbox.clone(), + ); + let ingress = runtime_ingress_with_jwks_auth().with_router(router); + let session = make_authenticated_session_info( + session_id, + TransportKind::Tcp, + family, + &["kv://acme/app/users#write"], + ); + ingress.on_open(session).await.unwrap(); + + // Act + let exact_begin = crate::benchkit::build_kv_begin(exact_route, 1, 0); + let (_, exact_payload) = crate::benchkit::extract_single_tlv_field(&exact_begin); + let exact_decision = ingress + .on_frame( + session_id, + ChannelId::Pub, + MessageType::new(100), + exact_payload, + ) + .await; + let exact_dispatch = receive_frame(&domain_mailbox, "exact KV begin dispatch"); + + let mismatched_routes = [ + "kv://other/app/users", + "kv://acme/other/users", + "kv://acme/app/other", + ]; + let mut denial_codes = Vec::new(); + for route in mismatched_routes { + let begin = crate::benchkit::build_kv_begin(route, 1, 0); + let (_, payload) = crate::benchkit::extract_single_tlv_field(&begin); + let decision = ingress + .on_frame(session_id, ChannelId::Pub, MessageType::new(100), payload) + .await; + assert_eq!(decision, IngressDecision::Accept); + let denial = receive_frame(&inbox_mailbox, "mismatched KV begin denial"); + denial_codes.push(decode_domain_error_code(denial.payload.as_ref())); + } + + // Assert + assert_eq!(exact_decision, IngressDecision::Accept); + assert_eq!(exact_dispatch.msg_type, MessageType::new(100)); + assert_eq!( + denial_codes, + vec![crate::protocol::error_codes::kv::ERR_UNAUTHORIZED; 3] + ); + assert!(domain_mailbox.receiver().try_recv().is_err()); } #[tokio::test] diff --git a/src/api/runtime_ingress/tests/connect_auth_claims.rs b/src/api/runtime_ingress/tests/connect_auth_claims.rs index 1895a3ec..09abd416 100644 --- a/src/api/runtime_ingress/tests/connect_auth_claims.rs +++ b/src/api/runtime_ingress/tests/connect_auth_claims.rs @@ -601,6 +601,18 @@ fn should_allow_stream_followup_after_begin_without_global_stream_write_permissi assert_eq!(ingress.session_count(), 1); } +struct BackpressureCapturingInbox; + +impl MailboxSink for BackpressureCapturingInbox { + fn deliver(&self, _envelope: Envelope) -> Result<(), DeliveryError> { + Ok(()) + } + + fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.deliver(envelope) + } +} + #[test] fn should_surface_router_backpressure_in_ingress_decision() { // Arrange @@ -609,6 +621,14 @@ fn should_surface_router_backpressure_in_ingress_decision() { let router = Arc::new(crate::runtime::Router::new()); router.register_domain_pattern("kv", Arc::new(BackpressuredSink)); + // A live session has an inbox; the rejection frame is written to it. + router.register( + crate::runtime::routing::RouteAddress::new( + RouteFamily::new(1), + crate::runtime::routing::Route::new("inbox://session/90"), + ), + Arc::new(BackpressureCapturingInbox) as Arc, + ); let ingress = RuntimeIngress::new(false).with_router(router); let session = make_session_info(90, TransportKind::Tcp); @@ -630,7 +650,10 @@ fn should_surface_router_backpressure_in_ingress_decision() { .await; // Assert - assert_eq!(decision, IngressDecision::Backpressure); + // The command was never enqueued, so the frame is rejected and the + // session kept; closing would strip the client of the one signal that + // makes a retry safe. + assert_eq!(decision, IngressDecision::Accept); assert!( metrics.counter_get(obs::METRIC_ROUTER_BACKPRESSURE) > backpressure_before, "expected router backpressure metric to increase" @@ -799,3 +822,98 @@ fn should_reject_qualified_route_with_a_different_domain_scheme() { .unwrap_err() .contains("notice message route must use notice://")); } + +#[test] +fn should_bound_full_connect_failure_diagnostics_per_window() { + // Arrange + // Full CONNECT-failure diagnostics are large (a SHA-256, a JSON + // parse of the attacker-supplied payload, and up to six bounded claim + // views) and are emitted for entirely unauthenticated peers, so an + // attacker looping CONNECT must not be able to drive unbounded + // ERROR-level log volume. + let budget = + crate::api::runtime_ingress::session_authenticator::ConnectDiagnosticsBudget::default(); + + // Act + let first_window = (0..50).map(|_| budget.acquire(1_000)).collect::>(); + let next_window = budget.acquire(2_000); + + // Assert + let full_count = first_window + .iter() + .filter(|grant| { + matches!( + grant, + crate::api::runtime_ingress::session_authenticator::ConnectDiagnosticsGrant::Full + ) + }) + .count(); + assert!( + (1..=5).contains(&full_count), + "expected between 1 and 5 full diagnostics per window, got {full_count}" + ); + assert!( + matches!( + first_window.last(), + Some(crate::api::runtime_ingress::session_authenticator::ConnectDiagnosticsGrant::Suppressed { .. }) + ), + "failures past the window budget must be suppressed" + ); + assert!( + matches!( + next_window, + crate::api::runtime_ingress::session_authenticator::ConnectDiagnosticsGrant::Full + ), + "a new window must allow full diagnostics again" + ); + // The suppressed failures are still accounted for, not silently dropped. + let suppressed = first_window + .iter() + .filter(|grant| matches!(grant, crate::api::runtime_ingress::session_authenticator::ConnectDiagnosticsGrant::Suppressed { .. })) + .count(); + assert_eq!(full_count + suppressed, 50); +} + +#[test] +fn should_hold_connect_diagnostics_bound_under_concurrent_window_rollover() { + // Arrange + // An attacker chooses the concurrency, so the per-window bound + // must hold when many CONNECT failures land on a window boundary at once + // - not just when they arrive one at a time. + use crate::api::runtime_ingress::session_authenticator::{ + ConnectDiagnosticsBudget, ConnectDiagnosticsGrant, + }; + + for round in 0..200_u64 { + let budget = Arc::new(ConnectDiagnosticsBudget::default()); + let full_grants = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let barrier = Arc::new(std::sync::Barrier::new(16)); + // Every thread sees the same stale-window boundary instant. + let now_millis = 5_000 + round; + + // Act + let handles = (0..16) + .map(|_| { + let budget = budget.clone(); + let full_grants = full_grants.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + if matches!(budget.acquire(now_millis), ConnectDiagnosticsGrant::Full) { + full_grants.fetch_add(1, Ordering::Relaxed); + } + }) + }) + .collect::>(); + for handle in handles { + handle.join().unwrap(); + } + + // Assert + let granted = full_grants.load(Ordering::Relaxed); + assert!( + granted <= 5, + "round {round}: {granted} full diagnostics granted in one window, bound is 5" + ); + } +} diff --git a/src/api/runtime_ingress/tests/domain_backpressure.rs b/src/api/runtime_ingress/tests/domain_backpressure.rs index 17245539..08b3617a 100644 --- a/src/api/runtime_ingress/tests/domain_backpressure.rs +++ b/src/api/runtime_ingress/tests/domain_backpressure.rs @@ -177,6 +177,44 @@ fn should_absorb_transient_domain_mailbox_backpressure_for_each_domain() { } } +struct CapturingInboxSink { + frames: Arc>>, +} + +impl MailboxSink for CapturingInboxSink { + fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { + let frame = envelope + .payload::() + .expect("client frame payload") + .clone(); + self.frames.lock().unwrap().push(frame); + Ok(()) + } + + fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.deliver(envelope) + } +} + +/// `REQ-PROTO-012`'s retryable set. A timeout must never answer with one of +/// these: they tell a compliant client the request was never accepted. A +/// backpressure rejection must always answer with one, for the same reason. +const DOCUMENTED_RETRYABLE_CODES: [u32; 12] = [ + 1004, 1014, 2014, 3006, 4005, 5001, 5007, 6001, 6002, 6003, 6004, 7010, +]; + +struct AlwaysTimingOutSink; + +impl MailboxSink for AlwaysTimingOutSink { + fn deliver(&self, _envelope: Envelope) -> Result<(), DeliveryError> { + Err(DeliveryError::Timeout) + } + + fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.deliver(envelope) + } +} + #[test] fn should_surface_sustained_high_lane_domain_mailbox_backpressure_for_each_domain() { // Arrange @@ -185,8 +223,17 @@ fn should_surface_sustained_high_lane_domain_mailbox_backpressure_for_each_domai for (index, case) in domain_ingress_cases().into_iter().enumerate() { let router = Arc::new(crate::runtime::Router::new()); router.register_domain_pattern(case.domain, Arc::new(AlwaysHighLaneBackpressuredSink)); - let ingress = RuntimeIngress::new(false).with_router(router); let session_id = 3_000 + u64::try_from(index).unwrap(); + router.register( + crate::runtime::routing::RouteAddress::new( + RouteFamily::new(1), + crate::runtime::routing::Route::new(format!("inbox://session/{session_id}")), + ), + Arc::new(CapturingInboxSink { + frames: Arc::new(Mutex::new(Vec::new())), + }) as Arc, + ); + let ingress = RuntimeIngress::new(false).with_router(router); let session = make_session_info(session_id, TransportKind::Tcp); // Act @@ -205,23 +252,117 @@ fn should_surface_sustained_high_lane_domain_mailbox_backpressure_for_each_domai // Assert assert_eq!( decision, - IngressDecision::Backpressure, - "sustained high-lane backpressure should remain visible for {}", + IngressDecision::Accept, + "sustained high-lane backpressure should reject the frame, not the session, for {}", + case.domain + ); + } +} + +#[test] +fn should_not_close_session_when_a_domain_command_times_out() { + // Arrange + // A domain actor that is merely slow must not cost the client its whole + // connection. The WebSocket is multiplexed, so closing it destroys every + // other domain's in-flight work on that session too - which is how one + // saturated Queue took down unrelated KV, Stream and Schedule traffic. + // + // The frame is answered with an indeterminate-outcome code, never a + // "queue full"/backpressure code: the command was already enqueued and may + // still execute, so telling the client it was rejected would invite a + // duplicate. + let rt = tokio::runtime::Runtime::new().unwrap(); + + for (index, case) in domain_ingress_cases().into_iter().enumerate() { + let router = Arc::new(crate::runtime::Router::new()); + router.register_domain_pattern(case.domain, Arc::new(AlwaysTimingOutSink)); + let session_id = 7_000 + u64::try_from(index).unwrap(); + // A real session has an inbox; the retryable error frame is written to + // it instead of the session being torn down. + let client_frames = Arc::new(Mutex::new(Vec::::new())); + router.register( + crate::runtime::routing::RouteAddress::new( + RouteFamily::new(1), + crate::runtime::routing::Route::new(format!("inbox://session/{session_id}")), + ), + Arc::new(CapturingInboxSink { + frames: client_frames.clone(), + }) as Arc, + ); + let ingress = RuntimeIngress::new(false).with_router(router); + let session = make_session_info(session_id, TransportKind::Tcp); + + // Act + let decision = rt.block_on(async { + ingress.on_open(session).await.unwrap(); + ingress + .on_frame( + session_id, + case.channel_id, + crate::protocol::tlv::MessageType::new(case.msg_type), + case.payload, + ) + .await + }); + + // Assert + assert!( + !matches!(decision, IngressDecision::Close(_)), + "a slow {} actor must not close the session, got {decision:?}", + case.domain + ); + let frames = client_frames.lock().unwrap(); + assert_eq!( + frames.len(), + 1, + "{} should answer the one frame with an error, got {frames:?}", + case.domain + ); + // Error body: [u8 flag][u32 code][string message]. + let body = &frames[0].payload; + assert_eq!(body[0], 1, "{} should send an error body", case.domain); + let code = u32::from_be_bytes([body[1], body[2], body[3], body[4]]); + // A timed-out command was already enqueued and may still run, so the + // code must not be one `REQ-PROTO-012` classifies as retryable. Those + // tell a compliant SDK the request was never accepted, and its + // `IsRetryable` helper (REQ-ERR-006) erases any prose caveat - so the + // client re-sends and duplicates the side effect. Only queue ACK is + // deduplicated. + assert!( + !DOCUMENTED_RETRYABLE_CODES.contains(&code), + "{} answered a timeout with retryable code {code}; a compliant client \ + would re-send a command that may already have applied", case.domain ); } } #[test] -fn should_surface_sustained_domain_mailbox_backpressure_for_each_domain() { +fn should_answer_sustained_mailbox_backpressure_without_killing_the_session() { // Arrange + // A full mailbox means the command was never enqueued, which is the one + // failure a client can safely retry. Closing the connection throws that + // information away: the caller cannot tell a rejected request from one that + // may have applied, so it must stop rather than risk a duplicate. The + // 2ms retry budget is far shorter than a saturated actor takes to drain, + // so this is reached under ordinary load. let rt = tokio::runtime::Runtime::new().unwrap(); for (index, case) in domain_ingress_cases().into_iter().enumerate() { let router = Arc::new(crate::runtime::Router::new()); router.register_domain_pattern(case.domain, Arc::new(AlwaysBackpressuredSink)); + let session_id = 8_000 + u64::try_from(index).unwrap(); + let client_frames = Arc::new(Mutex::new(Vec::::new())); + router.register( + crate::runtime::routing::RouteAddress::new( + RouteFamily::new(1), + crate::runtime::routing::Route::new(format!("inbox://session/{session_id}")), + ), + Arc::new(CapturingInboxSink { + frames: client_frames.clone(), + }) as Arc, + ); let ingress = RuntimeIngress::new(false).with_router(router); - let session_id = 2_000 + u64::try_from(index).unwrap(); let session = make_session_info(session_id, TransportKind::Tcp); // Act @@ -238,11 +379,154 @@ fn should_surface_sustained_domain_mailbox_backpressure_for_each_domain() { }); // Assert + // `Backpressure` is not good enough: the transport turns it into a + // close (see `should_treat_websocket_backpressure_as_terminal_session_error`). assert_eq!( decision, - IngressDecision::Backpressure, - "sustained backpressure should remain visible for {}", + IngressDecision::Accept, + "{} should answer the frame and keep the session", + case.domain + ); + let frames = client_frames.lock().unwrap(); + assert_eq!( + frames.len(), + 1, + "{} should answer with a rejection frame, got {frames:?}", case.domain ); + // The mirror of the timeout guard. Nothing was enqueued, and the + // message tells the client to retry with backoff - so the code must be + // one `REQ-PROTO-012` classifies as retryable. A fatal code here makes + // a compliant client give up on a request it could safely re-send. + let body = &frames[0].payload; + assert_eq!(body[0], 1, "{} should send an error body", case.domain); + let code = u32::from_be_bytes([body[1], body[2], body[3], body[4]]); + assert!( + DOCUMENTED_RETRYABLE_CODES.contains(&code), + "{} rejected a never-enqueued request with fatal code {code}; a compliant \ + client will not retry", + case.domain + ); + } +} + +struct FixedErrorSink(DeliveryError); + +impl MailboxSink for FixedErrorSink { + fn deliver(&self, _envelope: Envelope) -> Result<(), DeliveryError> { + Err(self.0.clone()) + } + + fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.deliver(envelope) + } +} + +#[test] +fn should_answer_terminal_delivery_failures_without_killing_the_session() { + // Arrange + // None of these is a client protocol violation, so none of them justifies + // destroying a multiplexed session: a dead queue actor must not take + // unrelated KV/Stream/RPC work with it, and an unframable response is a + // server-side bug the client should not pay for with its connection. + let rt = tokio::runtime::Runtime::new().unwrap(); + let failures = [ + DeliveryError::ActorStopped, + DeliveryError::SinkPanicked, + DeliveryError::InvalidPayload { + len: 70_000, + max: 65_535, + }, + ]; + + for (index, failure) in failures.into_iter().enumerate() { + let case = domain_ingress_cases() + .into_iter() + .next() + .expect("at least one domain case"); + let router = Arc::new(crate::runtime::Router::new()); + router.register_domain_pattern(case.domain, Arc::new(FixedErrorSink(failure.clone()))); + let session_id = 9_500 + u64::try_from(index).unwrap(); + let client_frames = Arc::new(Mutex::new(Vec::::new())); + router.register( + crate::runtime::routing::RouteAddress::new( + RouteFamily::new(1), + crate::runtime::routing::Route::new(format!("inbox://session/{session_id}")), + ), + Arc::new(CapturingInboxSink { + frames: client_frames.clone(), + }) as Arc, + ); + let ingress = RuntimeIngress::new(false).with_router(router); + let session = make_session_info(session_id, TransportKind::Tcp); + + // Act + let decision = rt.block_on(async { + ingress.on_open(session).await.unwrap(); + ingress + .on_frame( + session_id, + case.channel_id, + crate::protocol::tlv::MessageType::new(case.msg_type), + case.payload, + ) + .await + }); + + // Assert + assert_eq!( + decision, + IngressDecision::Accept, + "{failure:?} should be answered on the channel, not close the session" + ); + assert_eq!( + client_frames.lock().unwrap().len(), + 1, + "{failure:?} should produce one error frame" + ); } } + +#[test] +fn should_answer_unroutable_domain_frame_without_killing_the_session() { + // Arrange + // No sink is registered for the domain, so the router cannot find a route. + // That is a permanent condition for this request but says nothing about the + // session's other channels. + let rt = tokio::runtime::Runtime::new().unwrap(); + let case = domain_ingress_cases() + .into_iter() + .next() + .expect("at least one domain case"); + let router = Arc::new(crate::runtime::Router::new()); + let session_id = 9_600; + let client_frames = Arc::new(Mutex::new(Vec::::new())); + router.register( + crate::runtime::routing::RouteAddress::new( + RouteFamily::new(1), + crate::runtime::routing::Route::new(format!("inbox://session/{session_id}")), + ), + Arc::new(CapturingInboxSink { + frames: client_frames.clone(), + }) as Arc, + ); + let ingress = RuntimeIngress::new(false).with_router(router); + let session = make_session_info(session_id, TransportKind::Tcp); + + // Act + let decision = rt.block_on(async { + ingress.on_open(session).await.unwrap(); + ingress + .on_frame( + session_id, + case.channel_id, + crate::protocol::tlv::MessageType::new(case.msg_type), + case.payload, + ) + .await + }); + + // Assert + assert_eq!(decision, IngressDecision::Accept); + assert_eq!(client_frames.lock().unwrap().len(), 1); +} diff --git a/src/api/runtime_ingress/tests/session_lifecycle_and_cleanup.rs b/src/api/runtime_ingress/tests/session_lifecycle_and_cleanup.rs index 8470dfb0..9a159e6e 100644 --- a/src/api/runtime_ingress/tests/session_lifecycle_and_cleanup.rs +++ b/src/api/runtime_ingress/tests/session_lifecycle_and_cleanup.rs @@ -495,6 +495,73 @@ pub(super) fn should_call_event_handler() { assert_eq!(event_count.load(Ordering::SeqCst), 3); } +/// Distinguishes which mailbox lane a delivery arrived on. +#[derive(Default)] +pub(super) struct LaneTrackingSink { + normal_lane_sessions: Mutex>, + high_priority_sessions: Mutex>, +} + +impl MailboxSink for LaneTrackingSink { + fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { + let cleanup = envelope + .payload::() + .expect("cleanup payload"); + self.normal_lane_sessions + .lock() + .unwrap() + .push(cleanup.session_id); + Ok(()) + } + + fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { + let cleanup = envelope + .payload::() + .expect("cleanup payload"); + self.high_priority_sessions + .lock() + .unwrap() + .push(cleanup.session_id); + Ok(()) + } +} + +#[test] +pub(super) fn should_dispatch_session_cleanup_on_the_control_lane() { + // Arrange + // Cleanup is control-plane work (architecture.md: "A separate bounded + // control lane prevents control work from being hidden behind normal-lane + // pressure"). A busy Queue actor can hold 16,384 client messages ahead of + // anything on the normal lane; a cleanup command enqueued there can sit + // long past the coordinator's 2.3s give-up window, leaving a disconnected + // session's watches and reservations alive and inviting a duplicate + // cleanup ticket for the same session. + let router = crate::runtime::Router::new(); + let route_family = crate::runtime::routing::RouteFamily::new(11); + let session_id = 77; + let sink = Arc::new(LaneTrackingSink::default()); + router.register_domain_pattern(DispatchDomain::Queue.as_str(), sink.clone()); + + // Act + let _ = dispatch_session_cleanup_for_domains( + &router, + route_family, + session_id, + &[DispatchDomain::Queue], + ); + + // Assert + assert_eq!( + sink.high_priority_sessions.lock().unwrap().as_slice(), + &[session_id], + "session cleanup must be delivered on the control lane, not the normal one" + ); + assert!( + sink.normal_lane_sessions.lock().unwrap().is_empty(), + "session cleanup must not compete with client traffic on the normal lane" + ); +} + #[test] pub(super) fn should_dispatch_session_cleanup_to_all_registered_domains() { // Arrange @@ -683,6 +750,59 @@ async fn should_retry_pending_session_cleanup_without_later_traffic() { assert_eq!(queue_sink.recorded_sessions(), vec![session_id]); } +#[tokio::test] +async fn should_give_up_and_stop_retrying_session_cleanup_that_can_never_succeed() { + // Arrange: never register Queue's sink, so its cleanup can never + // succeed. Without a give-up threshold, the retry worker would keep + // this ticket pending forever (capped-but-endless exponential backoff), + // so the pending gauge would never return to zero for a genuinely dead + // domain actor. + let collector = crate::observability::metrics(); + let router = Arc::new(crate::runtime::Router::new()); + let admin_read_model = AdminReadModel::new(); + let ingress = make_cleanup_ingress(router.clone(), admin_read_model); + let session_id = 90; + let mut session = make_session_info(session_id, TransportKind::Tcp); + session.route_family = RouteFamily::new(90); + + for domain in DispatchDomain::SESSION_CLEANUP_ORDER { + if domain == DispatchDomain::Queue { + continue; + } + let sink = Arc::new(CleanupTrackingSink::default()); + router.register_domain_pattern(domain.as_str(), sink); + } + + ingress.on_open(session).await.unwrap(); + ingress.on_close(session_id, CloseReason::ClientClose).await; + assert!(ingress.pending_session_cleanups.contains_key(&session_id)); + let permanent_failures_before = + collector.counter_get(obs::METRIC_SESSION_CLEANUP_PERMANENT_FAILURES); + + // Act: Queue's sink is intentionally never registered, so this ticket + // can never succeed - the worker must eventually give up. + tokio::time::timeout(Duration::from_secs(10), async { + while ingress.pending_session_cleanups.contains_key(&session_id) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("cleanup worker should give up instead of retrying forever"); + + // Assert + assert!(!ingress.pending_session_cleanups.contains_key(&session_id)); + assert!( + collector.counter_get(obs::METRIC_SESSION_CLEANUP_PERMANENT_FAILURES) + > permanent_failures_before, + "expected a permanent-failure metric increment" + ); + assert_eq!( + collector.gauge_get(obs::METRIC_SESSION_CLEANUP_PENDING), + 0, + "pending gauge should return to zero after giving up" + ); +} + #[tokio::test] async fn should_cleanup_real_notice_domain_subscription_on_close() { // Arrange @@ -867,3 +987,113 @@ async fn should_cleanup_real_queue_inflight_on_close() { assert_queue_cleanup_admin_state(&admin_read_model, next_worker_session_id); } + +/// Fails cleanup forever for one session, and fails every other session +/// exactly once before succeeding - so the retry worker keeps observing +/// progress on other tickets while the stuck one never advances. +struct StickySessionFailureSink { + stuck_session_id: u64, + seen_once: Mutex>, +} + +impl StickySessionFailureSink { + fn new(stuck_session_id: u64) -> Self { + Self { + stuck_session_id, + seen_once: Mutex::new(std::collections::HashSet::new()), + } + } +} + +impl MailboxSink for StickySessionFailureSink { + fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { + let cleanup = envelope + .payload::() + .expect("cleanup payload"); + if cleanup.session_id == self.stuck_session_id { + // A merely busy actor, not a dead one. + return Err(DeliveryError::Timeout); + } + if self.seen_once.lock().unwrap().insert(cleanup.session_id) { + return Err(DeliveryError::Timeout); + } + Ok(()) + } + + fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.deliver(envelope) + } +} + +#[tokio::test] +async fn should_retry_stuck_cleanup_for_full_window_while_other_tickets_progress() { + // Arrange + // The give-up threshold is documented as ~2.3s of retrying, + // derived from an exponential backoff. That backoff is worker-global and + // is reset to its 10ms floor whenever *any other* ticket succeeds, so + // under normal session churn a stuck ticket must not be abandoned in a + // small fraction of the intended window. + let router = Arc::new(crate::runtime::Router::new()); + let admin_read_model = AdminReadModel::new(); + let ingress = Arc::new(make_cleanup_ingress(router.clone(), admin_read_model)); + let stuck_session_id = 9_100; + + for domain in DispatchDomain::SESSION_CLEANUP_ORDER { + if domain == DispatchDomain::Queue { + continue; + } + router.register_domain_pattern(domain.as_str(), Arc::new(CleanupTrackingSink::default())); + } + router.register_domain_pattern( + DispatchDomain::Queue.as_str(), + Arc::new(StickySessionFailureSink::new(stuck_session_id)), + ); + + let mut stuck = make_session_info(stuck_session_id, TransportKind::Tcp); + stuck.route_family = RouteFamily::new(91); + ingress.on_open(stuck).await.unwrap(); + ingress + .on_close(stuck_session_id, CloseReason::ClientClose) + .await; + assert!(ingress + .pending_session_cleanups + .contains_key(&stuck_session_id)); + + // Act + // Keep other tickets flowing through the worker so `made_progress` + // resets the shared backoff on essentially every pass. + let churn_ingress = ingress.clone(); + let churn = tokio::spawn(async move { + for index in 0..300_u64 { + let session_id = 9_200 + index; + let mut session = make_session_info(session_id, TransportKind::Tcp); + session.route_family = RouteFamily::new(91); + churn_ingress.on_open(session).await.unwrap(); + churn_ingress + .on_close(session_id, CloseReason::ClientClose) + .await; + tokio::time::sleep(Duration::from_millis(10)).await; + } + }); + + let started = std::time::Instant::now(); + tokio::time::timeout(Duration::from_secs(15), async { + while ingress + .pending_session_cleanups + .contains_key(&stuck_session_id) + { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("stuck cleanup ticket should eventually be given up on"); + let elapsed = started.elapsed(); + churn.abort(); + + // Assert + assert!( + elapsed >= Duration::from_secs(1), + "stuck ticket abandoned after only {elapsed:?}; concurrent progress on other \ + tickets must not collapse its retry window" + ); +} diff --git a/src/api/runtime_ingress/types_and_helpers.rs b/src/api/runtime_ingress/types_and_helpers.rs index a4c1143c..002460bd 100644 --- a/src/api/runtime_ingress/types_and_helpers.rs +++ b/src/api/runtime_ingress/types_and_helpers.rs @@ -46,7 +46,13 @@ pub(super) fn dispatch_session_cleanup_for_domains( crate::runtime::routing::RouteAddress::new(route_family, domain.cleanup_route()); let cleanup_envelope = crate::runtime::Envelope::new(cleanup_addr, cleanup.clone()); - if let Err(error) = router.route(cleanup_envelope) { + // Control-plane work: a busy Queue actor can hold up to 16,384 + // client messages ahead of anything on the normal lane, and the + // ingress coordinator gives up on a cleanup ticket after 2.3s. Cleanup + // must therefore ride the bounded control lane so it is never hidden + // behind normal-lane pressure (see architecture.md's Actor Mailbox + // section). + if let Err(error) = router.route_high_priority(cleanup_envelope) { warn!( session_id = session_id, route_family = route_family.id(), @@ -349,4 +355,7 @@ pub struct RuntimeIngress { pub(super) auth_claims_config: crate::auth::AuthClaimsConfig, /// Broker-local route-family resolver for verified identity claims. pub(super) route_family_resolver: crate::auth::RouteFamilyResolverConfig, + /// Bounds ERROR-level diagnostics emitted for unauthenticated CONNECTs. + pub(super) connect_diagnostics_budget: + Arc, } diff --git a/src/auth/claims/claims_tests.rs b/src/auth/claims/claims_tests.rs index 8eda82b4..394c23f2 100644 --- a/src/auth/claims/claims_tests.rs +++ b/src/auth/claims/claims_tests.rs @@ -846,3 +846,87 @@ fn should_support_okta_custom_permissions_shape() { ); assert_eq!(route_family, 5); } + +fn claims_from(payload: &serde_json::Value) -> crate::auth::RawClaims { + let b64 = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(payload.to_string()); + parse_jwt_noverify(&format!("{}.{}.{}", "{}", b64, "sig")).expect("parse jwt") +} + +#[test] +fn should_cascade_past_an_empty_permissions_claim_to_the_configured_claim() { + // Arrange + // Auth0 emits `permissions: []` whenever RBAC is enabled but no permissions + // are assigned for that API, even when the real grants live in a configured + // custom claim. Treating the empty array as a found source strands the + // token with zero rights and never consults the configured claim. + let claims = claims_from(&serde_json::json!({ + "iss": "https://idp.example/", + "aud": "fitz-broker", + "sub": "user:42", + "exp": 9_999_999_999_u64, + "tid": "acme-prod", + "permissions": [], + "fitz://permissions": ["queue://prod/jobs/**#write"] + })); + + // Act + let perms = claims + .normalized_permissions(None, Some("fitz://permissions"), DEFAULT_ROLE_CLAIM) + .expect("configured claim should be used"); + + // Assert + assert_eq!(perms.len(), 1); + assert_eq!(perms[0].raw, "queue://prod/jobs/**#write"); +} + +#[test] +fn should_cascade_past_an_empty_custom_claim_to_top_level_permissions() { + // Arrange + // The custom claim sits above `permissions`, so an empty one short-circuits + // the whole cascade including sources that do carry grants. + let claims = claims_from(&serde_json::json!({ + "iss": "https://idp.example/", + "aud": "fitz-broker", + "sub": "user:42", + "exp": 9_999_999_999_u64, + "tid": "acme-prod", + "https://acme.example/fitz": { "permissions": [] }, + "permissions": ["stream://prod/events/**#read"] + })); + + // Act + let perms = claims + .normalized_permissions(Some("https://acme.example/fitz"), None, DEFAULT_ROLE_CLAIM) + .expect("top-level permissions should be used"); + + // Assert + assert_eq!(perms.len(), 1); + assert_eq!(perms[0].raw, "stream://prod/events/**#read"); +} + +#[test] +fn should_reject_a_token_whose_every_permission_source_is_empty() { + // Arrange + // Cascading past empties must not end in a session that authenticated but + // can do nothing; with no source carrying grants the CONNECT is refused. + let claims = claims_from(&serde_json::json!({ + "iss": "https://idp.example/", + "aud": "fitz-broker", + "sub": "user:42", + "exp": 9_999_999_999_u64, + "tid": "acme-prod", + "permissions": [], + "roles": [], + "scope": "" + })); + + // Act + let result = claims.normalized_permissions(None, None, DEFAULT_ROLE_CLAIM); + + // Assert + let error = result.expect_err("an all-empty token must not authenticate"); + assert!( + error.contains("no permission source found"), + "unexpected error: {error}" + ); +} diff --git a/src/auth/claims/permissions.rs b/src/auth/claims/permissions.rs index 3e43442c..8c215740 100644 --- a/src/auth/claims/permissions.rs +++ b/src/auth/claims/permissions.rs @@ -6,9 +6,14 @@ impl RawClaims { /// Normalize permissions from claims using the prioritized sources: /// 1) configured namespaced custom claim /// 2) top-level permissions array (Auth0 RBAC) - /// 3) configured role claim array - /// 4) scp (space-delimited or array) - /// 5) scope (space-delimited string) + /// 3) configured permissions claim override + /// 4) configured role claim array + /// 5) scp (space-delimited or array) + /// 6) scope (space-delimited string) + /// + /// A source that is present but supplies no permission values is skipped + /// and the cascade continues, so an empty claim cannot mask a populated + /// one further down. /// /// # Errors /// @@ -20,42 +25,73 @@ impl RawClaims { permissions_claim_override: Option<&str>, role_claim: &str, ) -> Result, String> { + // A source that is present but carries no permission values is not a + // source. Auth0 emits `permissions: []` whenever RBAC is enabled with + // no permissions assigned for that API, even when the real grants live + // in a configured claim - so treating "present" as "found" stranded + // those tokens and reported the wrong claim as the cause. Each tier + // below yields `None` when it supplies nothing, and the cascade + // continues; if every tier is empty the chain still ends in + // "no permission source found", which refuses the CONNECT. if let Some(claim_name) = custom_claim { if let Some(perms) = self.custom_claim_permissions(claim_name)? { - return parse_permission_values(claim_name, perms, false, "permission"); + if let Some(parsed) = + parse_optional_permission_values(claim_name, perms, false, "permission")? + { + return Ok(parsed); + } } } if let Some(permissions) = &self.permissions { - return parse_permission_values( + if let Some(parsed) = parse_optional_permission_values( "permissions", permissions.clone(), false, "permission", - ); + )? { + return Ok(parsed); + } } if let Some(claim_name) = permissions_claim_override { if let Some(perms) = self.string_array_claim(claim_name, "permission")? { - return parse_permission_values(claim_name, perms, false, "permission"); + if let Some(parsed) = + parse_optional_permission_values(claim_name, perms, false, "permission")? + { + return Ok(parsed); + } } } if let Some(roles) = self.string_array_claim(role_claim, "role")? { - return parse_permission_values(role_claim, roles, false, "role"); + if let Some(parsed) = + parse_optional_permission_values(role_claim, roles, false, "role")? + { + return Ok(parsed); + } } if let Some(scp) = &self.scp { - return parse_permission_values("scp", scope_claim_values(scp), true, "scope string"); + if let Some(parsed) = parse_optional_permission_values( + "scp", + scope_claim_values(scp), + true, + "scope string", + )? { + return Ok(parsed); + } } if let Some(scope) = &self.scope { - return parse_permission_values( + if let Some(parsed) = parse_optional_permission_values( "scope", scope.split_whitespace().map(ToOwned::to_owned).collect(), true, "scope string", - ); + )? { + return Ok(parsed); + } } Err("no permission source found".to_string()) @@ -99,6 +135,24 @@ impl RawClaims { } } +/// Parse one candidate source, returning `None` when it supplies no permission +/// values at all. +/// +/// Malformed values still error: skipping those would let a typo silently +/// downgrade a token to whatever the next source happens to grant. Only a +/// source that says nothing is passed over. +fn parse_optional_permission_values( + source: &str, + values: Vec, + allow_resource_prefix: bool, + error_kind: &str, +) -> Result>, String> { + if values.iter().all(|value| value.trim().is_empty()) { + return Ok(None); + } + parse_permission_values(source, values, allow_resource_prefix, error_kind).map(Some) +} + fn parse_permission_values( source: &str, values: Vec, diff --git a/src/auth/diagnostics.rs b/src/auth/diagnostics.rs new file mode 100644 index 00000000..7314cf78 --- /dev/null +++ b/src/auth/diagnostics.rs @@ -0,0 +1,391 @@ +use base64::Engine; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; + +use super::AuthClaimsConfig; + +const MAX_DIAGNOSTIC_ARRAY_VALUES: usize = 16; +const MAX_DIAGNOSTIC_VALUE_CHARS: usize = 256; +const TOKEN_FINGERPRINT_HEX_CHARS: usize = 16; + +/// A bounded view of one untrusted JWT claim for failure logging. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JwtClaimDiagnostics { + pub name: String, + pub value_type: String, + pub values: Vec, + pub omitted_values: usize, + pub values_truncated: bool, +} + +/// Safe, bounded details extracted from a rejected JWT. +/// +/// This deliberately excludes the compact token, signature, subject, identity +/// values, and unrelated claims. Header and payload data are untrusted and are +/// exposed only as bounded diagnostic values. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JwtFailureDiagnostics { + pub token_fingerprint: String, + pub algorithm: Option, + pub key_id: Option, + pub payload_status: String, + pub issuer: Option, + pub audience: Option, + pub expires_at: Option, + pub not_before: Option, + pub expected_permission_sources: Vec, + pub presented_permission_sources: Vec, +} + +/// Extract bounded, non-secret diagnostics from a JWT that failed validation. +/// +/// The payload is decoded without signature verification strictly for logging +/// after the normal verification path has rejected the token. Callers must not +/// use this result for authentication or authorization decisions. +#[must_use] +pub fn jwt_failure_diagnostics( + compact: &str, + claims_config: &AuthClaimsConfig, +) -> JwtFailureDiagnostics { + let header = jsonwebtoken::decode_header(compact).ok(); + let algorithm = header.as_ref().map(|header| format!("{:?}", header.alg)); + let key_id = header + .and_then(|header| header.kid) + .map(|key_id| bounded_value(&key_id).0); + + let (payload_status, payload) = match decode_payload(compact) { + Ok(Value::Object(payload)) => ("decoded".to_string(), Some(payload)), + Ok(_) => ("payload is not a JSON object".to_string(), None), + Err(status) => (status.to_string(), None), + }; + + let expected_permission_sources = expected_permission_sources(claims_config); + let presented_permission_sources = payload.as_ref().map_or_else(Vec::new, |payload| { + presented_permission_sources(payload, claims_config) + }); + + JwtFailureDiagnostics { + token_fingerprint: token_fingerprint(compact), + algorithm, + key_id, + payload_status, + issuer: payload + .as_ref() + .and_then(|payload| claim_diagnostics(payload, "iss")), + audience: payload + .as_ref() + .and_then(|payload| claim_diagnostics(payload, "aud")), + expires_at: payload + .as_ref() + .and_then(|payload| claim_diagnostics(payload, "exp")), + not_before: payload + .as_ref() + .and_then(|payload| claim_diagnostics(payload, "nbf")), + expected_permission_sources, + presented_permission_sources, + } +} + +fn decode_payload(compact: &str) -> Result { + let mut parts = compact.split('.'); + let Some(_) = parts.next() else { + return Err("invalid compact JWT format"); + }; + let Some(payload) = parts.next() else { + return Err("invalid compact JWT format"); + }; + let Some(_) = parts.next() else { + return Err("invalid compact JWT format"); + }; + if parts.next().is_some() { + return Err("invalid compact JWT format"); + } + + let decoded = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(payload) + .or_else(|_| base64::engine::general_purpose::URL_SAFE.decode(payload)) + .map_err(|_| "JWT payload is not valid base64url")?; + serde_json::from_slice(&decoded).map_err(|_| "JWT payload is not valid JSON") +} + +fn token_fingerprint(compact: &str) -> String { + let digest = hex::encode(Sha256::digest(compact.as_bytes())); + digest[..TOKEN_FINGERPRINT_HEX_CHARS].to_string() +} + +fn expected_permission_sources(claims_config: &AuthClaimsConfig) -> Vec { + let mut sources = Vec::new(); + if let Some(custom_claim) = &claims_config.custom_claim { + push_unique(&mut sources, format!("{custom_claim}.permissions")); + } + push_unique(&mut sources, "permissions".to_string()); + if let Some(permissions_claim) = &claims_config.permissions_claim_override { + push_unique(&mut sources, permissions_claim.clone()); + } + push_unique(&mut sources, claims_config.role_claim.clone()); + push_unique(&mut sources, "scp".to_string()); + push_unique(&mut sources, "scope".to_string()); + sources +} + +fn presented_permission_sources( + payload: &Map, + claims_config: &AuthClaimsConfig, +) -> Vec { + let mut diagnostics = Vec::new(); + + if let Some(custom_claim) = &claims_config.custom_claim { + if let Some(value) = payload.get(custom_claim) { + if let Some(object) = value.as_object() { + if let Some(permissions) = object.get("permissions") { + diagnostics.push(JwtClaimDiagnostics::from_value( + format!("{custom_claim}.permissions"), + permissions, + )); + } else { + diagnostics.push(JwtClaimDiagnostics { + name: custom_claim.clone(), + value_type: "object without permissions".to_string(), + values: Vec::new(), + omitted_values: 0, + values_truncated: false, + }); + } + } else { + diagnostics.push(JwtClaimDiagnostics::from_value(custom_claim.clone(), value)); + } + } + } + + push_claim_if_present(&mut diagnostics, payload, "permissions"); + if let Some(permissions_claim) = &claims_config.permissions_claim_override { + push_claim_if_present(&mut diagnostics, payload, permissions_claim); + } + push_claim_if_present(&mut diagnostics, payload, &claims_config.role_claim); + push_claim_if_present(&mut diagnostics, payload, "scp"); + push_claim_if_present(&mut diagnostics, payload, "scope"); + + diagnostics +} + +fn push_unique(values: &mut Vec, value: String) { + if !values.iter().any(|existing| existing == &value) { + values.push(value); + } +} + +fn push_claim_if_present( + diagnostics: &mut Vec, + payload: &Map, + name: &str, +) { + if diagnostics.iter().any(|existing| existing.name == name) { + return; + } + if let Some(value) = payload.get(name) { + diagnostics.push(JwtClaimDiagnostics::from_value(name.to_string(), value)); + } +} + +fn claim_diagnostics(payload: &Map, name: &str) -> Option { + payload + .get(name) + .map(|value| JwtClaimDiagnostics::from_value(name.to_string(), value)) +} + +impl JwtClaimDiagnostics { + fn from_value(name: String, value: &Value) -> Self { + let value_type = json_value_type(value).to_string(); + let (values, omitted_values, values_truncated) = match value { + Value::Array(values) => { + let mut truncated = values.len() > MAX_DIAGNOSTIC_ARRAY_VALUES; + let summaries = values + .iter() + .take(MAX_DIAGNOSTIC_ARRAY_VALUES) + .map(|value| { + let (summary, value_truncated) = summarized_value(value); + truncated |= value_truncated; + summary + }) + .collect(); + ( + summaries, + values.len().saturating_sub(MAX_DIAGNOSTIC_ARRAY_VALUES), + truncated, + ) + } + Value::Object(_) => (Vec::new(), 0, false), + _ => { + let (summary, truncated) = summarized_value(value); + (vec![summary], 0, truncated) + } + }; + + Self { + name, + value_type, + values, + omitted_values, + values_truncated, + } + } +} + +fn summarized_value(value: &Value) -> (String, bool) { + match value { + Value::String(value) => bounded_value(value), + Value::Number(value) => (value.to_string(), false), + Value::Bool(value) => (value.to_string(), false), + Value::Null => ("null".to_string(), false), + Value::Array(_) => ("".to_string(), false), + Value::Object(_) => ("".to_string(), false), + } +} + +fn bounded_value(value: &str) -> (String, bool) { + let mut characters = value.chars(); + let bounded = characters + .by_ref() + .take(MAX_DIAGNOSTIC_VALUE_CHARS) + .collect::(); + let truncated = characters.next().is_some(); + if truncated { + (format!("{bounded}..."), true) + } else { + (bounded, false) + } +} + +fn json_value_type(value: &Value) -> &'static str { + match value { + Value::Null => "null", + Value::Bool(_) => "boolean", + Value::Number(_) => "number", + Value::String(_) => "string", + Value::Array(_) => "array", + Value::Object(_) => "object", + } +} + +#[cfg(test)] +mod tests { + use jsonwebtoken::{Algorithm, EncodingKey, Header}; + use serde_json::json; + + use super::*; + + #[test] + fn should_extract_permission_details_without_exposing_raw_jwt_or_subject() { + // Arrange + let mut header = Header::new(Algorithm::HS256); + header.kid = Some("key-42".to_string()); + let token = jsonwebtoken::encode( + &header, + &json!({ + "iss": "https://idp.example/", + "aud": ["fitz", "other-api"], + "sub": "private-subject", + "exp": 9_999_999_999_u64, + "nbf": 1_700_000_000_u64, + "permissions": ["notice://prod/orders/**#read", "bad\npermission"], + "roles": ["queue.write"], + "scope": "stream.read kv.write" + }), + &EncodingKey::from_secret(b"diagnostic-test-secret"), + ) + .unwrap(); + + // Act + let diagnostics = jwt_failure_diagnostics(&token, &AuthClaimsConfig::default()); + let rendered = format!("{diagnostics:?}"); + + // Assert + assert_eq!(diagnostics.algorithm.as_deref(), Some("HS256")); + assert_eq!(diagnostics.key_id.as_deref(), Some("key-42")); + assert_eq!(diagnostics.token_fingerprint.len(), 16); + assert_eq!( + diagnostics.expected_permission_sources, + ["permissions", "roles", "scp", "scope"] + ); + assert!(diagnostics + .presented_permission_sources + .iter() + .any(|source| source.name == "permissions" + && source.values == ["notice://prod/orders/**#read", "bad\npermission"])); + assert!(!rendered.contains(&token)); + assert!(!rendered.contains("private-subject")); + assert!(!rendered.contains("diagnostic-test-secret")); + } + + #[test] + fn should_bound_permission_claim_values_in_failure_diagnostics() { + // Arrange + let permission = format!("notice://realm/{}/**#read", "x".repeat(400)); + let permissions = vec![permission; MAX_DIAGNOSTIC_ARRAY_VALUES + 4]; + let token = jsonwebtoken::encode( + &Header::new(Algorithm::HS256), + &json!({ + "iss": "", + "aud": "fitz", + "sub": "subject", + "exp": 9_999_999_999_u64, + "permissions": permissions + }), + &EncodingKey::from_secret(b"diagnostic-test-secret"), + ) + .unwrap(); + + // Act + let diagnostics = jwt_failure_diagnostics(&token, &AuthClaimsConfig::default()); + let permissions = diagnostics + .presented_permission_sources + .iter() + .find(|source| source.name == "permissions") + .unwrap(); + + // Assert + assert_eq!(permissions.values.len(), MAX_DIAGNOSTIC_ARRAY_VALUES); + assert_eq!(permissions.omitted_values, 4); + assert!(permissions.values_truncated); + assert!(permissions + .values + .iter() + .all(|value| value.chars().count() <= MAX_DIAGNOSTIC_VALUE_CHARS + 3)); + } + + #[test] + fn should_report_malformed_configured_custom_permission_claim() { + // Arrange + let token = jsonwebtoken::encode( + &Header::new(Algorithm::HS256), + &json!({ + "iss": "", + "aud": "fitz", + "sub": "subject", + "exp": 9_999_999_999_u64, + "https://fitz.example/claims": {"roles": ["notice.read"]} + }), + &EncodingKey::from_secret(b"diagnostic-test-secret"), + ) + .unwrap(); + let config = AuthClaimsConfig::new( + "tid", + Some("https://fitz.example/claims".to_string()), + "roles", + ); + + // Act + let diagnostics = jwt_failure_diagnostics(&token, &config); + + // Assert + assert_eq!( + diagnostics.expected_permission_sources[0], + "https://fitz.example/claims.permissions" + ); + assert!(diagnostics + .presented_permission_sources + .iter() + .any(|source| source.name == "https://fitz.example/claims" + && source.value_type == "object without permissions")); + } +} diff --git a/src/auth/mod.rs b/src/auth/mod.rs index b74a3859..d655dd23 100644 --- a/src/auth/mod.rs +++ b/src/auth/mod.rs @@ -16,6 +16,7 @@ //! **Domains answer:** "Are you allowed to do this?" mod claims; +mod diagnostics; mod errors; mod jwks; mod realm; @@ -26,6 +27,7 @@ pub use claims::{ DEFAULT_ROLE_CLAIM, DEFAULT_ROUTE_FAMILY_CLAIM, ENV_AUTH_CUSTOM_CLAIM, ENV_AUTH_ROLE_CLAIM, ENV_ROUTE_FAMILY_CLAIM, ENV_ROUTE_FAMILY_MAP, }; +pub use diagnostics::{jwt_failure_diagnostics, JwtClaimDiagnostics, JwtFailureDiagnostics}; pub use errors::AuthError; pub use jwks::{ cache_jwks_from_json, cache_jwks_from_json_with_ttl, derive_jwks_url_from_issuer, diff --git a/src/boot/domains.rs b/src/boot/domains.rs index 5d8669a1..47660b4e 100644 --- a/src/boot/domains.rs +++ b/src/boot/domains.rs @@ -15,7 +15,7 @@ use crate::domains::lease::sink::LeaseDomainSink; use crate::domains::notice::sink::NoticeDomainSink; use crate::domains::queue::sink::QueueDomainSink; use crate::domains::rpc::sink::RpcDomainSink; -use crate::domains::schedule::sink::{ScheduleDomainSink, ScheduleObservability}; +use crate::domains::schedule::sink::ScheduleDomainSink; use crate::domains::stream::sink::StreamDomainSink; /// Generic domain sink: Forwards envelopes to domain actors. @@ -358,39 +358,39 @@ impl DomainHandles { } pub(crate) fn schedule_count(&self) -> usize { - ScheduleObservability::schedule_count(self.schedule.as_ref()) + self.schedule.schedule_count() } pub(crate) fn schedule_executions_per_minute(&self) -> f64 { - ScheduleObservability::executions_per_minute(self.schedule.as_ref()) + self.schedule.executions_per_minute() } pub(crate) fn schedule_subscription_count(&self) -> usize { - ScheduleObservability::subscription_count(self.schedule.as_ref()) + self.schedule.subscription_count() } pub(crate) fn schedule_pending_fire_count(&self) -> usize { - ScheduleObservability::pending_fire_count(self.schedule.as_ref()) + self.schedule.pending_fire_count() } pub(crate) fn schedule_pending_ack_retry_count(&self) -> usize { - ScheduleObservability::pending_ack_retry_count(self.schedule.as_ref()) + self.schedule.pending_ack_retry_count() } pub(crate) fn schedule_oldest_pending_claim_age_seconds(&self) -> u64 { - ScheduleObservability::oldest_pending_claim_age_seconds(self.schedule.as_ref()) + self.schedule.oldest_pending_claim_age_seconds() } pub(crate) fn schedule_notify_failure_count(&self) -> u64 { - ScheduleObservability::notify_failure_count(self.schedule.as_ref()) + self.schedule.notify_failure_count() } pub(crate) fn schedule_ack_failure_count(&self) -> u64 { - ScheduleObservability::ack_failure_count(self.schedule.as_ref()) + self.schedule.ack_failure_count() } pub(crate) fn schedule_overdue_normalization_count(&self) -> u64 { - ScheduleObservability::overdue_normalization_count(self.schedule.as_ref()) + self.schedule.overdue_normalization_count() } pub(crate) fn schedule_admin_pending_claims( @@ -853,7 +853,23 @@ mod tests { // Assert assert_eq!(snapshots.len(), DomainKind::ALL.len()); assert!(snapshots.iter().all(|snapshot| !snapshot.actor_running)); - assert!(snapshots.iter().all(|snapshot| snapshot.panic_count == 1)); + // Non-sharded domains (kv/queue/notice/lease/schedule) panic exactly + // one actor. Family-sharded domains (rpc/stream) are provisioned + // with 7 route families here (`domain_setup_options`) and must be + // panicked on *every* family to reach full exhaustion -- see + // `panic_actor_for_tests` on `RpcDomainSink`/`StreamDomainSink` -- + // so their panic_count legitimately lands at 7, not 1. + for snapshot in &snapshots { + let expected_panic_count = match snapshot.domain { + "rpc" | "stream" => 7, + _ => 1, + }; + assert_eq!( + snapshot.panic_count, expected_panic_count, + "unexpected panic_count for domain {}", + snapshot.domain + ); + } assert!(snapshots.iter().all(|snapshot| snapshot.restart_exhausted)); assert_eq!(domains.kv_active_transaction_count(), 0); assert_eq!(domains.queue_ready_message_count(), 0); diff --git a/src/boot/storage.rs b/src/boot/storage.rs index cb1a174b..be731a65 100644 --- a/src/boot/storage.rs +++ b/src/boot/storage.rs @@ -408,6 +408,14 @@ fn log_cloud_lease_contention( } } +/// Enclosing deadline for one synchronous Midge runtime response. +/// +/// Midge floors this at `storage_io_timeout + 30s`, so a provider callback can +/// exhaust its own budget before this expires. It is deliberately far longer +/// than any fitz domain deadline: the broker gives up on a request quickly and +/// tells the client to retry, while storage keeps working. +const STORAGE_RUNTIME_RESPONSE_TIMEOUT: Duration = Duration::from_secs(60); + fn build_midge_open_options( open_options: cntryl_midge::OpenOptionsBuilder, config: &BootConfig, @@ -434,6 +442,13 @@ fn build_midge_open_options( None => open_options, }; + // Set the storage-side deadline explicitly rather than inheriting the + // default, so the relationship between the two budgets is visible in code. + // Fitz's own domain actor-reply deadlines are far shorter and will always + // fire first; that is only safe because a domain timeout is answered with + // a retryable error frame instead of closing the session. + let open_options = open_options.runtime_response_timeout(STORAGE_RUNTIME_RESPONSE_TIMEOUT); + open_options .build() .map_err(|error| format!("Invalid Midge open options: {error}").into()) diff --git a/src/boot/storage/tests.rs b/src/boot/storage/tests.rs index 8f7990d4..52a4dfa3 100644 --- a/src/boot/storage/tests.rs +++ b/src/boot/storage/tests.rs @@ -283,6 +283,26 @@ fn should_skip_sqrzl_test_for_transport_errors() { ); } +#[test] +fn should_skip_sqrzl_test_for_missing_content_length() { + // Arrange: local mock GCS servers can reject a PUT without an explicit + // Content-Length header, which is a mock-server quirk unrelated to the + // recovery behavior under test. + let error = "prepare Sqrzl namespace failed: GCS setup request PUT /fitz-sqrzl-gcs failed \ + with status 411 Length Required: \ + MissingContentLength\ + Content-Length is required unless Transfer-Encoding is chunked."; + + // Act + let should_skip = should_skip_sqrzl_test(error); + + // Assert + assert!( + should_skip, + "expected a mock-server 411 Length Required response to be skippable" + ); +} + #[test] fn should_reject_cloud_storage_without_bucket() { // Arrange @@ -481,6 +501,7 @@ fn should_skip_sqrzl_test(error: &str) -> bool { || lower.contains("dns") || lower.contains("signaturedoesnotmatch") || lower.contains("status 403") + || lower.contains("status 411") || lower.contains("status 500") || lower.contains("lease acquisition i/o error") } diff --git a/src/client/iterator.ts b/src/client/iterator.ts deleted file mode 100644 index 5464ac4c..00000000 --- a/src/client/iterator.ts +++ /dev/null @@ -1,138 +0,0 @@ -/** - * Iterator[T] is a generic streaming iterator modeled after fitz-go's Iterator[T]. - * - * Usage pattern (manual): - * ```ts - * const it = await tx.scan(prefix, 100); - * try { - * while (it.next()) { - * const value = it.value(); - * // use value - * } - * if (it.err()) throw it.err(); - * } finally { - * it.close(); - * } - * ``` - * - * Usage pattern (with forEach helper): - * ```ts - * const it = await tx.scan(prefix, 100); - * return forEach(it, (value) => { - * // use value - * return Promise.resolve(undefined); - * }); - * ``` - */ -export interface Iterator { - /** - * Advances the iterator and returns true if a value is available. - */ - next(): boolean; - - /** - * Returns the current item (valid only after a successful next()). - */ - value(): T; - - /** - * Returns the first non-EOF error encountered. - */ - err(): Error | null; - - /** - * Closes/releases any resources associated with the iterator. - */ - close(): void; -} - -/** - * SliceIterator iterates over an in-memory slice/array. - * Used for batch results like KV SCAN where all items arrive in one response. - */ -export class SliceIterator implements Iterator { - private items: T[]; - private index: number = -1; - - constructor(items: T[]) { - this.items = items; - } - - next(): boolean { - this.index++; - return this.index < this.items.length; - } - - value(): T { - if (this.index < 0 || this.index >= this.items.length) { - throw new Error('SliceIterator.value() called in invalid state'); - } - return this.items[this.index]; - } - - err(): Error | null { - return null; // Slice iteration never produces errors - } - - close(): void { - // No-op: no resources to release - } -} - -/** - * AsyncIterableIterator is a JavaScript - * AsyncIterable/AsyncIterator that wraps an Iterator[T]. - * Useful for for-await-of loops in TypeScript. - */ -export class AsyncIterableIterator implements AsyncIterable { - constructor(private iterator: Iterator) {} - - [Symbol.asyncIterator](): AsyncIterator { - return { - next: async () => { - if (this.iterator.next()) { - return { - done: false, - value: this.iterator.value(), - }; - } - const err = this.iterator.err(); - this.iterator.close(); - if (err) { - throw err; - } - return { done: true, value: undefined }; - }, - }; - } -} - -/** - * forEach iterates over all items in the iterator, calling fn for each. - * Automatically handles close() and error checking. - * iteration stops on first error from either callback or iterator. - * - * Example: - * ```ts - * const it = await tx.scan(startKey, 100); - * return forEach(it, async (kv) => { - * console.log(`${kv.key}: ${kv.value}`); - * }); - * ``` - */ -export async function forEach( - it: Iterator, - fn: (item: T) => Promise -): Promise { - try { - while (it.next()) { - await fn(it.value()); - } - const err = it.err(); - if (err) { - throw err; - } - } finally { - it.close(); - } -} diff --git a/src/client/validation.rs b/src/client/validation.rs index 383ff0e4..e2ef5fee 100644 --- a/src/client/validation.rs +++ b/src/client/validation.rs @@ -281,7 +281,7 @@ pub struct SizeLimits { /// Maximum value size (default 100 MB) pub max_value_size: NonZeroUsize, - /// Maximum event size for streams (default 50 MB) + /// Maximum event body size for streams (defaults to the broker contract) pub max_event_size: NonZeroUsize, } @@ -365,7 +365,8 @@ impl Default for SizeLimits { Self { max_key_size: NonZeroUsize::new(1024 * 1024).unwrap(), // 1 MB max_value_size: NonZeroUsize::new(100 * 1024 * 1024).unwrap(), // 100 MB - max_event_size: NonZeroUsize::new(50 * 1024 * 1024).unwrap(), // 50 MB + max_event_size: NonZeroUsize::new(crate::domains::stream::protocol::MAX_EVENT_SIZE) + .unwrap(), } } } @@ -529,6 +530,14 @@ impl Default for IntegrityChecker { mod tests { use super::*; + #[test] + fn should_default_event_limit_to_the_stream_contract() { + assert_eq!( + SizeLimits::default().max_event_size.get(), + crate::domains::stream::protocol::MAX_EVENT_SIZE + ); + } + #[test] fn should_accept_empty_key_value_pairs() { // Arrange diff --git a/src/control/admin/read_model.rs b/src/control/admin/read_model.rs index 2a12496a..7c3bf7b2 100644 --- a/src/control/admin/read_model.rs +++ b/src/control/admin/read_model.rs @@ -179,6 +179,29 @@ impl AdminReadModel { collect_slice_matches(&transactions, |item| matches_realm(realm, &item.realm)) } + pub(crate) fn kv_transaction_count(&self) -> usize { + self.kv_transactions.read().len() + } + + pub(crate) fn kv_transaction_count_for_resource( + &self, + route_family: u64, + realm: &str, + area: &str, + resource: &str, + ) -> usize { + self.kv_transactions + .read() + .iter() + .filter(|transaction| { + transaction.route_family == route_family + && transaction.realm == realm + && transaction.area == area + && transaction.resource == resource + }) + .count() + } + pub fn replace_streams(&self, streams: Vec) { *self.streams.write() = streams; } @@ -793,6 +816,31 @@ mod tests { assert_eq!(transactions[0].resource, "orders"); } + #[test] + fn should_count_kv_transactions_without_materializing_snapshots() { + // Arrange + let read_model = AdminReadModel::default(); + for (tx_id, resource) in [(41, "users"), (42, "users"), (43, "orders")] { + read_model.upsert_kv_transaction(KvTransaction::snapshot( + 1, + tx_id, + 7, + "acme", + "app", + resource, + "2026-03-31T00:00:00Z", + )); + } + + // Act + let total = read_model.kv_transaction_count(); + let users = read_model.kv_transaction_count_for_resource(1, "acme", "app", "users"); + + // Assert + assert_eq!(total, 3); + assert_eq!(users, 2); + } + #[test] fn should_upsert_lease_given_incremental_update() { // Arrange diff --git a/src/domains/kv/actor/errors.rs b/src/domains/kv/actor/error_mapping.rs similarity index 57% rename from src/domains/kv/actor/errors.rs rename to src/domains/kv/actor/error_mapping.rs index 2d3036fb..7bd1f4e0 100644 --- a/src/domains/kv/actor/errors.rs +++ b/src/domains/kv/actor/error_mapping.rs @@ -1,11 +1,21 @@ +//! Borrowed classification of Midge failures into KV protocol errors. + use super::KvActor; use crate::domains::kv::KvError; impl KvActor { /// Map a Midge error to the KV domain contract. - #[allow(clippy::needless_pass_by_value)] - pub(super) fn map_midge_error(error: cntryl_midge::MidgeError) -> KvError { - Self::classify_midge_message(&error.to_string()) + /// + /// Only a small set of `MidgeError` variants are mapped as typed errors. + /// The remaining variants use best-effort message classification so older + /// or less-structured storage failures remain compatible. + pub(super) fn map_midge_error(error: &cntryl_midge::MidgeError) -> KvError { + match error { + cntryl_midge::MidgeError::Timeout(_) | cntryl_midge::MidgeError::Busy(_) => { + KvError::BackendUnavailable(error.to_string()) + } + _ => Self::classify_midge_message(&error.to_string()), + } } pub(super) fn classify_midge_message(message: &str) -> KvError { diff --git a/src/domains/kv/actor/introspection.rs b/src/domains/kv/actor/introspection.rs new file mode 100644 index 00000000..67fde563 --- /dev/null +++ b/src/domains/kv/actor/introspection.rs @@ -0,0 +1,42 @@ +//! Read-only crate-internal views over live transaction state. + +use super::KvActor; +use crate::domains::kv::KvResourceScope; + +/// Named transaction data used only by actor-state regression tests. +#[cfg(test)] +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct KvTransactionSnapshot { + pub(crate) tx_id: u64, + pub(crate) scope: KvResourceScope, +} + +impl KvActor { + #[must_use] + pub(crate) fn mutation_count_for_tx(&self, tx_id: u64) -> Option { + self.transactions.get(&tx_id).map(|tx| tx.mutation_count) + } + + #[must_use] + pub(crate) fn resource_scope_for_tx(&self, tx_id: u64) -> Option { + self.transactions.get(&tx_id).map(|tx| tx.scope.clone()) + } + + #[must_use] + #[cfg(test)] + pub(crate) fn active_transaction_snapshots(&self) -> Vec { + self.transactions + .iter() + .map(|(tx_id, tx)| KvTransactionSnapshot { + tx_id: *tx_id, + scope: tx.scope.clone(), + }) + .collect() + } + + #[must_use] + #[cfg(test)] + pub(crate) fn transaction_count(&self) -> usize { + self.transactions.len() + } +} diff --git a/src/domains/kv/actor/inventory.rs b/src/domains/kv/actor/inventory.rs deleted file mode 100644 index 6da4b54e..00000000 --- a/src/domains/kv/actor/inventory.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::collections::HashMap; - -#[derive(Clone, Copy, Debug)] -pub(super) struct KvKeyInventoryChange { - pub(super) before_bytes: Option, - pub(super) after_bytes: Option, -} - -#[derive(Default)] -pub(super) struct KvInventoryDelta { - pub(super) key_changes: HashMap, KvKeyInventoryChange>, - pub(super) estimate_incomplete: bool, -} - -impl KvInventoryDelta { - pub(super) fn is_empty(&self) -> bool { - self.key_changes.is_empty() && !self.estimate_incomplete - } - - pub(super) fn mark_incomplete(&mut self) { - self.estimate_incomplete = true; - } - - pub(super) fn record_key_change( - &mut self, - user_key: &[u8], - before_bytes: Option, - after_bytes: Option, - ) { - self.key_changes - .entry(user_key.to_vec()) - .and_modify(|change| change.after_bytes = after_bytes) - .or_insert(KvKeyInventoryChange { - before_bytes, - after_bytes, - }); - } -} diff --git a/src/domains/kv/actor/inventory_delta.rs b/src/domains/kv/actor/inventory_delta.rs new file mode 100644 index 00000000..5ccdd1b1 --- /dev/null +++ b/src/domains/kv/actor/inventory_delta.rs @@ -0,0 +1,88 @@ +//! Transient inventory bookkeeping and post-commit estimate persistence. +//! +//! A successful `INSERT` proves that a key was absent and can be counted +//! exactly. `PUT`, `DELETE`, and `DELETE_RANGE` deliberately avoid hot-path +//! reads and mark the estimate incomplete; the admin inventory path then +//! refreshes it from committed rows. + +use super::KvActor; +use crate::domains::kv::inventory::encode_estimate; +use crate::domains::kv::{KvError, KvResourceScope}; +use cntryl_midge::{ColumnFamilyId, Engine as MidgeEngine, TransactionMode}; +use std::collections::HashMap; + +#[derive(Default)] +pub(super) struct KvInventoryDelta { + inserted_key_bytes: HashMap, usize>, + estimate_incomplete: bool, +} + +impl KvInventoryDelta { + pub(super) fn is_empty(&self) -> bool { + self.inserted_key_bytes.is_empty() && !self.estimate_incomplete + } + + pub(super) fn mark_incomplete(&mut self) { + self.estimate_incomplete = true; + } + + pub(super) fn record_insert(&mut self, user_key: &[u8], stored_bytes: usize) { + self.inserted_key_bytes + .entry(user_key.to_vec()) + .or_insert(stored_bytes); + } +} + +impl KvActor { + pub(super) fn inventory_write_options( + committed: cntryl_midge::WriteOptions, + ) -> cntryl_midge::WriteOptions { + // Inventory estimates are best-effort admin bookkeeping, so we avoid + // imposing stronger durability than required for user data writes. + if committed.is_cloud_async() || committed.is_cloud_strict() { + cntryl_midge::WriteOptions::cloud_async() + } else { + cntryl_midge::WriteOptions::buffered() + } + } + + pub(super) fn apply_inventory_delta( + store: &MidgeEngine, + column_family: ColumnFamilyId, + scope: &KvResourceScope, + inventory_delta: &KvInventoryDelta, + write_options: cntryl_midge::WriteOptions, + ) -> Result<(), KvError> { + if inventory_delta.is_empty() { + return Ok(()); + } + + let key = Self::inventory_metadata_key(&scope.realm, &scope.area, &scope.resource); + let mut tx = store + .begin_tx(column_family, TransactionMode::ReadWrite) + .map_err(|error| Self::map_midge_error(&error))?; + let mut estimate = tx + .get(&key) + .map_err(|error| Self::map_midge_error(&error))? + .as_deref() + .map(crate::domains::kv::inventory::decode_estimate) + .transpose() + .map_err(KvError::BackendError)? + .unwrap_or_default(); + + for stored_bytes in inventory_delta.inserted_key_bytes.values() { + estimate.estimated_record_count = estimate.estimated_record_count.saturating_add(1); + estimate.estimated_storage_bytes = estimate + .estimated_storage_bytes + .saturating_add(*stored_bytes as u64); + } + if inventory_delta.estimate_incomplete { + estimate.estimate_complete = false; + } + + tx.put(key, encode_estimate(estimate), None) + .map_err(|error| Self::map_midge_error(&error))?; + tx.commit(write_options) + .map_err(|error| Self::map_midge_error(&error)) + } +} diff --git a/src/domains/kv/actor/key_layout.rs b/src/domains/kv/actor/key_layout.rs new file mode 100644 index 00000000..c4a211ac --- /dev/null +++ b/src/domains/kv/actor/key_layout.rs @@ -0,0 +1,83 @@ +//! Storage key layouts for committed user rows and admin inventory metadata. + +use super::KvActor; +use crate::runtime::routing::RouteFamily; +use crate::utils::storage_key::{self, DomainKeyspace}; +use lexkey::LexKey; + +pub(super) const KV_KEY_SCOPE_MARKER: u8 = 0x01; +const KV_INVENTORY_SCOPE_MARKER: u8 = 0x02; + +impl KvActor { + /// Resolve the column family from `RouteFamily`; resource isolation uses key prefixes. + /// + /// # Errors + /// Returns an error if the family would select the forbidden default column family. + pub(crate) fn resolve_column_family( + route_family: RouteFamily, + ) -> Result { + crate::runtime::cf_validation::validate_route_family(route_family)?; + Ok(route_family.id()) + } + + pub(crate) fn encode_scoped_key(prefix: &[u8], user_key: &[u8]) -> Vec { + let mut encoded = Vec::with_capacity(prefix.len() + user_key.len()); + encoded.extend_from_slice(prefix); + encoded.extend_from_slice(user_key); + encoded + } + + pub(crate) fn strip_scoped_prefix(prefix: &[u8], scoped_key: &[u8]) -> Option> { + scoped_key.strip_prefix(prefix).map(<[u8]>::to_vec) + } + + pub(crate) fn prefix_range_end(prefix: &[u8]) -> Vec { + crate::utils::storage_key::prefix_range_end(prefix) + } + + pub(crate) fn realm_resource_prefix(realm: &str, area: &str, resource: &str) -> Vec { + let mut encoder = storage_key::domain_marker_encoder( + realm, + DomainKeyspace::Kv, + KV_KEY_SCOPE_MARKER, + area.len() + resource.len() + 2, + ); + storage_key::encode_bytes_segment_into(&mut encoder, area.as_bytes()); + storage_key::encode_bytes_segment_into(&mut encoder, resource.as_bytes()); + encoder.into_vec() + } + + pub(crate) fn inventory_metadata_key(realm: &str, area: &str, resource: &str) -> Vec { + let mut encoder = storage_key::domain_marker_encoder( + realm, + DomainKeyspace::Kv, + KV_INVENTORY_SCOPE_MARKER, + area.len() + resource.len() + 2, + ); + storage_key::encode_bytes_segment_into(&mut encoder, area.as_bytes()); + storage_key::encode_bytes_segment_into(&mut encoder, resource.as_bytes()); + encoder.into_vec() + } + + pub(crate) fn parse_inventory_metadata_key(key: &[u8]) -> Option<(String, String, String)> { + let (realm, suffix) = storage_key::split_domain_key(key, DomainKeyspace::Kv)?; + if suffix.first().copied()? != KV_INVENTORY_SCOPE_MARKER { + return None; + } + + let mut parts = suffix[1..].split(|byte| *byte == LexKey::SEPARATOR); + let area = parts.next()?; + let resource = parts.next()?; + let trailing = parts.next(); + if area.is_empty() || resource.is_empty() || trailing != Some(&[]) || parts.next().is_some() + { + return None; + } + + Some(( + realm.to_string(), + String::from_utf8(area.to_vec()).ok()?, + String::from_utf8(resource.to_vec()).ok()?, + )) + } +} diff --git a/src/domains/kv/actor/keys.rs b/src/domains/kv/actor/keys.rs deleted file mode 100644 index 784ca493..00000000 --- a/src/domains/kv/actor/keys.rs +++ /dev/null @@ -1,31 +0,0 @@ -use super::KvActor; -use crate::runtime::routing::RouteFamily; - -impl KvActor { - /// Resolve the column family from `RouteFamily`; resource isolation uses key prefixes. - /// - /// # Errors - /// Returns an error if the family would select the forbidden default column family. - pub(crate) fn resolve_column_family( - route_family: RouteFamily, - _resource: &str, - ) -> Result { - crate::runtime::cf_validation::validate_route_family(route_family)?; - Ok(route_family.id()) - } - - pub(crate) fn encode_scoped_key(prefix: &[u8], user_key: &[u8]) -> Vec { - let mut encoded = Vec::with_capacity(prefix.len() + user_key.len()); - encoded.extend_from_slice(prefix); - encoded.extend_from_slice(user_key); - encoded - } - - pub(crate) fn strip_scoped_prefix(prefix: &[u8], scoped_key: &[u8]) -> Option> { - scoped_key.strip_prefix(prefix).map(<[u8]>::to_vec) - } - - pub(crate) fn prefix_range_end(prefix: &[u8]) -> Vec { - crate::utils::storage_key::prefix_range_end(prefix) - } -} diff --git a/src/domains/kv/actor/mod.rs b/src/domains/kv/actor/mod.rs index 3a3027cc..6aee10ea 100644 --- a/src/domains/kv/actor/mod.rs +++ b/src/domains/kv/actor/mod.rs @@ -1,118 +1,60 @@ -//! KV actor: durable committed writes with session-scoped transaction state -//! over Midge. +//! KV transaction state machine over Midge. //! -//! # Architecture -//! -//! The KV actor maintains per-session, broker-local transaction state -//! (`ActiveKvTx`). All KV operations execute within the context of an active -//! transaction bound to a single resource (table). `tx_id` values are runtime -//! handles for the current session only; reconnect or broker restart requires a -//! new `begin`. -//! -//! # Write Options -//! -//! Clients select a durability class via `WriteOptions` passed in `Begin`: -//! - `WriteOptions::synced()` - fsync on every commit (latency-first) -//! - `WriteOptions::buffered()` - no fsync, OS buffering (throughput-first) -//! -//! The broker may map canonical sync or buffered requests to an -//! operator-configured policy within the requested class (for example, cloud -//! strict or cloud asynchronous writes). The client controls the class, while -//! deployment configuration controls the concrete policy within that class. -//! These options apply only to committed writes. Open transaction handles, -//! uncommitted writes, and resource-lock ownership remain broker-local memory -//! and are lost on session disconnect or broker restart. -//! -//! # Invariants -//! -//! 1. All KV ops require an active transaction -//! 2. Transactions are scoped to a single resource -//! 3. `RouteFamily` -> `ColumnFamily` mapping is explicit (no default CF) -//! 4. No buffering, retries, or caching - direct Midge passthrough +//! Committed values are durable according to the transaction write policy. +//! Open transaction handles and uncommitted mutations are session-scoped, +//! broker-local state and disappear on cleanup or restart. -use bytes::Bytes; -use cntryl_midge::{ColumnFamilyId, Engine as MidgeEngine, TransactionMode}; +use cntryl_midge::{ColumnFamilyId, Engine as MidgeEngine}; use std::collections::HashMap; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::Instant; -use crate::auth::validate_realm_format; use crate::prelude::Actor; use crate::runtime::actor::Context; -use crate::utils::storage_key::{self, DomainKeyspace}; - -use super::protocol::{KvError, KvMessage, KvPair, KvResourceScope, KvResponse, ScanQuery, TxMode}; -mod errors; -mod inventory; -mod keys; +use super::protocol::{KvMessage, KvResourceScope, KvResponse}; -use inventory::KvInventoryDelta; +mod error_mapping; +mod introspection; +mod inventory_delta; +mod key_layout; +mod mutations; +mod scan; +mod transaction_access; +mod transactions; -const KV_KEY_SCOPE_MARKER: u8 = 0x01; -const KV_INVENTORY_SCOPE_MARKER: u8 = 0x02; -const KV_INVENTORY_VALUE_VERSION: u8 = 1; -const KV_INVENTORY_VALUE_LEN: usize = 18; -const KV_INVENTORY_RECORD_COUNT_RANGE: std::ops::Range = 2..10; -const KV_INVENTORY_STORAGE_BYTES_RANGE: std::ops::Range = 10..18; -const MAX_SCAN_ITEMS: usize = 1_024; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct KvInventoryEstimate { - pub estimated_record_count: u64, - pub estimated_storage_bytes: u64, - pub estimate_complete: bool, -} +use inventory_delta::KvInventoryDelta; +#[cfg(test)] +use key_layout::KV_KEY_SCOPE_MARKER; +#[cfg(test)] +use scan::MAX_SCAN_ITEMS; -impl Default for KvInventoryEstimate { - fn default() -> Self { - Self { - estimated_record_count: 0, - estimated_storage_bytes: 0, - estimate_complete: true, - } - } -} +#[cfg(test)] +use super::protocol::{KvError, ScanQuery, TxMode}; +#[cfg(test)] +use bytes::Bytes; -/// Active KV transaction state. -/// -/// This state is broker-local and in-memory only. Dropping the owning actor, -/// cleaning up the owning session, or restarting the broker aborts any -/// uncommitted work and discards the transaction handle instead of attempting -/// recovery. The broker also force-rolls back transactions that remain idle -/// beyond its configured transaction TTL so abandoned writers cannot retain a -/// resource lock indefinitely. -pub struct ActiveKvTx { - /// Complete route scope this transaction is bound to. - pub scope: KvResourceScope, - /// Client-declared transaction access mode. - pub mode: TxMode, - /// Cached realm/area/resource prefix for scoped-key encoding - pub scoped_prefix: Vec, - /// Resolved column family for this transaction - pub column_family: ColumnFamilyId, - /// Midge transaction handle - pub tx: cntryl_midge::Transaction, - /// Write options for commit - pub write_options: cntryl_midge::WriteOptions, - /// Successful mutating operations performed within this transaction. - pub mutation_count: u64, +/// One live transaction bound to a single KV resource. +struct ActiveKvTx { + scope: KvResourceScope, + scoped_prefix: Vec, + column_family: ColumnFamilyId, + tx: cntryl_midge::Transaction, + write_options: cntryl_midge::WriteOptions, + mutation_count: u64, last_activity: Instant, inventory_delta: KvInventoryDelta, } -/// KV actor managing transactions for a session +/// Session-scoped KV transaction state. pub struct KvActor { - /// Midge storage engine store: Arc, - /// Active transactions by server-assigned ID transactions: HashMap, - /// Next transaction ID to assign next_tx_id: u64, } impl KvActor { - /// Create a new KV actor + #[must_use] pub fn new(store: Arc) -> Self { Self { store, @@ -121,9 +63,8 @@ impl KvActor { } } - /// Handle KV message - pub fn handle(&mut self, msg: KvMessage) -> KvResponse { - match msg { + pub fn handle(&mut self, message: KvMessage) -> KvResponse { + match message { KvMessage::Begin { scope, mode, @@ -158,667 +99,14 @@ impl KvActor { } => self.handle_scan(tx_id, &scope, &query), } } - - /// Begin a new transaction - fn handle_begin( - &mut self, - scope: KvResourceScope, - mode: TxMode, - write_options: cntryl_midge::WriteOptions, - ) -> KvResponse { - // Validate realm format (strict opaque identifier check) - if validate_realm_format(&scope.realm).is_err() { - return KvResponse::Error { - error: KvError::InvalidRealm, - }; - } - - // Resolve column family from RouteFamily + resource - let Ok(cf) = Self::resolve_column_family(scope.route_family, &scope.resource) else { - return KvResponse::Error { - error: KvError::InvalidRouteFamily, - }; - }; - - // Create Midge transaction - let tx_mode = match mode { - TxMode::ReadOnly => TransactionMode::ReadOnly, - TxMode::ReadWrite => TransactionMode::ReadWrite, - }; - - let Some(next_tx_id) = self.next_tx_id.checked_add(1) else { - return KvResponse::Error { - error: KvError::InvalidRequest("transaction ID space exhausted".to_string()), - }; - }; - - match self.store.begin_tx(cf, tx_mode) { - Ok(tx) => { - let tx_id = self.next_tx_id; - self.next_tx_id = next_tx_id; - let scoped_prefix = - Self::realm_resource_prefix(&scope.realm, &scope.area, &scope.resource); - - tracing::trace!( - "KvActor assigning transaction ID: {}, next_tx_id is now: {}", - tx_id, - self.next_tx_id - ); - - self.transactions.insert( - tx_id, - ActiveKvTx { - scope, - mode, - scoped_prefix, - column_family: cf, - tx, - write_options, - mutation_count: 0, - last_activity: Instant::now(), - inventory_delta: KvInventoryDelta::default(), - }, - ); - KvResponse::BeginOk { tx_id } - } - Err(e) => KvResponse::Error { - error: Self::map_midge_error(e), - }, - } - } - - /// Commit a transaction by ID - fn handle_commit(&mut self, tx_id: u64, scope: &KvResourceScope) -> KvResponse { - let Some(active) = self.transactions.get(&tx_id) else { - return KvResponse::Error { - error: KvError::InvalidTxId, - }; - }; - if let Err(response) = Self::validate_operation_scope(active, scope) { - return response; - } - - match self.transactions.remove(&tx_id) { - None => KvResponse::Error { - error: KvError::InvalidTxId, - }, - Some(mut active) => { - let inventory_scope = active.scope.clone(); - let inventory_column_family = active.column_family; - let inventory_delta = std::mem::take(&mut active.inventory_delta); - // Use write options provided by user at transaction begin - match active.tx.commit(active.write_options) { - Ok(()) => { - if let Err(error) = Self::apply_inventory_delta( - &self.store, - inventory_column_family, - &inventory_scope, - &inventory_delta, - ) { - tracing::warn!(?error, "KV inventory estimate update failed"); - } - KvResponse::CommitOk - } - Err(e) => KvResponse::Error { - error: Self::map_midge_error(e), - }, - } - } - } - } - - /// Rollback a transaction by ID - fn handle_rollback(&mut self, tx_id: u64, scope: &KvResourceScope) -> KvResponse { - let Some(active) = self.transactions.get(&tx_id) else { - return KvResponse::Error { - error: KvError::InvalidTxId, - }; - }; - if let Err(response) = Self::validate_operation_scope(active, scope) { - return response; - } - - match self.transactions.remove(&tx_id) { - None => KvResponse::Error { - error: KvError::InvalidTxId, - }, - Some(_active) => { - // Transaction is dropped, automatically rolled back by Midge - KvResponse::RollbackOk - } - } - } - - /// Get a value by key - fn handle_get(&mut self, tx_id: u64, scope: &KvResourceScope, key: &Bytes) -> KvResponse { - let active = match self.get_transaction_or_err(tx_id) { - Ok(tx) => tx, - Err(err) => return err, - }; - - if let Err(response) = Self::validate_operation_scope(active, scope) { - return response; - } - - let scoped_key = Self::encode_scoped_key(&active.scoped_prefix, key); - - match active.tx.get(&scoped_key) { - Ok(Some(value)) => KvResponse::GetResult { - found: true, - value: Some(value), - }, - Ok(None) => KvResponse::GetResult { - found: false, - value: None, - }, - Err(e) => KvResponse::Error { - error: Self::map_midge_error(e), - }, - } - } - - /// Put (upsert) a key-value pair - fn handle_put( - &mut self, - tx_id: u64, - scope: &KvResourceScope, - key: &Bytes, - value: &Bytes, - ) -> KvResponse { - let active = match self.get_transaction_or_err(tx_id) { - Ok(tx) => tx, - Err(err) => return err, - }; - - if let Err(response) = Self::validate_operation_scope(active, scope) { - return response; - } - - let scoped_key = Self::encode_scoped_key(&active.scoped_prefix, key); - match active.tx.put(scoped_key, value.to_vec(), None) { - Ok(()) => { - active.mutation_count = active.mutation_count.saturating_add(1); - active.inventory_delta.mark_incomplete(); - KvResponse::PutOk - } - Err(e) => KvResponse::Error { - error: Self::map_midge_error(e), - }, - } - } - - /// Insert a key-value pair (fail if exists) - fn handle_insert( - &mut self, - tx_id: u64, - scope: &KvResourceScope, - key: &Bytes, - value: &Bytes, - ) -> KvResponse { - let active = match self.get_transaction_or_err(tx_id) { - Ok(tx) => tx, - Err(err) => return err, - }; - - if let Err(response) = Self::validate_operation_scope(active, scope) { - return response; - } - - // Check if key exists first - let scoped_key = Self::encode_scoped_key(&active.scoped_prefix, key); - - match active.tx.get(&scoped_key) { - Ok(Some(_)) => { - // Key exists, insert should fail - KvResponse::Error { - error: KvError::AlreadyExists, - } - } - Ok(None) => { - // Key doesn't exist, proceed with insert - match active.tx.put(scoped_key, value.to_vec(), None) { - Ok(()) => { - active.mutation_count = active.mutation_count.saturating_add(1); - active.inventory_delta.record_key_change( - key, - None, - Some(key.len() + value.len()), - ); - KvResponse::InsertOk - } - Err(e) => KvResponse::Error { - error: Self::map_midge_error(e), - }, - } - } - Err(e) => KvResponse::Error { - error: Self::map_midge_error(e), - }, - } - } - - /// Delete a key - fn handle_delete(&mut self, tx_id: u64, scope: &KvResourceScope, key: &Bytes) -> KvResponse { - let active = match self.get_transaction_or_err(tx_id) { - Ok(tx) => tx, - Err(err) => return err, - }; - - if let Err(response) = Self::validate_operation_scope(active, scope) { - return response; - } - - let scoped_key = Self::encode_scoped_key(&active.scoped_prefix, key); - match active.tx.delete(scoped_key) { - Ok(()) => { - active.mutation_count = active.mutation_count.saturating_add(1); - active.inventory_delta.mark_incomplete(); - KvResponse::DeleteOk - } - Err(e) => KvResponse::Error { - error: Self::map_midge_error(e), - }, - } - } - - /// Delete a range of keys [start, end) - fn handle_delete_range( - &mut self, - tx_id: u64, - scope: &KvResourceScope, - start: &Bytes, - end: &Bytes, - ) -> KvResponse { - let active = match self.get_transaction_or_err(tx_id) { - Ok(tx) => tx, - Err(err) => return err, - }; - - if let Err(response) = Self::validate_operation_scope(active, scope) { - return response; - } - - // Validate range - if start >= end { - return KvResponse::Error { - error: KvError::InvalidRequest("start must be less than end".to_string()), - }; - } - - let scoped_start = Self::encode_scoped_key(&active.scoped_prefix, start); - let scoped_end = Self::encode_scoped_key(&active.scoped_prefix, end); - - match active.tx.delete_range(scoped_start, scoped_end) { - Ok(()) => { - active.mutation_count = active.mutation_count.saturating_add(1); - active.inventory_delta.mark_incomplete(); - KvResponse::DeleteRangeOk - } - Err(e) => KvResponse::Error { - error: Self::map_midge_error(e), - }, - } - } - - /// Scan a range of keys - fn handle_scan( - &mut self, - tx_id: u64, - scope: &KvResourceScope, - query: &ScanQuery, - ) -> KvResponse { - let active = match self.get_transaction_or_err(tx_id) { - Ok(tx) => tx, - Err(err) => return err, - }; - - if let Err(response) = Self::validate_operation_scope(active, scope) { - return response; - } - - let prefix = active.scoped_prefix.clone(); - let Some((midge_query, effective_limit)) = Self::build_scan_query(&prefix, query) else { - return KvResponse::ScanResult { - items: Vec::new(), - has_more: false, - }; - }; - match active.tx.scan(&midge_query) { - Ok(iterator) => Self::collect_scan_items(iterator, &prefix, effective_limit), - Err(e) => KvResponse::Error { - error: Self::map_midge_error(e), - }, - } - } - - fn build_scan_query(prefix: &[u8], query: &ScanQuery) -> Option<(cntryl_midge::Query, usize)> { - let (start_key, end_key) = Self::scan_bounds(prefix, query)?; - let effective_limit = query.limit.unwrap_or(MAX_SCAN_ITEMS).min(MAX_SCAN_ITEMS); - let mut midge_query = cntryl_midge::Query::new() - .prefix(Bytes::copy_from_slice(prefix)) - .start_key(Bytes::from(start_key)) - .end_key(Bytes::from(end_key)) - .limit(effective_limit.saturating_add(1)); - if query.reverse { - midge_query = midge_query.reverse(); - } - Some((midge_query, effective_limit)) - } - - fn collect_scan_items( - iterator: cntryl_midge::ScanIterator<'_>, - prefix: &[u8], - effective_limit: usize, - ) -> KvResponse { - let mut items = Vec::new(); - for entry in iterator { - let (key, value) = match entry { - Ok(row) => row, - Err(error) => { - return KvResponse::Error { - error: Self::map_midge_error(error), - }; - } - }; - let Some(user_key) = Self::strip_scoped_prefix(prefix, &key) else { - continue; - }; - items.push(KvPair { - key: Bytes::from(user_key), - value, - }); - } - let has_more = items.len() > effective_limit; - items.truncate(effective_limit); - KvResponse::ScanResult { items, has_more } - } - - /// Get active transaction or return error - fn get_transaction_or_err(&mut self, tx_id: u64) -> Result<&mut ActiveKvTx, KvResponse> { - let transaction = self - .transactions - .get_mut(&tx_id) - .ok_or_else(|| KvResponse::Error { - error: KvError::InvalidTxId, - })?; - transaction.last_activity = Instant::now(); - Ok(transaction) - } - - pub(crate) fn expire_idle_transactions(&mut self, ttl: Duration) -> Vec { - let now = Instant::now(); - let expired: Vec<_> = self - .transactions - .iter() - .filter_map(|(tx_id, transaction)| { - (now.saturating_duration_since(transaction.last_activity) >= ttl).then_some(*tx_id) - }) - .collect(); - for tx_id in &expired { - self.transactions.remove(tx_id); - } - expired - } - - pub(crate) fn rollback_transaction(&mut self, tx_id: u64) -> bool { - self.transactions.remove(&tx_id).is_some() - } - - #[must_use] - pub fn mutation_count_for_tx(&self, tx_id: u64) -> Option { - self.transactions.get(&tx_id).map(|tx| tx.mutation_count) - } - - #[must_use] - pub fn resource_scope_for_tx(&self, tx_id: u64) -> Option<(u64, String, String, String)> { - self.transactions.get(&tx_id).map(|tx| { - ( - u64::from(tx.column_family), - tx.scope.realm.clone(), - tx.scope.area.clone(), - tx.scope.resource.clone(), - ) - }) - } - - #[must_use] - pub fn active_transaction_scopes(&self) -> Vec<(u64, u64, String, String, String)> { - self.transactions - .iter() - .map(|(tx_id, tx)| { - ( - *tx_id, - u64::from(tx.column_family), - tx.scope.realm.clone(), - tx.scope.area.clone(), - tx.scope.resource.clone(), - ) - }) - .collect() - } - - #[must_use] - pub fn transaction_count(&self) -> usize { - self.transactions.len() - } - - fn validate_operation_scope( - active: &ActiveKvTx, - scope: &KvResourceScope, - ) -> Result<(), KvResponse> { - if scope.route_family != active.scope.route_family { - return Err(KvResponse::Error { - error: KvError::InvalidRouteFamily, - }); - } - if scope.realm != active.scope.realm { - return Err(KvResponse::Error { - error: KvError::RealmMismatch, - }); - } - if scope.area != active.scope.area || scope.resource != active.scope.resource { - return Err(KvResponse::Error { - error: KvError::TxScopeViolation { - expected: format!("{}/{}", active.scope.area, active.scope.resource), - actual: format!("{}/{}", scope.area, scope.resource), - }, - }); - } - - Ok(()) - } - - fn scan_bounds(prefix: &[u8], query: &ScanQuery) -> Option<(Vec, Vec)> { - if let (Some(start), Some(end)) = (&query.start, &query.end) { - let interval_is_empty = if query.reverse { - start <= end - } else { - start >= end - }; - if interval_is_empty { - return None; - } - } - - if query.reverse { - let lower = query.end.as_ref().map_or_else( - || prefix.to_vec(), - |key| Self::immediate_successor(Self::encode_scoped_key(prefix, key)), - ); - let upper = query.start.as_ref().map_or_else( - || Self::prefix_range_end(prefix), - |key| Self::immediate_successor(Self::encode_scoped_key(prefix, key)), - ); - Some((lower, upper)) - } else { - let lower = query.start.as_ref().map_or_else( - || prefix.to_vec(), - |key| Self::encode_scoped_key(prefix, key), - ); - let upper = query.end.as_ref().map_or_else( - || Self::prefix_range_end(prefix), - |key| Self::encode_scoped_key(prefix, key), - ); - Some((lower, upper)) - } - } - - fn immediate_successor(mut key: Vec) -> Vec { - key.push(0); - key - } - - pub(crate) fn realm_resource_prefix(realm: &str, area: &str, resource: &str) -> Vec { - let mut encoder = storage_key::domain_marker_encoder( - realm, - DomainKeyspace::Kv, - KV_KEY_SCOPE_MARKER, - area.len() + resource.len() + 2, - ); - storage_key::encode_bytes_segment_into(&mut encoder, area.as_bytes()); - storage_key::encode_bytes_segment_into(&mut encoder, resource.as_bytes()); - encoder.into_vec() - } - - pub(crate) fn inventory_metadata_key(realm: &str, area: &str, resource: &str) -> Vec { - let mut encoder = storage_key::domain_marker_encoder( - realm, - DomainKeyspace::Kv, - KV_INVENTORY_SCOPE_MARKER, - area.len() + resource.len() + 2, - ); - storage_key::encode_bytes_segment_into(&mut encoder, area.as_bytes()); - storage_key::encode_bytes_segment_into(&mut encoder, resource.as_bytes()); - encoder.into_vec() - } - - pub(crate) fn parse_inventory_metadata_key(key: &[u8]) -> Option<(String, String, String)> { - let (realm, suffix) = storage_key::split_domain_key(key, DomainKeyspace::Kv)?; - if suffix.first().copied()? != KV_INVENTORY_SCOPE_MARKER { - return None; - } - - let mut parts = suffix[1..].split(|byte| *byte == lexkey::LexKey::SEPARATOR); - let area = parts.next()?; - let resource = parts.next()?; - let trailing = parts.next(); - if area.is_empty() || resource.is_empty() || trailing != Some(&[]) || parts.next().is_some() - { - return None; - } - - Some(( - realm.to_string(), - String::from_utf8(area.to_vec()).ok()?, - String::from_utf8(resource.to_vec()).ok()?, - )) - } - - pub(crate) fn encode_inventory_estimate(estimate: KvInventoryEstimate) -> Vec { - let mut out = Vec::with_capacity(KV_INVENTORY_VALUE_LEN); - out.push(KV_INVENTORY_VALUE_VERSION); - out.push(u8::from(estimate.estimate_complete)); - out.extend_from_slice(&estimate.estimated_record_count.to_be_bytes()); - out.extend_from_slice(&estimate.estimated_storage_bytes.to_be_bytes()); - out - } - - pub(crate) fn decode_inventory_estimate(bytes: &[u8]) -> Result { - if bytes.len() != KV_INVENTORY_VALUE_LEN - || bytes.first().copied() != Some(KV_INVENTORY_VALUE_VERSION) - { - return Err("invalid KV inventory metadata value".to_string()); - } - - let estimated_record_count = u64::from_be_bytes( - bytes[KV_INVENTORY_RECORD_COUNT_RANGE] - .try_into() - .map_err(|_| "invalid KV inventory record count".to_string())?, - ); - let estimated_storage_bytes = u64::from_be_bytes( - bytes[KV_INVENTORY_STORAGE_BYTES_RANGE] - .try_into() - .map_err(|_| "invalid KV inventory storage estimate".to_string())?, - ); - - Ok(KvInventoryEstimate { - estimated_record_count, - estimated_storage_bytes, - estimate_complete: bytes[1] != 0, - }) - } - - fn apply_inventory_delta( - store: &MidgeEngine, - column_family: ColumnFamilyId, - scope: &KvResourceScope, - inventory_delta: &KvInventoryDelta, - ) -> Result<(), KvError> { - if inventory_delta.is_empty() { - return Ok(()); - } - - let key = Self::inventory_metadata_key(&scope.realm, &scope.area, &scope.resource); - let mut tx = store - .begin_tx(column_family, TransactionMode::ReadWrite) - .map_err(Self::map_midge_error)?; - let mut estimate = tx - .get(&key) - .map_err(Self::map_midge_error)? - .as_deref() - .map(Self::decode_inventory_estimate) - .transpose() - .map_err(KvError::BackendError)? - .unwrap_or_default(); - - for change in inventory_delta.key_changes.values() { - match (change.before_bytes, change.after_bytes) { - (None, Some(after)) => { - estimate.estimated_record_count = - estimate.estimated_record_count.saturating_add(1); - estimate.estimated_storage_bytes = estimate - .estimated_storage_bytes - .saturating_add(after as u64); - } - (Some(before), None) => { - estimate.estimated_record_count = - estimate.estimated_record_count.saturating_sub(1); - estimate.estimated_storage_bytes = estimate - .estimated_storage_bytes - .saturating_sub(before as u64); - } - (Some(before), Some(after)) => { - if after >= before { - estimate.estimated_storage_bytes = estimate - .estimated_storage_bytes - .saturating_add((after - before) as u64); - } else { - estimate.estimated_storage_bytes = estimate - .estimated_storage_bytes - .saturating_sub((before - after) as u64); - } - } - (None, None) => {} - } - } - - if inventory_delta.estimate_incomplete { - estimate.estimate_complete = false; - } - - tx.put(key, Self::encode_inventory_estimate(estimate), None) - .map_err(Self::map_midge_error)?; - tx.commit(cntryl_midge::WriteOptions::buffered()) - .map_err(Self::map_midge_error) - } } impl Actor for KvActor { type Message = KvMessage; - fn receive(&mut self, msg: Self::Message, ctx: &mut Context) { - let response = self.handle(msg); - let _ = ctx.reply(response); + fn receive(&mut self, message: Self::Message, context: &mut Context) { + let response = self.handle(message); + let _ = context.reply(response); } } diff --git a/src/domains/kv/actor/mutations.rs b/src/domains/kv/actor/mutations.rs new file mode 100644 index 00000000..d786174d --- /dev/null +++ b/src/domains/kv/actor/mutations.rs @@ -0,0 +1,149 @@ +//! Transaction-scoped GET and mutation operations. + +use super::KvActor; +use crate::domains::kv::{KvError, KvResourceScope, KvResponse}; +use bytes::Bytes; + +impl KvActor { + pub(super) fn handle_get( + &mut self, + tx_id: u64, + scope: &KvResourceScope, + key: &Bytes, + ) -> KvResponse { + let active = match self.scoped_transaction_or_err(tx_id, scope) { + Ok(tx) => tx, + Err(error) => return error, + }; + + let scoped_key = Self::encode_scoped_key(&active.scoped_prefix, key); + match active.tx.get(&scoped_key) { + Ok(Some(value)) => KvResponse::GetResult { + found: true, + value: Some(value), + }, + Ok(None) => KvResponse::GetResult { + found: false, + value: None, + }, + Err(error) => KvResponse::Error { + error: Self::map_midge_error(&error), + }, + } + } + + pub(super) fn handle_put( + &mut self, + tx_id: u64, + scope: &KvResourceScope, + key: &Bytes, + value: &Bytes, + ) -> KvResponse { + let active = match self.scoped_transaction_or_err(tx_id, scope) { + Ok(tx) => tx, + Err(error) => return error, + }; + + let scoped_key = Self::encode_scoped_key(&active.scoped_prefix, key); + match active.tx.put(scoped_key, value.to_vec(), None) { + Ok(()) => { + active.mutation_count = active.mutation_count.saturating_add(1); + active.inventory_delta.mark_incomplete(); + KvResponse::PutOk + } + Err(error) => KvResponse::Error { + error: Self::map_midge_error(&error), + }, + } + } + + pub(super) fn handle_insert( + &mut self, + tx_id: u64, + scope: &KvResourceScope, + key: &Bytes, + value: &Bytes, + ) -> KvResponse { + let active = match self.scoped_transaction_or_err(tx_id, scope) { + Ok(tx) => tx, + Err(error) => return error, + }; + + let scoped_key = Self::encode_scoped_key(&active.scoped_prefix, key); + match active.tx.get(&scoped_key) { + Ok(Some(_)) => KvResponse::Error { + error: KvError::AlreadyExists, + }, + Ok(None) => match active.tx.put(scoped_key, value.to_vec(), None) { + Ok(()) => { + active.mutation_count = active.mutation_count.saturating_add(1); + active + .inventory_delta + .record_insert(key, key.len() + value.len()); + KvResponse::InsertOk + } + Err(error) => KvResponse::Error { + error: Self::map_midge_error(&error), + }, + }, + Err(error) => KvResponse::Error { + error: Self::map_midge_error(&error), + }, + } + } + + pub(super) fn handle_delete( + &mut self, + tx_id: u64, + scope: &KvResourceScope, + key: &Bytes, + ) -> KvResponse { + let active = match self.scoped_transaction_or_err(tx_id, scope) { + Ok(tx) => tx, + Err(error) => return error, + }; + + let scoped_key = Self::encode_scoped_key(&active.scoped_prefix, key); + match active.tx.delete(scoped_key) { + Ok(()) => { + active.mutation_count = active.mutation_count.saturating_add(1); + active.inventory_delta.mark_incomplete(); + KvResponse::DeleteOk + } + Err(error) => KvResponse::Error { + error: Self::map_midge_error(&error), + }, + } + } + + pub(super) fn handle_delete_range( + &mut self, + tx_id: u64, + scope: &KvResourceScope, + start: &Bytes, + end: &Bytes, + ) -> KvResponse { + let active = match self.scoped_transaction_or_err(tx_id, scope) { + Ok(tx) => tx, + Err(error) => return error, + }; + if start >= end { + return KvResponse::Error { + error: KvError::InvalidRequest("start must be less than end".to_string()), + }; + } + + let scoped_start = Self::encode_scoped_key(&active.scoped_prefix, start); + let scoped_end = Self::encode_scoped_key(&active.scoped_prefix, end); + match active.tx.delete_range(scoped_start, scoped_end) { + Ok(()) => { + active.mutation_count = active.mutation_count.saturating_add(1); + active.inventory_delta.mark_incomplete(); + KvResponse::DeleteRangeOk + } + Err(error) => KvResponse::Error { + error: Self::map_midge_error(&error), + }, + } + } +} diff --git a/src/domains/kv/actor/scan.rs b/src/domains/kv/actor/scan.rs new file mode 100644 index 00000000..8ea35236 --- /dev/null +++ b/src/domains/kv/actor/scan.rs @@ -0,0 +1,189 @@ +//! Range scans with item, pagination, and wire-size bounds. + +use super::KvActor; +use crate::domains::kv::{KvError, KvPair, KvResourceScope, KvResponse, ScanQuery}; +use bytes::Bytes; + +pub(super) const MAX_SCAN_ITEMS: usize = 1_024; + +impl KvActor { + pub(super) fn handle_scan( + &mut self, + tx_id: u64, + scope: &KvResourceScope, + query: &ScanQuery, + ) -> KvResponse { + let active = match self.scoped_transaction_or_err(tx_id, scope) { + Ok(tx) => tx, + Err(error) => return error, + }; + + let prefix = active.scoped_prefix.clone(); + let Some((midge_query, effective_limit)) = Self::build_scan_query(&prefix, query) else { + return KvResponse::ScanResult { + items: Vec::new(), + has_more: false, + }; + }; + match active.tx.scan(&midge_query) { + Ok(iterator) => Self::collect_scan_items(iterator, &prefix, effective_limit), + Err(error) => KvResponse::Error { + error: Self::map_midge_error(&error), + }, + } + } + + fn build_scan_query(prefix: &[u8], query: &ScanQuery) -> Option<(cntryl_midge::Query, usize)> { + let (start_key, end_key) = Self::scan_bounds(prefix, query)?; + let effective_limit = query + .limit + .filter(|&limit| limit > 0) + .unwrap_or(MAX_SCAN_ITEMS) + .min(MAX_SCAN_ITEMS); + let mut midge_query = cntryl_midge::Query::new() + .prefix(Bytes::copy_from_slice(prefix)) + .start_key(Bytes::from(start_key)) + .end_key(Bytes::from(end_key)) + .limit(effective_limit.saturating_add(1)); + if query.reverse { + midge_query = midge_query.reverse(); + } + Some((midge_query, effective_limit)) + } + + fn truncated_key_for_error(key: &[u8]) -> String { + const MAX_ECHOED_KEY_BYTES: usize = 64; + + let head = &key[..key.len().min(MAX_ECHOED_KEY_BYTES)]; + let rendered = String::from_utf8_lossy(head); + if key.len() > MAX_ECHOED_KEY_BYTES { + format!("{rendered}...") + } else { + rendered.into_owned() + } + } + + fn collect_scan_items( + iterator: cntryl_midge::ScanIterator<'_>, + prefix: &[u8], + effective_limit: usize, + ) -> KvResponse { + let ceiling = crate::domains::kv::scan_wire_budget::kv_scan_response_byte_ceiling(); + let mut items: Vec = Vec::new(); + let mut used = 0usize; + let mut has_more = false; + let mut unresumable_boundary: Option = None; + for entry in iterator { + let (key, value) = match entry { + Ok(row) => row, + Err(error) => { + return KvResponse::Error { + error: Self::map_midge_error(&error), + }; + } + }; + let Some(user_key) = Self::strip_scoped_prefix(prefix, &key) else { + continue; + }; + if let Some(boundary) = unresumable_boundary.as_ref() { + return KvResponse::Error { + error: KvError::InvalidRequest(format!( + "scan key {} ({} bytes) cannot become a continuation start_key without \ + itself exceeding the request wire limit", + Self::truncated_key_for_error(boundary), + boundary.len() + )), + }; + } + if items.len() >= effective_limit { + has_more = true; + break; + } + let cost = crate::domains::kv::scan_wire_budget::kv_scan_item_wire_bytes( + user_key.len(), + value.len(), + ); + let unresumable = user_key.len() + > crate::domains::kv::scan_wire_budget::kv_scan_continuation_max_key_bytes(); + if used.saturating_add(cost) > ceiling { + if items.is_empty() { + return KvResponse::Error { + error: KvError::InvalidRequest(format!( + "scan pair {} ({} byte key) is {cost} wire bytes, exceeding the \ + {ceiling}-byte limit a scan response can return", + Self::truncated_key_for_error(&user_key), + user_key.len() + )), + }; + } + has_more = true; + break; + } + used = used.saturating_add(cost); + items.push(KvPair { + key: Bytes::from(user_key), + value, + }); + unresumable_boundary = if unresumable { + items.last().map(|item| item.key.clone()) + } else { + None + }; + } + KvResponse::ScanResult { items, has_more } + } + + fn scan_bounds(prefix: &[u8], query: &ScanQuery) -> Option<(Vec, Vec)> { + if let (Some(start), Some(end)) = (&query.start, &query.end) { + let interval_is_empty = if query.reverse { + start <= end + } else { + start >= end + }; + if interval_is_empty { + return None; + } + } + + if query.reverse { + let lower = query.end.as_ref().map_or_else( + || prefix.to_vec(), + |key| Self::immediate_successor(Self::encode_scoped_key(prefix, key)), + ); + let upper = query.start.as_ref().map_or_else( + || Self::prefix_range_end(prefix), + |key| { + let scoped = Self::encode_scoped_key(prefix, key); + if query.start_exclusive { + scoped + } else { + Self::immediate_successor(scoped) + } + }, + ); + Some((lower, upper)) + } else { + let lower = query.start.as_ref().map_or_else( + || prefix.to_vec(), + |key| { + let scoped = Self::encode_scoped_key(prefix, key); + if query.start_exclusive { + Self::immediate_successor(scoped) + } else { + scoped + } + }, + ); + let upper = query.end.as_ref().map_or_else( + || Self::prefix_range_end(prefix), + |key| Self::encode_scoped_key(prefix, key), + ); + Some((lower, upper)) + } + } + + fn immediate_successor(mut key: Vec) -> Vec { + key.push(0); + key + } +} diff --git a/src/domains/kv/actor/tests.rs b/src/domains/kv/actor/tests.rs index 70652b74..1cc25678 100644 --- a/src/domains/kv/actor/tests.rs +++ b/src/domains/kv/actor/tests.rs @@ -7,6 +7,56 @@ pub(super) fn test_actor() -> KvActor { } mod conflict_and_error_paths; -mod scope_and_scan; +mod inventory; +mod lifecycle; +mod range_and_pagination; +mod scope; mod state_model; -mod transaction_core; +mod wire_budget; +mod write_policy; + +pub(super) fn begin_with_scope(actor: &mut KvActor, scope: KvResourceScope) -> u64 { + let response = actor.handle(KvMessage::Begin { + scope, + mode: TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }); + let KvResponse::BeginOk { tx_id } = response else { + panic!("expected transaction begin, got {response:?}"); + }; + tx_id +} + +pub(super) fn put_scan_keys( + actor: &mut KvActor, + tx_id: u64, + scope: &KvResourceScope, + keys: &[&[u8]], +) { + for key in keys { + let response = actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: Bytes::copy_from_slice(key), + value: Bytes::copy_from_slice(key), + }); + assert!(matches!(response, KvResponse::PutOk)); + } +} + +pub(super) fn scan_keys( + actor: &mut KvActor, + tx_id: u64, + scope: &KvResourceScope, + query: ScanQuery, +) -> (Vec, bool) { + let response = actor.handle(KvMessage::Scan { + tx_id, + scope: scope.clone(), + query, + }); + let KvResponse::ScanResult { items, has_more } = response else { + panic!("expected scan result, got {response:?}"); + }; + (items.into_iter().map(|item| item.key).collect(), has_more) +} diff --git a/src/domains/kv/actor/tests/conflict_and_error_paths.rs b/src/domains/kv/actor/tests/conflict_and_error_paths.rs index a98ae826..43b23820 100644 --- a/src/domains/kv/actor/tests/conflict_and_error_paths.rs +++ b/src/domains/kv/actor/tests/conflict_and_error_paths.rs @@ -280,6 +280,7 @@ fn should_scan_empty_table_returns_empty_result() { end: None, limit: None, reverse: false, + start_exclusive: false, }, }); @@ -489,3 +490,24 @@ fn should_reject_empty_resource_in_follow_up_scope() { } )); } + +#[test] +fn should_classify_storage_timeout_as_backend_unavailable() { + // Arrange + // Midge bounds its synchronous runtime waits, so a slow storage op now + // returns a typed timeout where it previously blocked. Classifying by + // message text alone drops it into the generic bucket and reports + // transient saturation as a permanent failure. + let error = cntryl_midge::MidgeError::Timeout( + "runtime request Put request_id=42 exceeded response timeout 60s".to_string(), + ); + + // Act + let classification = KvActor::map_midge_error(&error); + + // Assert + assert!( + matches!(classification, KvError::BackendUnavailable(_)), + "a storage timeout is transient, got {classification:?}" + ); +} diff --git a/src/domains/kv/actor/tests/inventory.rs b/src/domains/kv/actor/tests/inventory.rs new file mode 100644 index 00000000..d5d28f9a --- /dev/null +++ b/src/domains/kv/actor/tests/inventory.rs @@ -0,0 +1,197 @@ +use super::*; + +#[test] +fn should_map_inventory_write_options_to_matching_local_or_cloud_class() { + // Arrange + let local_options = [ + cntryl_midge::WriteOptions::sync(), + cntryl_midge::WriteOptions::buffered(), + cntryl_midge::WriteOptions::best_effort(), + ]; + let cloud_options = [ + cntryl_midge::WriteOptions::cloud_async(), + cntryl_midge::WriteOptions::cloud_strict(), + ]; + + // Act + let local_inventory_options = local_options.map(KvActor::inventory_write_options); + let cloud_inventory_options = cloud_options.map(KvActor::inventory_write_options); + + // Assert + assert_eq!( + local_inventory_options, + [cntryl_midge::WriteOptions::buffered(); 3] + ); + assert_eq!( + cloud_inventory_options, + [cntryl_midge::WriteOptions::cloud_async(); 2] + ); +} + +#[test] +fn should_persist_inventory_estimate_after_commit_in_cloud_mode() { + // Arrange: a cloud-backed engine only accepts cloud_async()/cloud_strict() + // commits; sync()/buffered() are rejected as local-only. + let tempdir = tempfile::TempDir::new().expect("create cloud simulation directory"); + let store = Arc::new( + cntryl_midge::Engine::open( + cntryl_midge::OpenOptions::cloud_simulated( + tempdir.path(), + "fitz-kv-inventory-test", + "background", + ) + .build() + .expect("build cloud-simulated options"), + ) + .expect("open cloud-simulated engine"), + ); + store + .create_column_family("cf_1") + .expect("create route-family column family"); + let mut actor = KvActor::new(store.clone()); + let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "cloud-shared"); + + let KvResponse::BeginOk { tx_id } = actor.handle(KvMessage::Begin { + scope: scope.clone(), + mode: TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::cloud_async(), + }) else { + panic!("transaction should begin"); + }; + assert!(matches!( + actor.handle(KvMessage::Insert { + tx_id, + scope: scope.clone(), + key: Bytes::from_static(b"key"), + value: Bytes::from_static(b"value"), + }), + KvResponse::InsertOk + )); + + // Act + let commit = actor.handle(KvMessage::Commit { + tx_id, + scope: scope.clone(), + }); + + // Assert: the primary write always succeeds regardless of the inventory + // bug, so the real assertion is that the inventory estimate is actually + // persisted afterward. + assert!(matches!(commit, KvResponse::CommitOk)); + let inventory_key = KvActor::inventory_metadata_key(&scope.realm, &scope.area, &scope.resource); + let read_tx = store + .begin_tx(1, cntryl_midge::TransactionMode::ReadOnly) + .expect("begin inventory read transaction"); + let stored = read_tx + .get(&inventory_key) + .expect("read inventory metadata"); + let estimate = crate::domains::kv::inventory::decode_estimate( + stored + .as_deref() + .expect("inventory estimate should be persisted even in cloud mode"), + ) + .expect("decode persisted inventory estimate"); + assert_eq!(estimate.estimated_record_count, 1); +} + +#[test] +fn should_commit_disjoint_writes_without_inventory_conflict() { + // Arrange + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "shared"); + let begin = |actor: &mut KvActor| { + let KvResponse::BeginOk { tx_id } = actor.handle(KvMessage::Begin { + scope: scope.clone(), + mode: TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }) else { + panic!("transaction should begin"); + }; + tx_id + }; + let first = begin(&mut actor); + let second = begin(&mut actor); + for (tx_id, key) in [(first, "first"), (second, "second")] { + assert!(matches!( + actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: Bytes::from(key), + value: Bytes::from_static(b"value"), + }), + KvResponse::PutOk + )); + } + + // Act + let first_commit = actor.handle(KvMessage::Commit { + tx_id: first, + scope: scope.clone(), + }); + let second_commit = actor.handle(KvMessage::Commit { + tx_id: second, + scope, + }); + + // Assert + assert!(matches!(first_commit, KvResponse::CommitOk)); + assert!(matches!(second_commit, KvResponse::CommitOk)); +} + +#[test] +fn should_mark_inventory_incomplete_for_put_without_adding_a_hot_path_read() { + // Arrange + let mut actor = test_actor(); + let store = actor.store.clone(); + let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "conservative"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + assert!(matches!( + actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: Bytes::from_static(b"key"), + value: Bytes::from_static(b"value"), + }), + KvResponse::PutOk + )); + + // Act + let response = actor.handle(KvMessage::Commit { + tx_id, + scope: scope.clone(), + }); + + // Assert + assert!(matches!(response, KvResponse::CommitOk)); + let read_tx = store + .begin_tx(1, cntryl_midge::TransactionMode::ReadOnly) + .expect("begin inventory read transaction"); + let encoded = read_tx + .get(&KvActor::inventory_metadata_key( + &scope.realm, + &scope.area, + &scope.resource, + )) + .expect("read inventory estimate") + .expect("incomplete estimate should be persisted"); + let estimate = crate::domains::kv::inventory::decode_estimate(&encoded) + .expect("decode inventory estimate"); + assert!(!estimate.estimate_complete); +} + +#[test] +pub(super) fn should_encode_kv_scope_prefix_with_typed_segments() { + // Arrange + let expected = { + let mut bytes = b"acme\0kv\0".to_vec(); + bytes.push(KV_KEY_SCOPE_MARKER); + bytes.extend_from_slice(b"users\0profiles\0"); + bytes + }; + + // Act + let prefix = KvActor::realm_resource_prefix("acme", "users", "profiles"); + + // Assert + assert_eq!(prefix, expected); +} diff --git a/src/domains/kv/actor/tests/transaction_core.rs b/src/domains/kv/actor/tests/lifecycle.rs similarity index 54% rename from src/domains/kv/actor/tests/transaction_core.rs rename to src/domains/kv/actor/tests/lifecycle.rs index c18eae1f..7fe86c53 100644 --- a/src/domains/kv/actor/tests/transaction_core.rs +++ b/src/domains/kv/actor/tests/lifecycle.rs @@ -1,48 +1,5 @@ use super::*; -#[test] -fn should_commit_disjoint_writes_without_inventory_conflict() { - // Arrange - let mut actor = test_actor(); - let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "shared"); - let begin = |actor: &mut KvActor| { - let KvResponse::BeginOk { tx_id } = actor.handle(KvMessage::Begin { - scope: scope.clone(), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }) else { - panic!("transaction should begin"); - }; - tx_id - }; - let first = begin(&mut actor); - let second = begin(&mut actor); - for (tx_id, key) in [(first, "first"), (second, "second")] { - assert!(matches!( - actor.handle(KvMessage::Put { - tx_id, - scope: scope.clone(), - key: Bytes::from(key), - value: Bytes::from_static(b"value"), - }), - KvResponse::PutOk - )); - } - - // Act - let first_commit = actor.handle(KvMessage::Commit { - tx_id: first, - scope: scope.clone(), - }); - let second_commit = actor.handle(KvMessage::Commit { - tx_id: second, - scope, - }); - - // Assert - assert!(matches!(first_commit, KvResponse::CommitOk)); - assert!(matches!(second_commit, KvResponse::CommitOk)); -} #[test] pub(super) fn should_begin_transaction_for_resource() { // Arrange @@ -218,227 +175,6 @@ pub(super) fn should_preserve_kv_scope_given_follow_up_put_on_same_transaction() } } -#[test] -pub(super) fn should_reject_insert_when_key_exists() { - // Arrange - let mut actor = test_actor(); - let begin_response = actor.handle(KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "test".to_string(), - "kv".to_string(), - "table1".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }); - let KvResponse::BeginOk { tx_id } = begin_response else { - panic!("Expected BeginOk"); - }; - - let key = Bytes::from("testkey"); - actor.handle(KvMessage::Insert { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), - key: key.clone(), - value: Bytes::from("value1"), - }); - - // Act - Try to insert again - let response = actor.handle(KvMessage::Insert { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), - key: key.clone(), - value: Bytes::from("value2"), - }); - - // Assert - assert!(matches!( - response, - KvResponse::Error { - error: KvError::AlreadyExists - } - )); -} - -#[test] -pub(super) fn should_validate_delete_range_parameters() { - // Arrange - let mut actor = test_actor(); - let begin_response = actor.handle(KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "test".to_string(), - "kv".to_string(), - "table1".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }); - let KvResponse::BeginOk { tx_id } = begin_response else { - panic!("Expected BeginOk"); - }; - - // Act - End before start - let response = actor.handle(KvMessage::DeleteRange { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), - start: Bytes::from("z"), - end: Bytes::from("a"), - }); - - // Assert - assert!(matches!( - response, - KvResponse::Error { - error: KvError::InvalidRequest(_) - } - )); -} - -#[test] -pub(super) fn should_reject_route_family_zero() { - // Arrange - let mut actor = test_actor(); - - // Act - let result = actor.handle(KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(0), - "test".to_string(), - "kv".to_string(), - "table1".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }); - - // Assert - assert!(matches!( - result, - KvResponse::Error { - error: KvError::InvalidRouteFamily, - } - )); -} - -#[test] -pub(super) fn should_delete_existing_key() { - // Arrange - let mut actor = test_actor(); - let begin_response = actor.handle(KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "test".to_string(), - "kv".to_string(), - "table1".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }); - let KvResponse::BeginOk { tx_id } = begin_response else { - panic!("Expected BeginOk"); - }; - - let key = Bytes::from("delkey"); - actor.handle(KvMessage::Put { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), - key: key.clone(), - value: Bytes::from("value1"), - }); - - // Act - Delete the key - let delete_response = actor.handle(KvMessage::Delete { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), - key: key.clone(), - }); - - // Assert delete succeeds - assert!(matches!(delete_response, KvResponse::DeleteOk)); - - // Verify key is gone - let get_response = actor.handle(KvMessage::Get { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), - key: key.clone(), - }); - assert!(matches!( - get_response, - KvResponse::GetResult { - found: false, - value: None - } - )); -} - -#[test] -pub(super) fn should_scan_key_range() { - // Arrange - let mut actor = test_actor(); - let begin_response = actor.handle(KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "test".to_string(), - "kv".to_string(), - "table1".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }); - let KvResponse::BeginOk { tx_id } = begin_response else { - panic!("Expected BeginOk"); - }; - - // Add multiple keys - for i in 0..5 { - actor.handle(KvMessage::Put { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), - key: Bytes::from(format!("key{i:02}")), - value: Bytes::from(format!("value{i}")), - }); - } - - // Act - Scan range [key01, key04) - let response = actor.handle(KvMessage::Scan { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), - query: ScanQuery { - start: Some(Bytes::from("key01")), - end: Some(Bytes::from("key04")), - limit: None, - reverse: false, - }, - }); - - // Assert - match response { - KvResponse::ScanResult { items, .. } => { - assert!(items.len() >= 2); // At least key01, key02, key03 - } - _ => panic!("Expected ScanResult"), - } -} - -#[test] -pub(super) fn should_encode_kv_scope_prefix_with_typed_segments() { - // Arrange - let expected = { - let mut bytes = b"acme\0kv\0".to_vec(); - bytes.push(KV_KEY_SCOPE_MARKER); - bytes.extend_from_slice(b"users\0profiles\0"); - bytes - }; - - // Act - let prefix = KvActor::realm_resource_prefix("acme", "users", "profiles"); - - // Assert - assert_eq!(prefix, expected); -} - #[test] pub(super) fn should_commit_empty_transaction() { // Arrange @@ -724,136 +460,3 @@ pub(super) fn should_enforce_realm_isolation_for_kv() { _ => panic!("Expected realm-scoped values to be returned"), } } -#[test] -pub(super) fn should_reject_delete_range_with_invalid_bounds() { - // Arrange - let mut actor = test_actor(); - let begin_response = actor.handle(KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "test".to_string(), - "kv".to_string(), - "table1".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }); - let KvResponse::BeginOk { tx_id } = begin_response else { - panic!("Expected BeginOk"); - }; - - // Act - End < Start - let response = actor.handle(KvMessage::DeleteRange { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), - start: Bytes::from("zzz"), - end: Bytes::from("aaa"), - }); - - // Assert - assert!(matches!( - response, - KvResponse::Error { - error: KvError::InvalidRequest(_) - } - )); -} - -#[test] -pub(super) fn should_scan_with_limit() { - // Arrange - let mut actor = test_actor(); - let begin_response = actor.handle(KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "test".to_string(), - "kv".to_string(), - "table1".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }); - let KvResponse::BeginOk { tx_id } = begin_response else { - panic!("Expected BeginOk"); - }; - - // Add 10 keys - for i in 0..10 { - actor.handle(KvMessage::Put { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), - key: Bytes::from(format!("k{i:02}")), - value: Bytes::from(format!("v{i}")), - }); - } - - // Act - Scan with limit of 3 - let response = actor.handle(KvMessage::Scan { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), - query: ScanQuery { - start: None, - end: None, - limit: Some(3), - reverse: false, - }, - }); - - // Assert - match response { - KvResponse::ScanResult { items, has_more } => { - assert_eq!(items.len(), 3); - assert!(has_more); - } - _ => panic!("Expected ScanResult"), - } -} - -#[test] -pub(super) fn should_scan_reverse() { - // Arrange - let mut actor = test_actor(); - let begin_response = actor.handle(KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "test".to_string(), - "kv".to_string(), - "table1".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }); - let KvResponse::BeginOk { tx_id } = begin_response else { - panic!("Expected BeginOk"); - }; - - // Add keys - for i in 0..5 { - actor.handle(KvMessage::Put { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), - key: Bytes::from(format!("k{i}")), - value: Bytes::from(format!("v{i}")), - }); - } - - // Act - Scan reverse - let response = actor.handle(KvMessage::Scan { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), - query: ScanQuery { - start: None, - end: None, - limit: None, - reverse: true, - }, - }); - - // Assert - Just verify it returns results (order depends on storage) - match response { - KvResponse::ScanResult { items, .. } => { - assert!(!items.is_empty()); - } - _ => panic!("Expected ScanResult"), - } -} diff --git a/src/domains/kv/actor/tests/range_and_pagination.rs b/src/domains/kv/actor/tests/range_and_pagination.rs new file mode 100644 index 00000000..ec7a2812 --- /dev/null +++ b/src/domains/kv/actor/tests/range_and_pagination.rs @@ -0,0 +1,590 @@ +use super::*; + +#[test] +fn should_treat_explicit_zero_limit_as_unbounded_not_a_dead_end() { + // Arrange + // `limit=0` is a legal encoding on the wire (has_limit=1, limit=0), but + // treated literally it makes every SCAN return zero items with + // `has_more=1` and no key to resume from - a request the client can never + // make progress on, no matter how many times it retries. An explicit zero + // is therefore folded into "no limit supplied", matching what an omitted + // limit already means. + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "zero-limit"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + assert!(matches!( + actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: Bytes::from_static(b"only-key"), + value: Bytes::from_static(b"value"), + }), + KvResponse::PutOk + )); + + // Act + let response = actor.handle(KvMessage::Scan { + tx_id, + scope, + query: ScanQuery { + start: None, + end: None, + limit: Some(0), + reverse: false, + start_exclusive: false, + }, + }); + + // Assert + let KvResponse::ScanResult { items, has_more } = response else { + panic!("expected ScanResult, got {response:?}"); + }; + assert_eq!( + items.len(), + 1, + "an explicit zero limit must not starve the page" + ); + assert!(!has_more); +} + +#[test] +fn should_cap_scan_when_client_omits_limit() { + // Arrange + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "bounded-scan"); + let KvResponse::BeginOk { tx_id } = actor.handle(KvMessage::Begin { + scope: scope.clone(), + mode: TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }) else { + panic!("transaction should begin"); + }; + for index in 0..=MAX_SCAN_ITEMS { + assert!(matches!( + actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: Bytes::from(format!("key-{index:04}")), + value: Bytes::from_static(b"value"), + }), + KvResponse::PutOk + )); + } + + // Act + let response = actor.handle(KvMessage::Scan { + tx_id, + scope, + query: ScanQuery { + start: None, + end: None, + limit: None, + reverse: false, + start_exclusive: false, + }, + }); + + // Assert + let KvResponse::ScanResult { items, has_more } = response else { + panic!("scan should succeed"); + }; + assert_eq!(items.len(), MAX_SCAN_ITEMS); + assert!(has_more); +} + +#[test] +fn should_apply_forward_plus_reverse_scan_boundaries() { + // Arrange + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + put_scan_keys(&mut actor, tx_id, &scope, &[b"a", b"b", b"c", b"c\0", b"d"]); + + // Act + let (forward, _) = scan_keys( + &mut actor, + tx_id, + &scope, + ScanQuery { + start: Some(Bytes::from_static(b"b")), + end: Some(Bytes::from_static(b"d")), + limit: None, + reverse: false, + start_exclusive: false, + }, + ); + let (reverse, _) = scan_keys( + &mut actor, + tx_id, + &scope, + ScanQuery { + start: Some(Bytes::from_static(b"c")), + end: Some(Bytes::from_static(b"a")), + limit: None, + reverse: true, + start_exclusive: false, + }, + ); + + // Assert + assert_eq!( + forward, + [b"b".as_slice(), b"c", b"c\0"].map(Bytes::copy_from_slice) + ); + assert_eq!(reverse, [b"c".as_slice(), b"b"].map(Bytes::copy_from_slice)); +} + +#[test] +fn should_handle_every_omitted_scan_bound_combination() { + // Arrange + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + put_scan_keys(&mut actor, tx_id, &scope, &[b"a", b"b", b"c"]); + + // Act + let queries = [ + (Some(Bytes::from_static(b"b")), None, false), + (None, Some(Bytes::from_static(b"c")), false), + (Some(Bytes::from_static(b"b")), None, true), + (None, Some(Bytes::from_static(b"b")), true), + ]; + let results = queries.map(|(start, end, reverse)| { + scan_keys( + &mut actor, + tx_id, + &scope, + ScanQuery { + start, + end, + limit: None, + reverse, + start_exclusive: false, + }, + ) + .0 + }); + + // Assert + assert_eq!( + results[0], + [Bytes::from_static(b"b"), Bytes::from_static(b"c")] + ); + assert_eq!( + results[1], + [Bytes::from_static(b"a"), Bytes::from_static(b"b")] + ); + assert_eq!( + results[2], + [Bytes::from_static(b"b"), Bytes::from_static(b"a")] + ); + assert_eq!(results[3], [Bytes::from_static(b"c")]); +} + +#[test] +fn should_lower_reverse_exact_bounds_around_binary_successors() { + // Arrange + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + put_scan_keys(&mut actor, tx_id, &scope, &[b"k", b"k\0", b"k\0\0", b"l"]); + + // Act + let (keys, _) = scan_keys( + &mut actor, + tx_id, + &scope, + ScanQuery { + start: Some(Bytes::from_static(b"k\0")), + end: Some(Bytes::from_static(b"k")), + limit: None, + reverse: true, + start_exclusive: false, + }, + ); + + // Assert + assert_eq!(keys, [Bytes::from_static(b"k\0")]); +} + +#[test] +fn should_report_has_more_for_limited_scans_in_both_directions() { + // Arrange + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + put_scan_keys(&mut actor, tx_id, &scope, &[b"a", b"b", b"c"]); + + // Act + let forward = scan_keys( + &mut actor, + tx_id, + &scope, + ScanQuery { + start: None, + end: None, + limit: Some(2), + reverse: false, + start_exclusive: false, + }, + ); + let reverse = scan_keys( + &mut actor, + tx_id, + &scope, + ScanQuery { + start: None, + end: None, + limit: Some(3), + reverse: true, + start_exclusive: false, + }, + ); + + // Assert + assert_eq!( + forward, + ( + vec![Bytes::from_static(b"a"), Bytes::from_static(b"b")], + true + ) + ); + assert_eq!( + reverse, + ( + vec![ + Bytes::from_static(b"c"), + Bytes::from_static(b"b"), + Bytes::from_static(b"a"), + ], + false, + ) + ); +} + +#[test] +fn should_treat_zero_limit_as_unbounded_for_an_empty_match_set() { + // Arrange + // + // `limit=0` is folded into "no limit supplied" (unbounded), not a dead + // end: see `should_treat_explicit_zero_limit_as_unbounded_not_a_dead_end` + // for the populated case. This test covers the other half - an + // unbounded scan over a range with no matches must still report zero + // items and `has_more=false`, not the old dead-end `has_more=true`. + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + put_scan_keys(&mut actor, tx_id, &scope, &[b"a"]); + + // Act + let empty = scan_keys( + &mut actor, + tx_id, + &scope, + ScanQuery { + start: Some(Bytes::from_static(b"z")), + end: None, + limit: Some(0), + reverse: false, + start_exclusive: false, + }, + ); + + // Assert + assert_eq!(empty, (Vec::new(), false)); +} + +#[test] +fn should_return_empty_success_for_equal_or_inverted_scan_intervals() { + // Arrange + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + put_scan_keys(&mut actor, tx_id, &scope, &[b"a", b"b"]); + + // Act + let forward = scan_keys( + &mut actor, + tx_id, + &scope, + ScanQuery { + start: Some(Bytes::from_static(b"b")), + end: Some(Bytes::from_static(b"a")), + limit: None, + reverse: false, + start_exclusive: false, + }, + ); + let reverse = scan_keys( + &mut actor, + tx_id, + &scope, + ScanQuery { + start: Some(Bytes::from_static(b"a")), + end: Some(Bytes::from_static(b"b")), + limit: None, + reverse: true, + start_exclusive: false, + }, + ); + + // Assert + assert_eq!(forward, (Vec::new(), false)); + assert_eq!(reverse, (Vec::new(), false)); +} + +#[test] +fn should_scan_keys_that_begin_with_the_range_end_marker() { + // Arrange + // KV appends raw, unencoded user bytes after a lexkey-encoded prefix, so a + // user key may begin with 0xff - the same byte lexkey uses as its range end + // marker. Bounding the scan with `prefix || 0xff` therefore sorts such keys + // outside their own resource: the write succeeds and the key is then + // invisible to every scan, which is silent data loss from the client's + // point of view. + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "end-marker-keys"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + let keys: [Vec; 5] = [ + b"aaa".to_vec(), + vec![0x80, 0x80], + vec![0xfe, 0xfe], + vec![0xff, 0x00], + vec![0xff, 0xff], + ]; + for key in &keys { + assert!(matches!( + actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: Bytes::from(key.clone()), + value: Bytes::from_static(b"v"), + }), + KvResponse::PutOk + )); + } + + // Act + let (scanned, _) = scan_keys( + &mut actor, + tx_id, + &scope, + ScanQuery { + start: None, + end: None, + limit: None, + reverse: false, + start_exclusive: false, + }, + ); + + // Assert + assert_eq!( + scanned.len(), + keys.len(), + "every stored key must be scannable, got {scanned:?}" + ); + for key in &keys { + assert!( + scanned.iter().any(|found| found.as_ref() == key.as_slice()), + "key {key:?} was stored but is invisible to scan" + ); + } +} + +#[test] +fn should_page_a_byte_bounded_scan_to_completion_via_start_key() { + // Arrange + // An omitted limit does not mean unlimited: a page also ends at the + // response frame budget. The spec's continuation rule must therefore work + // for a client that supplied no limit at all - re-issue with `start_key` + // set to the last key returned, which is inclusive and so repeats as the + // first item of the next page. + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "paged-scan"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + let value = Bytes::from(vec![b'v'; 1024]); + let total = 300; + for index in 0..total { + assert!(matches!( + actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: Bytes::from(format!("key-{index:04}")), + value: value.clone(), + }), + KvResponse::PutOk + )); + } + + // Act + let mut seen: Vec = Vec::new(); + let mut start: Option = None; + let mut pages = 0; + loop { + let response = actor.handle(KvMessage::Scan { + tx_id, + scope: scope.clone(), + query: ScanQuery { + start: start.clone(), + end: None, + limit: None, + reverse: false, + start_exclusive: false, + }, + }); + let KvResponse::ScanResult { items, has_more } = response else { + panic!("scan should succeed, got {response:?}"); + }; + pages += 1; + assert!(pages < 50, "pagination must converge"); + assert!(!items.is_empty(), "each page must make forward progress"); + + for item in &items { + // `start_key` is inclusive, so the first item of a continuation + // repeats the previous page's last key. + if seen.last() == Some(&item.key) { + continue; + } + seen.push(item.key.clone()); + } + if !has_more { + break; + } + start = Some(items.last().expect("non-empty page").key.clone()); + } + + // Assert + assert!(pages > 1, "1 KiB values must not fit in a single page"); + assert_eq!( + seen.len(), + total, + "following has_more must recover every key" + ); + for index in 0..total { + let expected = Bytes::from(format!("key-{index:04}")); + assert!(seen.contains(&expected), "key {index} was never returned"); + } +} + +#[test] +fn should_advance_pagination_when_only_one_pair_fits_a_page() { + // Arrange + // Two adjacent pairs that are each wire-valid but cannot share a page. The + // first page returns only pair A. Resuming with an inclusive `start_key` + // returns pair A again, forever: the documented procedure never reaches + // pair B, so a compliant client loops and the data is unreachable. + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "single-pair-pages"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + let big = Bytes::from(vec![b'v'; 40_000]); + for key in ["key-a", "key-b"] { + assert!(matches!( + actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: Bytes::from(key), + value: big.clone(), + }), + KvResponse::PutOk + )); + } + + // Act + let mut seen: Vec = Vec::new(); + let mut start: Option = None; + let mut exclusive = false; + for _ in 0..8 { + let response = actor.handle(KvMessage::Scan { + tx_id, + scope: scope.clone(), + query: ScanQuery { + start: start.clone(), + end: None, + limit: None, + reverse: false, + start_exclusive: exclusive, + }, + }); + let KvResponse::ScanResult { items, has_more } = response else { + panic!("scan should succeed"); + }; + for item in &items { + if !seen.contains(&item.key) { + seen.push(item.key.clone()); + } + } + if !has_more { + break; + } + start = Some(items.last().expect("non-empty page").key.clone()); + exclusive = true; + } + + // Assert + assert_eq!( + seen.len(), + 2, + "pagination must reach both pairs, saw {seen:?}" + ); +} + +#[test] +fn should_page_forward_without_the_exclusive_flag_via_successor_key() { + // Arrange + // The documented fallback for clients that cannot yet encode + // `start_exclusive`: resume from the last returned key followed by a single + // 0x00 byte, which is that key's immediate successor and so an exclusive + // resume using only fields those clients already send. Forward scans must + // therefore paginate to completion today, with no wire change. + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "successor-paging"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + let big = Bytes::from(vec![b'v'; 40_000]); + for key in ["key-a", "key-b", "key-c"] { + assert!(matches!( + actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: Bytes::from(key), + value: big.clone(), + }), + KvResponse::PutOk + )); + } + + // Act + let mut seen: Vec = Vec::new(); + let mut start: Option = None; + for _ in 0..10 { + let response = actor.handle(KvMessage::Scan { + tx_id, + scope: scope.clone(), + query: ScanQuery { + start: start.clone(), + end: None, + limit: None, + reverse: false, + // Deliberately never set: this is the old-client path. + start_exclusive: false, + }, + }); + let KvResponse::ScanResult { items, has_more } = response else { + panic!("scan should succeed"); + }; + for item in &items { + if !seen.contains(&item.key) { + seen.push(item.key.clone()); + } + } + if !has_more { + break; + } + let mut resume = items.last().expect("non-empty page").key.to_vec(); + resume.push(0); + start = Some(Bytes::from(resume)); + } + + // Assert + assert_eq!(seen.len(), 3, "forward paging must complete, saw {seen:?}"); +} diff --git a/src/domains/kv/actor/tests/scope.rs b/src/domains/kv/actor/tests/scope.rs new file mode 100644 index 00000000..cfd3a5cb --- /dev/null +++ b/src/domains/kv/actor/tests/scope.rs @@ -0,0 +1,123 @@ +use super::*; + +#[test] +fn should_return_named_transaction_introspection_values() { + // Arrange + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + + // Act + let resource_scope = actor.resource_scope_for_tx(tx_id); + let snapshots = actor.active_transaction_snapshots(); + + // Assert + assert_eq!(resource_scope, Some(scope.clone())); + assert_eq!(snapshots.len(), 1); + assert_eq!(snapshots[0].tx_id, tx_id); + assert_eq!(snapshots[0].scope, scope); +} + +#[test] +fn should_reject_kv_put_given_realm_mismatch_without_mutation() { + // Arrange + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "realm-a", "area", "table"); + let tx_id = begin_with_scope(&mut actor, scope); + + // Act + let response = actor.handle(KvMessage::Put { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "realm-b", "area", "table"), + key: Bytes::from_static(b"key"), + value: Bytes::from_static(b"value"), + }); + + // Assert + assert!(matches!( + response, + KvResponse::Error { + error: KvError::RealmMismatch + } + )); + assert_eq!(actor.mutation_count_for_tx(tx_id), Some(0)); +} + +#[test] +fn should_reject_operation_with_area_mismatching_transaction_without_mutation() { + // Arrange + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area-a", "table"); + let tx_id = begin_with_scope(&mut actor, scope); + + // Act + let response = actor.handle(KvMessage::Put { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "realm", "area-b", "table"), + key: Bytes::from_static(b"key"), + value: Bytes::from_static(b"value"), + }); + + // Assert + assert!(matches!( + response, + KvResponse::Error { + error: KvError::TxScopeViolation { .. } + } + )); + assert_eq!(actor.mutation_count_for_tx(tx_id), Some(0)); +} + +#[test] +fn should_reject_kv_commit_given_any_scope_component_mismatch() { + // Arrange + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + + // Act + let response = actor.handle(KvMessage::Commit { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "other", "area", "table"), + }); + + // Assert + assert!(matches!( + response, + KvResponse::Error { + error: KvError::RealmMismatch + } + )); + assert_eq!(actor.transaction_count(), 1); + assert!(matches!( + actor.handle(KvMessage::Rollback { tx_id, scope }), + KvResponse::RollbackOk + )); +} + +#[test] +fn should_keep_transaction_active_when_rollback_scope_mismatches() { + // Arrange + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + + // Act + let response = actor.handle(KvMessage::Rollback { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "realm", "other", "table"), + }); + + // Assert + assert!(matches!( + response, + KvResponse::Error { + error: KvError::TxScopeViolation { .. } + } + )); + assert_eq!(actor.transaction_count(), 1); + assert!(matches!( + actor.handle(KvMessage::Rollback { tx_id, scope }), + KvResponse::RollbackOk + )); +} diff --git a/src/domains/kv/actor/tests/scope_and_scan.rs b/src/domains/kv/actor/tests/scope_and_scan.rs deleted file mode 100644 index 91b8dee3..00000000 --- a/src/domains/kv/actor/tests/scope_and_scan.rs +++ /dev/null @@ -1,427 +0,0 @@ -use super::*; - -#[test] -fn should_cap_scan_when_client_omits_limit() { - // Arrange - let mut actor = test_actor(); - let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "bounded-scan"); - let KvResponse::BeginOk { tx_id } = actor.handle(KvMessage::Begin { - scope: scope.clone(), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }) else { - panic!("transaction should begin"); - }; - for index in 0..=MAX_SCAN_ITEMS { - assert!(matches!( - actor.handle(KvMessage::Put { - tx_id, - scope: scope.clone(), - key: Bytes::from(format!("key-{index:04}")), - value: Bytes::from_static(b"value"), - }), - KvResponse::PutOk - )); - } - - // Act - let response = actor.handle(KvMessage::Scan { - tx_id, - scope, - query: ScanQuery { - start: None, - end: None, - limit: None, - reverse: false, - }, - }); - - // Assert - let KvResponse::ScanResult { items, has_more } = response else { - panic!("scan should succeed"); - }; - assert_eq!(items.len(), MAX_SCAN_ITEMS); - assert!(has_more); -} -fn begin_with_scope(actor: &mut KvActor, scope: KvResourceScope) -> u64 { - let response = actor.handle(KvMessage::Begin { - scope, - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }); - let KvResponse::BeginOk { tx_id } = response else { - panic!("expected transaction begin, got {response:?}"); - }; - tx_id -} - -fn put_scan_keys(actor: &mut KvActor, tx_id: u64, scope: &KvResourceScope, keys: &[&[u8]]) { - for key in keys { - let response = actor.handle(KvMessage::Put { - tx_id, - scope: scope.clone(), - key: Bytes::copy_from_slice(key), - value: Bytes::copy_from_slice(key), - }); - assert!(matches!(response, KvResponse::PutOk)); - } -} - -fn scan_keys( - actor: &mut KvActor, - tx_id: u64, - scope: &KvResourceScope, - query: ScanQuery, -) -> (Vec, bool) { - let response = actor.handle(KvMessage::Scan { - tx_id, - scope: scope.clone(), - query, - }); - let KvResponse::ScanResult { items, has_more } = response else { - panic!("expected scan result, got {response:?}"); - }; - (items.into_iter().map(|item| item.key).collect(), has_more) -} - -#[test] -fn should_reject_kv_put_given_realm_mismatch_without_mutation() { - // Arrange - let mut actor = test_actor(); - let scope = KvResourceScope::new(RouteFamily::new(1), "realm-a", "area", "table"); - let tx_id = begin_with_scope(&mut actor, scope); - - // Act - let response = actor.handle(KvMessage::Put { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "realm-b", "area", "table"), - key: Bytes::from_static(b"key"), - value: Bytes::from_static(b"value"), - }); - - // Assert - assert!(matches!( - response, - KvResponse::Error { - error: KvError::RealmMismatch - } - )); - assert_eq!(actor.mutation_count_for_tx(tx_id), Some(0)); -} - -#[test] -fn should_reject_operation_with_area_mismatching_transaction_without_mutation() { - // Arrange - let mut actor = test_actor(); - let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area-a", "table"); - let tx_id = begin_with_scope(&mut actor, scope); - - // Act - let response = actor.handle(KvMessage::Put { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "realm", "area-b", "table"), - key: Bytes::from_static(b"key"), - value: Bytes::from_static(b"value"), - }); - - // Assert - assert!(matches!( - response, - KvResponse::Error { - error: KvError::TxScopeViolation { .. } - } - )); - assert_eq!(actor.mutation_count_for_tx(tx_id), Some(0)); -} - -#[test] -fn should_reject_kv_commit_given_any_scope_component_mismatch() { - // Arrange - let mut actor = test_actor(); - let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); - let tx_id = begin_with_scope(&mut actor, scope.clone()); - - // Act - let response = actor.handle(KvMessage::Commit { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "other", "area", "table"), - }); - - // Assert - assert!(matches!( - response, - KvResponse::Error { - error: KvError::RealmMismatch - } - )); - assert_eq!(actor.transaction_count(), 1); - assert!(matches!( - actor.handle(KvMessage::Rollback { tx_id, scope }), - KvResponse::RollbackOk - )); -} - -#[test] -fn should_keep_transaction_active_when_rollback_scope_mismatches() { - // Arrange - let mut actor = test_actor(); - let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); - let tx_id = begin_with_scope(&mut actor, scope.clone()); - - // Act - let response = actor.handle(KvMessage::Rollback { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "realm", "other", "table"), - }); - - // Assert - assert!(matches!( - response, - KvResponse::Error { - error: KvError::TxScopeViolation { .. } - } - )); - assert_eq!(actor.transaction_count(), 1); - assert!(matches!( - actor.handle(KvMessage::Rollback { tx_id, scope }), - KvResponse::RollbackOk - )); -} - -#[test] -fn should_apply_forward_plus_reverse_scan_boundaries() { - // Arrange - let mut actor = test_actor(); - let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); - let tx_id = begin_with_scope(&mut actor, scope.clone()); - put_scan_keys(&mut actor, tx_id, &scope, &[b"a", b"b", b"c", b"c\0", b"d"]); - - // Act - let (forward, _) = scan_keys( - &mut actor, - tx_id, - &scope, - ScanQuery { - start: Some(Bytes::from_static(b"b")), - end: Some(Bytes::from_static(b"d")), - limit: None, - reverse: false, - }, - ); - let (reverse, _) = scan_keys( - &mut actor, - tx_id, - &scope, - ScanQuery { - start: Some(Bytes::from_static(b"c")), - end: Some(Bytes::from_static(b"a")), - limit: None, - reverse: true, - }, - ); - - // Assert - assert_eq!( - forward, - [b"b".as_slice(), b"c", b"c\0"].map(Bytes::copy_from_slice) - ); - assert_eq!(reverse, [b"c".as_slice(), b"b"].map(Bytes::copy_from_slice)); -} - -#[test] -fn should_handle_every_omitted_scan_bound_combination() { - // Arrange - let mut actor = test_actor(); - let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); - let tx_id = begin_with_scope(&mut actor, scope.clone()); - put_scan_keys(&mut actor, tx_id, &scope, &[b"a", b"b", b"c"]); - - // Act - let queries = [ - (Some(Bytes::from_static(b"b")), None, false), - (None, Some(Bytes::from_static(b"c")), false), - (Some(Bytes::from_static(b"b")), None, true), - (None, Some(Bytes::from_static(b"b")), true), - ]; - let results = queries.map(|(start, end, reverse)| { - scan_keys( - &mut actor, - tx_id, - &scope, - ScanQuery { - start, - end, - limit: None, - reverse, - }, - ) - .0 - }); - - // Assert - assert_eq!( - results[0], - [Bytes::from_static(b"b"), Bytes::from_static(b"c")] - ); - assert_eq!( - results[1], - [Bytes::from_static(b"a"), Bytes::from_static(b"b")] - ); - assert_eq!( - results[2], - [Bytes::from_static(b"b"), Bytes::from_static(b"a")] - ); - assert_eq!(results[3], [Bytes::from_static(b"c")]); -} - -#[test] -fn should_lower_reverse_exact_bounds_around_binary_successors() { - // Arrange - let mut actor = test_actor(); - let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); - let tx_id = begin_with_scope(&mut actor, scope.clone()); - put_scan_keys(&mut actor, tx_id, &scope, &[b"k", b"k\0", b"k\0\0", b"l"]); - - // Act - let (keys, _) = scan_keys( - &mut actor, - tx_id, - &scope, - ScanQuery { - start: Some(Bytes::from_static(b"k\0")), - end: Some(Bytes::from_static(b"k")), - limit: None, - reverse: true, - }, - ); - - // Assert - assert_eq!(keys, [Bytes::from_static(b"k\0")]); -} - -#[test] -fn should_report_has_more_for_limited_scans_in_both_directions() { - // Arrange - let mut actor = test_actor(); - let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); - let tx_id = begin_with_scope(&mut actor, scope.clone()); - put_scan_keys(&mut actor, tx_id, &scope, &[b"a", b"b", b"c"]); - - // Act - let forward = scan_keys( - &mut actor, - tx_id, - &scope, - ScanQuery { - start: None, - end: None, - limit: Some(2), - reverse: false, - }, - ); - let reverse = scan_keys( - &mut actor, - tx_id, - &scope, - ScanQuery { - start: None, - end: None, - limit: Some(3), - reverse: true, - }, - ); - - // Assert - assert_eq!( - forward, - ( - vec![Bytes::from_static(b"a"), Bytes::from_static(b"b")], - true - ) - ); - assert_eq!( - reverse, - ( - vec![ - Bytes::from_static(b"c"), - Bytes::from_static(b"b"), - Bytes::from_static(b"a"), - ], - false, - ) - ); -} - -#[test] -fn should_report_matches_without_returning_items_for_zero_limit() { - // Arrange - let mut actor = test_actor(); - let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); - let tx_id = begin_with_scope(&mut actor, scope.clone()); - put_scan_keys(&mut actor, tx_id, &scope, &[b"a"]); - - // Act - let populated = scan_keys( - &mut actor, - tx_id, - &scope, - ScanQuery { - start: None, - end: None, - limit: Some(0), - reverse: false, - }, - ); - let empty = scan_keys( - &mut actor, - tx_id, - &scope, - ScanQuery { - start: Some(Bytes::from_static(b"z")), - end: None, - limit: Some(0), - reverse: false, - }, - ); - - // Assert - assert_eq!(populated, (Vec::new(), true)); - assert_eq!(empty, (Vec::new(), false)); -} - -#[test] -fn should_return_empty_success_for_equal_or_inverted_scan_intervals() { - // Arrange - let mut actor = test_actor(); - let scope = KvResourceScope::new(RouteFamily::new(1), "realm", "area", "table"); - let tx_id = begin_with_scope(&mut actor, scope.clone()); - put_scan_keys(&mut actor, tx_id, &scope, &[b"a", b"b"]); - - // Act - let forward = scan_keys( - &mut actor, - tx_id, - &scope, - ScanQuery { - start: Some(Bytes::from_static(b"b")), - end: Some(Bytes::from_static(b"a")), - limit: None, - reverse: false, - }, - ); - let reverse = scan_keys( - &mut actor, - tx_id, - &scope, - ScanQuery { - start: Some(Bytes::from_static(b"a")), - end: Some(Bytes::from_static(b"b")), - limit: None, - reverse: true, - }, - ); - - // Assert - assert_eq!(forward, (Vec::new(), false)); - assert_eq!(reverse, (Vec::new(), false)); -} diff --git a/src/domains/kv/actor/tests/wire_budget.rs b/src/domains/kv/actor/tests/wire_budget.rs new file mode 100644 index 00000000..da2c7bdf --- /dev/null +++ b/src/domains/kv/actor/tests/wire_budget.rs @@ -0,0 +1,240 @@ +use super::*; + +#[test] +fn should_bound_scan_response_to_one_wire_frame() { + // Arrange + // A scan response is carried as one TLV value with a u16 length. The item + // cap alone does not bound it: 1 KiB values overflow the frame at roughly + // 63 items, far below the 1,024-item ceiling, and a client that omits + // `limit` takes that default. Every pair here is individually legal; only + // the aggregate is unencodable. + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "wire-bounded-scan"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + let value = Bytes::from(vec![b'v'; 1024]); + for index in 0..300 { + assert!(matches!( + actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: Bytes::from(format!("key-{index:04}")), + value: value.clone(), + }), + KvResponse::PutOk + )); + } + + // Act + let response = actor.handle(KvMessage::Scan { + tx_id, + scope, + query: ScanQuery { + start: None, + end: None, + limit: None, + reverse: false, + start_exclusive: false, + }, + }); + + // Assert + let KvResponse::ScanResult { items, has_more } = response else { + panic!("scan should succeed, got {response:?}"); + }; + let encoded = crate::dispatch::protocol::kv::encode_response(&KvResponse::ScanResult { + items: items.clone(), + has_more, + }); + assert!( + u16::try_from(encoded.len()).is_ok(), + "scan response is {} bytes, past the {}-byte TLV value limit", + encoded.len(), + u16::MAX + ); + assert!(!items.is_empty(), "the page must make forward progress"); + assert!( + has_more, + "a truncated page must tell the client to continue" + ); +} + +#[test] +fn should_refuse_a_key_that_cannot_become_a_continuation_boundary() { + // Arrange + // A key can be large enough to fit its own PUT and to fit once inside a + // SCAN response, yet still be too large to safely echo back as + // `start_key` in a follow-up SCAN request - the request has the same + // wire ceiling as the response. Manufacturing `has_more=1` for such a + // key would hand the client a page it can never resume past, silently + // stranding every later key. This must fail loudly at the boundary + // instead, exactly as an unencodable pair already does. + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "huge-key-resume"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + // Lexicographically first, so it lands on page one; a second key exists + // so a real scan WOULD have more to return, forcing `has_more` to depend + // on whether the huge key can serve as a resume boundary. + let huge_key = Bytes::from([vec![b'0'; 4], vec![b'k'; 65_505]].concat()); + let next_key = Bytes::from_static(b"1-next-key"); + assert!(matches!( + actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: huge_key.clone(), + value: Bytes::from_static(b"v"), + }), + KvResponse::PutOk + )); + assert!(matches!( + actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: next_key, + value: Bytes::from_static(b"v"), + }), + KvResponse::PutOk + )); + + // Act + let response = actor.handle(KvMessage::Scan { + tx_id, + scope, + query: ScanQuery { + start: None, + end: None, + limit: None, + reverse: false, + start_exclusive: false, + }, + }); + + // Assert + match response { + KvResponse::Error { .. } => {} + KvResponse::ScanResult { has_more, .. } => { + assert!( + !has_more, + "a page must never promise a continuation it cannot honour" + ); + } + other => panic!("expected Error or ScanResult, got {other:?}"), + } +} + +#[test] +fn should_return_frame_valid_terminal_key_larger_than_continuation_limit() { + // Arrange + // A key only needs to fit a continuation request when another matching row + // remains. This key is deliberately one byte beyond that conservative + // boundary, but its PUT and the terminal SCAN response both remain valid + // u16-sized TLV payloads. + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "terminal-large-key"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + let key_len = crate::domains::kv::scan_wire_budget::kv_scan_continuation_max_key_bytes() + 1; + let terminal_key = Bytes::from(vec![b'z'; key_len]); + let value = Bytes::from_static(b"v"); + let route = format!("kv://{}/{}/{}", scope.realm, scope.area, scope.resource); + let put_payload_len = 20 + route.len() + terminal_key.len() + value.len(); + assert!(u16::try_from(put_payload_len).is_ok()); + assert!(matches!( + actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: terminal_key.clone(), + value: value.clone(), + }), + KvResponse::PutOk + )); + + // Act + let response = actor.handle(KvMessage::Scan { + tx_id, + scope, + query: ScanQuery { + start: None, + end: None, + limit: None, + reverse: false, + start_exclusive: false, + }, + }); + + // Assert + let KvResponse::ScanResult { items, has_more } = &response else { + panic!("terminal large key should remain scannable, got {response:?}"); + }; + assert_eq!(items.len(), 1); + assert_eq!(items[0].key, terminal_key); + assert_eq!(items[0].value, value); + assert!(!has_more, "a terminal result needs no continuation"); + let encoded = crate::dispatch::protocol::kv::encode_response(&response); + assert!(u16::try_from(encoded.len()).is_ok()); +} + +#[test] +fn should_keep_oversized_scan_pair_error_inside_one_wire_frame() { + // Arrange + // The error for an unencodable pair must not itself be unencodable. A key + // can approach the frame limit on its own, and lossy UTF-8 conversion + // widens every invalid byte to three, so echoing it whole would recreate + // the failure this branch exists to prevent. + // + // The scan must fail loudly here rather than silently omit the pair: a + // skipped entry would make SCAN report success while permanently missing + // an in-range, authoritative value - and if it were the last entry, + // `has_more` would read false too, leaving the client no way to detect + // the gap. An explicit, retried-forever-safe error is the honest + // response; only a direct GET can still return this particular value. + let mut actor = test_actor(); + let scope = KvResourceScope::new(RouteFamily::new(1), "test", "kv", "oversized-pair"); + let tx_id = begin_with_scope(&mut actor, scope.clone()); + // Invalid UTF-8 throughout, so lossy conversion expands every byte + // threefold, and large enough that echoing it whole would blow the frame. + // The key must be hostile enough that echoing it whole would itself blow + // the frame: 30,000 invalid bytes render as 90,000 replacement characters, + // well past u16::MAX. A short or printable key would pass even with the + // truncation removed, testing nothing. + // + // The value then pushes the pair past the exact budget (8 + key + value > + // 65_529). Note the pair is unreachable over the wire - a PUT is itself one + // TLV value - so this branch guards in-process writers and data stored + // before the budget existed. + let hostile_key = Bytes::from(vec![0xffu8; 30_000]); + let value = Bytes::from(vec![b'v'; 40_000]); + assert!(matches!( + actor.handle(KvMessage::Put { + tx_id, + scope: scope.clone(), + key: hostile_key, + value, + }), + KvResponse::PutOk + )); + + // Act + let response = actor.handle(KvMessage::Scan { + tx_id, + scope, + query: ScanQuery { + start: None, + end: None, + limit: None, + reverse: false, + start_exclusive: false, + }, + }); + + // Assert + let encoded = crate::dispatch::protocol::kv::encode_response(&response); + assert!( + u16::try_from(encoded.len()).is_ok(), + "the oversized-pair error is itself {} bytes, past the {}-byte TLV limit", + encoded.len(), + u16::MAX + ); + assert!( + matches!(response, KvResponse::Error { .. }), + "an unencodable pair must be reported, got {response:?}" + ); +} diff --git a/src/domains/kv/actor/tests/write_policy.rs b/src/domains/kv/actor/tests/write_policy.rs new file mode 100644 index 00000000..f3013f34 --- /dev/null +++ b/src/domains/kv/actor/tests/write_policy.rs @@ -0,0 +1,342 @@ +use super::*; + +#[test] +pub(super) fn should_reject_insert_when_key_exists() { + // Arrange + let mut actor = test_actor(); + let begin_response = actor.handle(KvMessage::Begin { + scope: KvResourceScope::new( + RouteFamily::new(1), + "test".to_string(), + "kv".to_string(), + "table1".to_string(), + ), + mode: TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }); + let KvResponse::BeginOk { tx_id } = begin_response else { + panic!("Expected BeginOk"); + }; + + let key = Bytes::from("testkey"); + actor.handle(KvMessage::Insert { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), + key: key.clone(), + value: Bytes::from("value1"), + }); + + // Act - Try to insert again + let response = actor.handle(KvMessage::Insert { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), + key: key.clone(), + value: Bytes::from("value2"), + }); + + // Assert + assert!(matches!( + response, + KvResponse::Error { + error: KvError::AlreadyExists + } + )); +} + +#[test] +pub(super) fn should_validate_delete_range_parameters() { + // Arrange + let mut actor = test_actor(); + let begin_response = actor.handle(KvMessage::Begin { + scope: KvResourceScope::new( + RouteFamily::new(1), + "test".to_string(), + "kv".to_string(), + "table1".to_string(), + ), + mode: TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }); + let KvResponse::BeginOk { tx_id } = begin_response else { + panic!("Expected BeginOk"); + }; + + // Act - End before start + let response = actor.handle(KvMessage::DeleteRange { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), + start: Bytes::from("z"), + end: Bytes::from("a"), + }); + + // Assert + assert!(matches!( + response, + KvResponse::Error { + error: KvError::InvalidRequest(_) + } + )); +} + +#[test] +pub(super) fn should_reject_route_family_zero() { + // Arrange + let mut actor = test_actor(); + + // Act + let result = actor.handle(KvMessage::Begin { + scope: KvResourceScope::new( + RouteFamily::new(0), + "test".to_string(), + "kv".to_string(), + "table1".to_string(), + ), + mode: TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }); + + // Assert + assert!(matches!( + result, + KvResponse::Error { + error: KvError::InvalidRouteFamily, + } + )); +} + +#[test] +pub(super) fn should_delete_existing_key() { + // Arrange + let mut actor = test_actor(); + let begin_response = actor.handle(KvMessage::Begin { + scope: KvResourceScope::new( + RouteFamily::new(1), + "test".to_string(), + "kv".to_string(), + "table1".to_string(), + ), + mode: TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }); + let KvResponse::BeginOk { tx_id } = begin_response else { + panic!("Expected BeginOk"); + }; + + let key = Bytes::from("delkey"); + actor.handle(KvMessage::Put { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), + key: key.clone(), + value: Bytes::from("value1"), + }); + + // Act - Delete the key + let delete_response = actor.handle(KvMessage::Delete { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), + key: key.clone(), + }); + + // Assert delete succeeds + assert!(matches!(delete_response, KvResponse::DeleteOk)); + + // Verify key is gone + let get_response = actor.handle(KvMessage::Get { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), + key: key.clone(), + }); + assert!(matches!( + get_response, + KvResponse::GetResult { + found: false, + value: None + } + )); +} + +#[test] +pub(super) fn should_scan_key_range() { + // Arrange + let mut actor = test_actor(); + let begin_response = actor.handle(KvMessage::Begin { + scope: KvResourceScope::new( + RouteFamily::new(1), + "test".to_string(), + "kv".to_string(), + "table1".to_string(), + ), + mode: TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }); + let KvResponse::BeginOk { tx_id } = begin_response else { + panic!("Expected BeginOk"); + }; + + // Add multiple keys + for i in 0..5 { + actor.handle(KvMessage::Put { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), + key: Bytes::from(format!("key{i:02}")), + value: Bytes::from(format!("value{i}")), + }); + } + + // Act - Scan range [key01, key04) + let response = actor.handle(KvMessage::Scan { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), + query: ScanQuery { + start: Some(Bytes::from("key01")), + end: Some(Bytes::from("key04")), + limit: None, + reverse: false, + start_exclusive: false, + }, + }); + + // Assert + match response { + KvResponse::ScanResult { items, .. } => { + assert!(items.len() >= 2); // At least key01, key02, key03 + } + _ => panic!("Expected ScanResult"), + } +} + +#[test] +pub(super) fn should_reject_delete_range_with_invalid_bounds() { + // Arrange + let mut actor = test_actor(); + let begin_response = actor.handle(KvMessage::Begin { + scope: KvResourceScope::new( + RouteFamily::new(1), + "test".to_string(), + "kv".to_string(), + "table1".to_string(), + ), + mode: TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }); + let KvResponse::BeginOk { tx_id } = begin_response else { + panic!("Expected BeginOk"); + }; + + // Act - End < Start + let response = actor.handle(KvMessage::DeleteRange { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), + start: Bytes::from("zzz"), + end: Bytes::from("aaa"), + }); + + // Assert + assert!(matches!( + response, + KvResponse::Error { + error: KvError::InvalidRequest(_) + } + )); +} + +#[test] +pub(super) fn should_scan_with_limit() { + // Arrange + let mut actor = test_actor(); + let begin_response = actor.handle(KvMessage::Begin { + scope: KvResourceScope::new( + RouteFamily::new(1), + "test".to_string(), + "kv".to_string(), + "table1".to_string(), + ), + mode: TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }); + let KvResponse::BeginOk { tx_id } = begin_response else { + panic!("Expected BeginOk"); + }; + + // Add 10 keys + for i in 0..10 { + actor.handle(KvMessage::Put { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), + key: Bytes::from(format!("k{i:02}")), + value: Bytes::from(format!("v{i}")), + }); + } + + // Act - Scan with limit of 3 + let response = actor.handle(KvMessage::Scan { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), + query: ScanQuery { + start: None, + end: None, + limit: Some(3), + reverse: false, + start_exclusive: false, + }, + }); + + // Assert + match response { + KvResponse::ScanResult { items, has_more } => { + assert_eq!(items.len(), 3); + assert!(has_more); + } + _ => panic!("Expected ScanResult"), + } +} + +#[test] +pub(super) fn should_scan_reverse() { + // Arrange + let mut actor = test_actor(); + let begin_response = actor.handle(KvMessage::Begin { + scope: KvResourceScope::new( + RouteFamily::new(1), + "test".to_string(), + "kv".to_string(), + "table1".to_string(), + ), + mode: TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }); + let KvResponse::BeginOk { tx_id } = begin_response else { + panic!("Expected BeginOk"); + }; + + // Add keys + for i in 0..5 { + actor.handle(KvMessage::Put { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), + key: Bytes::from(format!("k{i}")), + value: Bytes::from(format!("v{i}")), + }); + } + + // Act - Scan reverse + let response = actor.handle(KvMessage::Scan { + tx_id, + scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "table1".to_string()), + query: ScanQuery { + start: None, + end: None, + limit: None, + reverse: true, + start_exclusive: false, + }, + }); + + // Assert - Just verify it returns results (order depends on storage) + match response { + KvResponse::ScanResult { items, .. } => { + assert!(!items.is_empty()); + } + _ => panic!("Expected ScanResult"), + } +} diff --git a/src/domains/kv/actor/transaction_access.rs b/src/domains/kv/actor/transaction_access.rs new file mode 100644 index 00000000..3cfe8c1c --- /dev/null +++ b/src/domains/kv/actor/transaction_access.rs @@ -0,0 +1,48 @@ +//! Transaction lookup, activity tracking, and operation-scope validation. + +use super::{ActiveKvTx, KvActor}; +use crate::domains::kv::{KvError, KvResourceScope, KvResponse}; +use std::time::Instant; + +impl KvActor { + pub(super) fn scoped_transaction_or_err( + &mut self, + tx_id: u64, + scope: &KvResourceScope, + ) -> Result<&mut ActiveKvTx, KvResponse> { + let transaction = self + .transactions + .get_mut(&tx_id) + .ok_or_else(|| KvResponse::Error { + error: KvError::InvalidTxId, + })?; + transaction.last_activity = Instant::now(); + Self::validate_operation_scope(transaction, scope)?; + Ok(transaction) + } + + pub(super) fn validate_operation_scope( + active: &ActiveKvTx, + scope: &KvResourceScope, + ) -> Result<(), KvResponse> { + if scope.route_family != active.scope.route_family { + return Err(KvResponse::Error { + error: KvError::InvalidRouteFamily, + }); + } + if scope.realm != active.scope.realm { + return Err(KvResponse::Error { + error: KvError::RealmMismatch, + }); + } + if scope.area != active.scope.area || scope.resource != active.scope.resource { + return Err(KvResponse::Error { + error: KvError::TxScopeViolation { + expected: format!("{}/{}", active.scope.area, active.scope.resource), + actual: format!("{}/{}", scope.area, scope.resource), + }, + }); + } + Ok(()) + } +} diff --git a/src/domains/kv/actor/transactions.rs b/src/domains/kv/actor/transactions.rs new file mode 100644 index 00000000..b87dbdab --- /dev/null +++ b/src/domains/kv/actor/transactions.rs @@ -0,0 +1,139 @@ +//! Transaction creation, completion, rollback, and idle expiry. + +use super::{ActiveKvTx, KvActor, KvInventoryDelta}; +use crate::auth::validate_realm_format; +use crate::domains::kv::{KvError, KvResourceScope, KvResponse, TxMode}; +use cntryl_midge::TransactionMode; +use std::time::{Duration, Instant}; + +impl KvActor { + pub(super) fn handle_begin( + &mut self, + scope: KvResourceScope, + mode: TxMode, + write_options: cntryl_midge::WriteOptions, + ) -> KvResponse { + if validate_realm_format(&scope.realm).is_err() { + return KvResponse::Error { + error: KvError::InvalidRealm, + }; + } + + let Ok(column_family) = Self::resolve_column_family(scope.route_family) else { + return KvResponse::Error { + error: KvError::InvalidRouteFamily, + }; + }; + let transaction_mode = match mode { + TxMode::ReadOnly => TransactionMode::ReadOnly, + TxMode::ReadWrite => TransactionMode::ReadWrite, + }; + let Some(next_tx_id) = self.next_tx_id.checked_add(1) else { + return KvResponse::Error { + error: KvError::InvalidRequest("transaction ID space exhausted".to_string()), + }; + }; + + match self.store.begin_tx(column_family, transaction_mode) { + Ok(tx) => { + let tx_id = self.next_tx_id; + self.next_tx_id = next_tx_id; + let scoped_prefix = + Self::realm_resource_prefix(&scope.realm, &scope.area, &scope.resource); + self.transactions.insert( + tx_id, + ActiveKvTx { + scope, + scoped_prefix, + column_family, + tx, + write_options, + mutation_count: 0, + last_activity: Instant::now(), + inventory_delta: KvInventoryDelta::default(), + }, + ); + KvResponse::BeginOk { tx_id } + } + Err(error) => KvResponse::Error { + error: Self::map_midge_error(&error), + }, + } + } + + pub(super) fn handle_commit(&mut self, tx_id: u64, scope: &KvResourceScope) -> KvResponse { + let Some(active) = self.transactions.get(&tx_id) else { + return KvResponse::Error { + error: KvError::InvalidTxId, + }; + }; + if let Err(response) = Self::validate_operation_scope(active, scope) { + return response; + } + + let Some(mut active) = self.transactions.remove(&tx_id) else { + return KvResponse::Error { + error: KvError::InvalidTxId, + }; + }; + let inventory_scope = active.scope.clone(); + let inventory_column_family = active.column_family; + let inventory_delta = std::mem::take(&mut active.inventory_delta); + let inventory_write_options = Self::inventory_write_options(active.write_options); + match active.tx.commit(active.write_options) { + Ok(()) => { + if let Err(error) = Self::apply_inventory_delta( + &self.store, + inventory_column_family, + &inventory_scope, + &inventory_delta, + inventory_write_options, + ) { + tracing::warn!( + ?error, + scope = %inventory_scope.realm, + resource = %inventory_scope.resource, + "KV inventory estimate update failed after successful data commit; returning CommitOk for data path while estimates may drift" + ); + } + KvResponse::CommitOk + } + Err(error) => KvResponse::Error { + error: Self::map_midge_error(&error), + }, + } + } + + pub(super) fn handle_rollback(&mut self, tx_id: u64, scope: &KvResourceScope) -> KvResponse { + let Some(active) = self.transactions.get(&tx_id) else { + return KvResponse::Error { + error: KvError::InvalidTxId, + }; + }; + if let Err(response) = Self::validate_operation_scope(active, scope) { + return response; + } + + self.transactions.remove(&tx_id); + KvResponse::RollbackOk + } + + pub(crate) fn expire_idle_transactions(&mut self, ttl: Duration) -> Vec { + let now = Instant::now(); + let expired = self + .transactions + .iter() + .filter_map(|(tx_id, transaction)| { + (now.saturating_duration_since(transaction.last_activity) >= ttl).then_some(*tx_id) + }) + .collect::>(); + for tx_id in &expired { + self.transactions.remove(tx_id); + } + expired + } + + pub(crate) fn rollback_transaction(&mut self, tx_id: u64) -> bool { + self.transactions.remove(&tx_id).is_some() + } +} diff --git a/src/domains/kv/projection.rs b/src/domains/kv/admin_projection.rs similarity index 59% rename from src/domains/kv/projection.rs rename to src/domains/kv/admin_projection.rs index 3d175e32..c1d56356 100644 --- a/src/domains/kv/projection.rs +++ b/src/domains/kv/admin_projection.rs @@ -1,7 +1,10 @@ +//! Live KV transaction and latency projection for the admin read model. + use crate::control::admin::read_model::AdminReadModel; use crate::control::admin::{KvLatencySnapshot, KvTransaction}; use parking_lot::Mutex; use std::collections::{HashMap, VecDeque}; +#[cfg(test)] use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -35,6 +38,8 @@ impl KvRollingLatency { let mut samples = self.samples.iter().copied().collect::>(); samples.sort_by(f64::total_cmp); let sum = samples.iter().sum::(); + // Nearest-rank p95 is one-based: ceil(n * 0.95), converted back to a + // zero-based index after saturating arithmetic keeps small samples safe. let p95_index = samples .len() .saturating_mul(95) @@ -57,29 +62,34 @@ struct KvResourceLatency { /// Admin projection for the KV domain. /// -/// Applies live transaction changes incrementally and can rebuild the complete -/// admin read model snapshot when reconciliation is requested. +/// Applies live transaction changes incrementally and keeps admin state +/// synchronized in production by replaying runtime updates. /// Projection failure must never affect domain correctness. -pub struct KvAdminProjection { +pub(crate) struct KvAdminProjection { read_model: Arc, + #[cfg(test)] dirty: AtomicBool, latencies: Mutex>, } impl KvAdminProjection { - pub fn new(read_model: Arc) -> Self { + #[must_use] + pub(crate) fn new(read_model: Arc) -> Self { Self { read_model, + #[cfg(test)] dirty: AtomicBool::new(false), latencies: Mutex::new(HashMap::new()), } } - pub fn mark_dirty(&self) { + #[cfg(test)] + pub(crate) fn mark_dirty(&self) { self.dirty.store(true, Ordering::Relaxed); } - pub fn refresh_if_dirty(&self, build_transactions: F) + #[cfg(test)] + pub(crate) fn refresh_if_dirty(&self, build_transactions: F) where F: FnOnce() -> Vec, { @@ -89,24 +99,24 @@ impl KvAdminProjection { } } - pub fn upsert_transaction(&self, transaction: KvTransaction) { + pub(crate) fn upsert_transaction(&self, transaction: KvTransaction) { self.read_model.upsert_kv_transaction(transaction); } - pub fn remove_transaction(&self, session_id: u64, tx_id: u64) { + pub(crate) fn remove_transaction(&self, session_id: u64, tx_id: u64) { self.read_model.remove_kv_transaction(session_id, tx_id); } - pub fn remove_session_transactions(&self, session_id: u64) { + pub(crate) fn remove_session_transactions(&self, session_id: u64) { self.read_model .remove_kv_transactions_for_session(session_id); } - pub fn active_transaction_count(&self) -> usize { - self.read_model.kv_transactions(None).len() + pub(crate) fn active_transaction_count(&self) -> usize { + self.read_model.kv_transaction_count() } - pub fn active_transactions_for_resource( + pub(crate) fn active_transactions_for_resource( &self, family_id: u64, realm: &str, @@ -114,15 +124,7 @@ impl KvAdminProjection { resource: &str, ) -> usize { self.read_model - .kv_transactions(None) - .into_iter() - .filter(|transaction| { - transaction.route_family == family_id - && transaction.realm == realm - && transaction.area == area - && transaction.resource == resource - }) - .count() + .kv_transaction_count_for_resource(family_id, realm, area, resource) } pub(crate) fn record_read_latency(&self, key: &KvResourceLockKey, latency_ms: f64) { @@ -156,47 +158,5 @@ impl KvAdminProjection { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn should_refresh_projection_when_marked_dirty() { - // Arrange - let read_model = AdminReadModel::new(); - let projection = KvAdminProjection::new(read_model.clone()); - projection.mark_dirty(); - - // Act - projection.refresh_if_dirty(|| { - vec![KvTransaction::snapshot( - 1, - 41, - 7, - "acme", - "app", - "users", - "2026-07-01T00:00:00Z", - )] - }); - - // Assert - assert_eq!(read_model.kv_transactions(None).len(), 1); - } - - #[test] - fn should_record_projection_latency_by_operation_kind() { - // Arrange - let read_model = AdminReadModel::new(); - let projection = KvAdminProjection::new(read_model); - let key = KvResourceLockKey::new(1, "acme", "app", "users"); - - // Act - projection.record_write_latency(&key, 5.0); - projection.record_read_latency(&key, 3.0); - let (reads, writes) = projection.latency_snapshots(&key); - - // Assert - assert!((reads.avg_ms - 3.0).abs() < f64::EPSILON); - assert!((writes.avg_ms - 5.0).abs() < f64::EPSILON); - } -} +#[path = "tests/admin_projection.rs"] +mod tests; diff --git a/src/domains/kv/inventory.rs b/src/domains/kv/inventory.rs new file mode 100644 index 00000000..e7df9aff --- /dev/null +++ b/src/domains/kv/inventory.rs @@ -0,0 +1,62 @@ +//! Shared persisted format for per-resource KV inventory estimates. +//! +//! The actor write path in `actor/inventory_delta.rs` updates this metadata, +//! while `sink/admin/inventory.rs` reads and refreshes it for admin views. + +const VALUE_VERSION: u8 = 1; +const VALUE_LEN: usize = 18; +const RECORD_COUNT_RANGE: std::ops::Range = 2..10; +const STORAGE_BYTES_RANGE: std::ops::Range = 10..18; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct KvInventoryEstimate { + pub(crate) estimated_record_count: u64, + pub(crate) estimated_storage_bytes: u64, + pub(crate) estimate_complete: bool, +} + +impl Default for KvInventoryEstimate { + fn default() -> Self { + Self { + estimated_record_count: 0, + estimated_storage_bytes: 0, + estimate_complete: true, + } + } +} + +pub(crate) fn encode_estimate(estimate: KvInventoryEstimate) -> Vec { + let mut out = Vec::with_capacity(VALUE_LEN); + out.push(VALUE_VERSION); + out.push(u8::from(estimate.estimate_complete)); + out.extend_from_slice(&estimate.estimated_record_count.to_be_bytes()); + out.extend_from_slice(&estimate.estimated_storage_bytes.to_be_bytes()); + out +} + +/// Decode one persisted inventory estimate. +/// +/// # Errors +/// +/// Returns an error when the value has an unknown version or invalid length. +pub(crate) fn decode_estimate(bytes: &[u8]) -> Result { + if bytes.len() != VALUE_LEN || bytes.first().copied() != Some(VALUE_VERSION) { + return Err("invalid KV inventory metadata value".to_string()); + } + + let estimated_record_count = u64::from_be_bytes( + bytes[RECORD_COUNT_RANGE] + .try_into() + .map_err(|_| "invalid KV inventory record count".to_string())?, + ); + let estimated_storage_bytes = u64::from_be_bytes( + bytes[STORAGE_BYTES_RANGE] + .try_into() + .map_err(|_| "invalid KV inventory storage estimate".to_string())?, + ); + Ok(KvInventoryEstimate { + estimated_record_count, + estimated_storage_bytes, + estimate_complete: bytes[1] != 0, + }) +} diff --git a/src/domains/kv/metrics.rs b/src/domains/kv/metrics.rs index 3f0a46bb..ec7de0bb 100644 --- a/src/domains/kv/metrics.rs +++ b/src/domains/kv/metrics.rs @@ -1,3 +1,5 @@ +//! KV request, outcome, latency, and live-state metric collection. + use crate::observability::metrics::{DomainMetricSet, MetricsCollector}; use std::time::Instant; @@ -10,13 +12,13 @@ pub const METRIC_SUBSCRIPTIONS_GAUGE: &str = "fitz_kv_subscriptions_gauge"; pub const METRIC_NOTIFY_DROPS_TOTAL: &str = "fitz_kv_notify_drops_total"; #[derive(Clone)] -pub struct KvMetrics { +pub(crate) struct KvMetrics { metrics: DomainMetricSet, } impl KvMetrics { #[must_use] - pub fn new(collector: MetricsCollector) -> Self { + pub(crate) fn new(collector: MetricsCollector) -> Self { Self { metrics: DomainMetricSet::new( collector, @@ -29,27 +31,27 @@ impl KvMetrics { } #[must_use] - pub fn record_request_start(&self) -> Instant { + pub(crate) fn record_request_start(&self) -> Instant { self.metrics.record_request_start() } - pub fn record_success(&self, started_at: Instant) { + pub(crate) fn record_success(&self, started_at: Instant) { self.metrics.record_success(started_at); } - pub fn record_failure(&self, started_at: Instant) { + pub(crate) fn record_failure(&self, started_at: Instant) { self.metrics.record_failure(started_at); } - pub fn counter_inc(&self, name: &str) { + pub(crate) fn counter_inc(&self, name: &str) { self.metrics.counter_inc(name); } - pub fn set_active_transactions(&self, count: usize) { + pub(crate) fn set_active_transactions(&self, count: usize) { self.metrics.gauge_set(METRIC_ACTIVE_GAUGE, count as u64); } - pub fn set_subscription_count(&self, count: usize) { + pub(crate) fn set_subscription_count(&self, count: usize) { self.metrics .gauge_set(METRIC_SUBSCRIPTIONS_GAUGE, count as u64); } diff --git a/src/domains/kv/mod.rs b/src/domains/kv/mod.rs index 8975e685..7a5930bb 100644 --- a/src/domains/kv/mod.rs +++ b/src/domains/kv/mod.rs @@ -47,18 +47,21 @@ //! - Default column family (CF=0) is FORBIDDEN //! - All KV persistence MUST specify explicit CF via `RouteFamily` -pub mod actor; -pub mod metrics; -pub mod projection; -pub mod protocol; -pub mod session; +mod actor; +mod admin_projection; +mod inventory; +pub(crate) mod metrics; +mod protocol; +mod scan_wire_budget; pub mod sink; -pub mod watch; +mod watch_registry; pub use actor::KvActor; -pub use metrics::KvMetrics; pub use protocol::{ KvClientFrame, KvClientNotification, KvClientRequest, KvClientResponse, KvError, KvMessage, KvNotification, KvPair, KvResourceScope, KvResponse, KvSubscriptionMessage, ScanQuery, TxMode, }; -pub use session::SessionActor; +pub use sink::{ + AdminKvCommittedPair, AdminKvPrefixScanResult, AdminKvRowsRequest, AdminKvRowsResult, + KvDomainSink, +}; diff --git a/src/domains/kv/protocol.rs b/src/domains/kv/protocol.rs index 1b54390e..f26d26c1 100644 --- a/src/domains/kv/protocol.rs +++ b/src/domains/kv/protocol.rs @@ -120,25 +120,8 @@ impl KvMessage { } #[cfg(test)] -mod scope_tests { - use super::*; - - #[test] - fn should_expose_scope_for_every_kv_message_variant() { - // Arrange - let scope = KvResourceScope::new(RouteFamily::new(7), "realm", "area", "resource"); - let message = KvMessage::Rollback { - tx_id: 1, - scope: scope.clone(), - }; - - // Act - let actual = message.scope(); - - // Assert - assert_eq!(actual, &scope); - } -} +#[path = "tests/protocol.rs"] +mod tests; /// KV watch messages handled by `KvDomainSink` before actor dispatch. #[derive(Debug, Clone)] @@ -165,6 +148,7 @@ pub struct KvClientRequest { } impl KvClientRequest { + #[must_use] pub fn new(meta: ClientFrameMeta, frame: Result) -> Self { Self { meta, frame } } @@ -185,6 +169,7 @@ pub struct KvClientResponse { } impl KvClientResponse { + #[must_use] pub fn new(meta: ClientFrameMeta, response: KvResponse) -> Self { Self { meta, response } } @@ -245,6 +230,15 @@ pub struct ScanQuery { pub limit: Option, /// Reverse scan order pub reverse: bool, + /// Treat `start` as exclusive rather than inclusive. + /// + /// Continuation needs this. A page bounded by the response byte budget can + /// hold a single pair, and resuming from an inclusive `start` then returns + /// that same pair forever - the scan never advances and later keys are + /// unreachable. Resuming exclusively guarantees progress in both + /// directions. Absent on the wire from older clients, where it defaults to + /// `false` and the inclusive behaviour is unchanged. + pub start_exclusive: bool, } /// KV operation response diff --git a/src/domains/kv/scan_wire_budget.rs b/src/domains/kv/scan_wire_budget.rs new file mode 100644 index 00000000..db244738 --- /dev/null +++ b/src/domains/kv/scan_wire_budget.rs @@ -0,0 +1,69 @@ +//! Wire-size budget for KV scan responses. +//! +//! A scan response is carried as a single TLV value with a `u16` length, so the +//! page must be bounded by bytes and not only by item count. Every pair can be +//! individually small and legal while the aggregate is unencodable: at 1 KiB +//! values the frame overflows at roughly 63 items, far below the 1,024-item +//! cap a client gets when it omits `limit`. + +/// A KV response is carried as one TLV value, whose length prefix is a `u16`. +pub(crate) const MAX_KV_RESPONSE_PAYLOAD_BYTES: usize = u16::MAX as usize; + +/// A scan response's non-item bytes, exactly as `encode_response` writes them: +/// the status flag (1), the item count (4), and the `has_more` marker (1). +/// +/// Exact rather than generous on purpose. Over-charging rejects responses that +/// are wire-valid: a single 300-byte key with a 65,200-byte value encodes to +/// 65,514 bytes and fits, but a padded budget refuses it. `scan_wire_budget` +/// tests assert these constants against the real encoded length, so codec drift +/// fails a test instead of silently re-introducing false rejections. +const KV_SCAN_ENVELOPE_OVERHEAD_BYTES: usize = 6; + +/// One encoded pair's fixed parts: the `u32` length prefixes on key and value. +const KV_SCAN_ITEM_FIXED_OVERHEAD_BYTES: usize = 8; + +/// Largest total item payload a scan response can carry. +#[must_use] +pub(crate) fn kv_scan_response_byte_ceiling() -> usize { + MAX_KV_RESPONSE_PAYLOAD_BYTES.saturating_sub(KV_SCAN_ENVELOPE_OVERHEAD_BYTES) +} + +/// A SCAN request is carried as one TLV value too. Resuming a page requires +/// re-issuing SCAN with `start_key` set to the last returned key (plus +/// `start_exclusive`), so a page boundary is only useful if that key can +/// itself be echoed back inside a fresh request. +/// +/// Generous, fixed reserve for everything else a SCAN request carries besides +/// the key: `tx_id` (8), the route string (realm/area/resource - normally a +/// few dozen bytes, budgeted here at up to 512), and the fixed flag/length +/// bytes for `has_start`, `start_key_len`, `has_end`, `has_limit`, `reverse`, +/// and `start_exclusive` (roughly 20). Deliberately generous so this reserve +/// stays safe even if the request envelope grows. +const KV_SCAN_CONTINUATION_REQUEST_RESERVE_BYTES: usize = 1024; + +/// Largest key that can safely become a page's resume boundary. +/// +/// A key longer than this may still fit comfortably in the PUT that wrote it +/// and in a SCAN response that returns it once, but re-issuing it as +/// `start_key` in a continuation request could overflow the request's own +/// wire ceiling - producing a page whose `has_more=1` promises a continuation +/// that cannot actually be sent. Bounding it here means such a key is instead +/// refused up front with an explicit error (the same one already used when a +/// pair cannot fit any response), rather than manufacturing an unusable +/// resume point. +#[must_use] +pub(crate) fn kv_scan_continuation_max_key_bytes() -> usize { + MAX_KV_RESPONSE_PAYLOAD_BYTES.saturating_sub(KV_SCAN_CONTINUATION_REQUEST_RESERVE_BYTES) +} + +/// Conservative wire cost of one encoded scan pair. +#[must_use] +pub(crate) fn kv_scan_item_wire_bytes(key_len: usize, value_len: usize) -> usize { + KV_SCAN_ITEM_FIXED_OVERHEAD_BYTES + .saturating_add(key_len) + .saturating_add(value_len) +} + +#[cfg(test)] +#[path = "tests/scan_wire_budget.rs"] +mod tests; diff --git a/src/domains/kv/session.rs b/src/domains/kv/session.rs deleted file mode 100644 index 9a170ae8..00000000 --- a/src/domains/kv/session.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! KV domain session authorization helpers. -//! -//! Responsibilities: -//! - Enforce session-level authorization for KV operations -//! - Forward authorized operations to the `KvActor` -//! -//! Authorization is checked using the realm field from `KvMessage::Begin`, -//! which is mapped to a route pattern for permission checking. - -use crate::auth::Access; -use crate::domains::kv::actor::KvActor; -use crate::domains::kv::protocol::{KvMessage, KvResponse, TxMode}; -use crate::runtime::routing::Route; -use crate::session::permissions::SessionPermissions; -use crate::session::session::SessionId; - -/// Lightweight `SessionActor` helpers for the KV domain. -/// See the module documentation for its authorization and forwarding responsibilities. -pub struct SessionActor { - pub session_id: SessionId, - pub permissions: SessionPermissions, -} - -impl SessionActor { - #[must_use] - pub fn new(session_id: SessionId, permissions: SessionPermissions) -> Self { - Self { - session_id, - permissions, - } - } - - /// Attempt to begin a KV transaction. - /// - /// # Errors - /// - /// Returns an error when authorization fails, the message is not `Begin`, or - /// the actor returns [`KvResponse::Error`]. - pub fn begin(&self, msg: KvMessage, kv_actor: &mut KvActor) -> Result<(), String> { - if let KvMessage::Begin { - ref scope, mode, .. - } = msg - { - let realm = &scope.realm; - // Extract realm-based route for authorization check - // Format: "kv://realm" for basic realm-level authorization - let route = Route::new(format!("kv://{realm}")); - - // Authorization policy: **write implies readwrite**, **read implies readonly**. - // Authorization depends on transaction mode: - // - ReadOnly: requires Read OR Write permission - // - ReadWrite: requires Write permission - match mode { - TxMode::ReadOnly => { - if !self.permissions.allows(&route, Access::Read) - && !self.permissions.allows(&route, Access::Write) - { - return Err(format!("unauthorized: realm '{realm}'")); - } - } - TxMode::ReadWrite => { - if !self.permissions.allows(&route, Access::Write) { - return Err(format!( - "unauthorized: write access required for realm '{realm}'" - )); - } - } - } - - // Forward to actor and check for errors - let response = kv_actor.handle(msg); - match response { - KvResponse::Error { error } => Err(format!("kv error: {error}")), - _ => Ok(()), - } - } else { - Err("invalid message type for begin".to_string()) - } - } - - /// Forward subsequent KV operations (after begin). - /// Realm authorization was already checked at begin time. - /// - pub fn operation(&self, kv_actor: &mut KvActor, msg: KvMessage) -> KvResponse { - kv_actor.handle(msg) - } -} diff --git a/src/domains/kv/sink/actor_commands.rs b/src/domains/kv/sink/actor_commands.rs deleted file mode 100644 index 02b472db..00000000 --- a/src/domains/kv/sink/actor_commands.rs +++ /dev/null @@ -1,46 +0,0 @@ -use super::model::{KvDomainCommand, KvDomainSink}; -use std::time::Duration; - -impl KvDomainSink { - #[cfg(test)] - pub(super) fn stop_actor_for_tests(&self) { - self.actor.stop(); - } - - fn send_unit_actor_command( - &self, - operation: &'static str, - build_command: impl FnOnce(crossbeam_channel::Sender<()>) -> KvDomainCommand, - ) { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = self.actor.try_send_high_priority(build_command(reply_tx)) { - tracing::warn!(domain = "kv", operation, error = %error, "KV actor command enqueue failed"); - return; - } - - if let Err(error) = reply_rx.recv_timeout(Duration::from_secs(1)) { - tracing::warn!(domain = "kv", operation, error = %error, "KV actor command reply failed"); - } - } - - pub fn cleanup_session(&self, session_id: u64) { - self.send_unit_actor_command("cleanup_session", |reply| { - KvDomainCommand::CleanupSession(session_id, reply) - }); - } - - pub fn active_transaction_count(&self) -> usize { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = self - .actor - .try_send_high_priority(KvDomainCommand::ReadActiveTransactionCount(reply_tx)) - { - tracing::warn!(domain = "kv", error = %error, "KV active-transaction query enqueue failed"); - return 0; - } - - reply_rx - .recv_timeout(Duration::from_secs(1)) - .unwrap_or_default() - } -} diff --git a/src/domains/kv/sink/admin/inventory.rs b/src/domains/kv/sink/admin/inventory.rs new file mode 100644 index 00000000..c533eb66 --- /dev/null +++ b/src/domains/kv/sink/admin/inventory.rs @@ -0,0 +1,231 @@ +//! Resource inventory enumeration and estimate refresh behavior. +//! +//! Persisted estimates use the shared codec in `domains::kv::inventory`; the +//! actor-side update policy lives in `actor/inventory_delta.rs`. + +use super::super::locks::KvResourceLockKey; +use super::super::state::KvDomainRuntime; +use crate::domains::kv::inventory::{decode_estimate, encode_estimate, KvInventoryEstimate}; +use crate::domains::kv::KvActor; + +const ADMIN_INVENTORY_REFRESH_LIMIT: usize = 10_000; + +impl KvDomainRuntime<'_> { + /// Build an admin inventory snapshot for the requested route family scope. + /// + /// # Errors + /// + /// Returns an error when the underlying storage inventory scan fails. + pub(super) fn admin_inventory( + &self, + family: Option, + ) -> Result, String> { + let families = if let Some(family) = family { + vec![family.id()] + } else { + self.core + .store + .list_column_families() + .map_err(|error| error.to_string())? + .into_iter() + .map(|handle| handle.id()) + .filter(|family_id| *family_id != 0) + .collect::>() + }; + + let mut entries = Vec::new(); + for family_id in families { + entries.extend(self.admin_inventory_for_family(u64::from(family_id))?); + } + entries.sort_by(|left, right| { + ( + left.route_family, + left.realm.as_str(), + left.area.as_str(), + left.resource.as_str(), + ) + .cmp(&( + right.route_family, + right.realm.as_str(), + right.area.as_str(), + right.resource.as_str(), + )) + }); + Ok(entries) + } + + /// Read one admin inventory entry for a specific KV resource. + /// + /// # Errors + /// + /// Returns an error when storage reads, estimate refreshes, or estimate + /// decoding fails. + pub(super) fn admin_inventory_resource( + &self, + route_family: crate::runtime::routing::RouteFamily, + realm: &str, + area: &str, + resource: &str, + ) -> Result, String> { + let family_id = route_family.as_u64(); + let column_family = KvActor::resolve_column_family(route_family)?; + let key = KvActor::inventory_metadata_key(realm, area, resource); + let tx = self + .core + .store + .begin_tx(column_family, cntryl_midge::TransactionMode::ReadOnly) + .map_err(|error| error.to_string())?; + + let estimate = if let Some(value) = tx.get(&key).map_err(|error| error.to_string())? { + decode_estimate(&value)? + } else { + let refreshed = + self.refresh_inventory_estimate(family_id, realm, area, resource, false)?; + if refreshed.estimated_record_count == 0 && refreshed.estimate_complete { + return Ok(None); + } + refreshed + }; + let estimate = if estimate.estimate_complete { + estimate + } else { + self.refresh_inventory_estimate(family_id, realm, area, resource, true)? + }; + + Ok(Some(self.inventory_entry_from_estimate( + family_id, realm, area, resource, estimate, + ))) + } + + pub(super) fn admin_inventory_for_family( + &self, + family_id: u64, + ) -> Result, String> { + let route_family = crate::runtime::routing::RouteFamily::try_from(family_id) + .map_err(|_| format!("invalid route family ID: {family_id}"))?; + let column_family = KvActor::resolve_column_family(route_family)?; + let tx = self + .core + .store + .begin_tx(column_family, cntryl_midge::TransactionMode::ReadOnly) + .map_err(|error| error.to_string())?; + let mut iterator = tx + .scan(&cntryl_midge::Query::new()) + .map_err(|error| error.to_string())?; + let mut discovered = Vec::new(); + + for entry in iterator.by_ref() { + let (key, value) = entry.map_err(|error| error.to_string())?; + let Some((realm, area, resource)) = KvActor::parse_inventory_metadata_key(&key) else { + continue; + }; + let estimate = decode_estimate(&value)?; + discovered.push((realm, area, resource, estimate)); + } + + drop(iterator); + drop(tx); + + discovered + .into_iter() + .map(|(realm, area, resource, estimate)| { + let estimate = if estimate.estimate_complete { + estimate + } else { + self.refresh_inventory_estimate(family_id, &realm, &area, &resource, true)? + }; + Ok(self + .inventory_entry_from_estimate(family_id, &realm, &area, &resource, estimate)) + }) + .collect() + } + + pub(super) fn refresh_inventory_estimate( + &self, + family_id: u64, + realm: &str, + area: &str, + resource: &str, + persist_empty: bool, + ) -> Result { + let route_family = crate::runtime::routing::RouteFamily::try_from(family_id) + .map_err(|_| format!("invalid route family ID: {family_id}"))?; + let column_family = KvActor::resolve_column_family(route_family)?; + let read_tx = self + .core + .store + .begin_tx(column_family, cntryl_midge::TransactionMode::ReadOnly) + .map_err(|error| error.to_string())?; + let resource_prefix = KvActor::realm_resource_prefix(realm, area, resource); + let mut rows = Self::scan_scoped_prefix( + &read_tx, + &resource_prefix, + &resource_prefix, + &resource_prefix, + ADMIN_INVENTORY_REFRESH_LIMIT.saturating_add(1), + )?; + let has_more = rows.len() > ADMIN_INVENTORY_REFRESH_LIMIT; + rows.truncate(ADMIN_INVENTORY_REFRESH_LIMIT); + let count = u64::try_from(rows.len()).unwrap_or(u64::MAX); + let storage_bytes = rows.iter().fold(0u64, |total, item| { + total + .saturating_add(item.key.len() as u64) + .saturating_add(item.value.len() as u64) + }); + + let estimate_complete = !has_more; + let estimate = KvInventoryEstimate { + estimated_record_count: count, + estimated_storage_bytes: storage_bytes, + estimate_complete, + }; + + drop(read_tx); + + if persist_empty || estimate.estimated_record_count > 0 || !estimate.estimate_complete { + let mut write_tx = self + .core + .store + .begin_tx(column_family, cntryl_midge::TransactionMode::ReadWrite) + .map_err(|error| error.to_string())?; + write_tx + .put( + KvActor::inventory_metadata_key(realm, area, resource), + encode_estimate(estimate), + None, + ) + .map_err(|error| error.to_string())?; + write_tx + .commit(self.core.sync_write_options) + .map_err(|error| error.to_string())?; + } + + Ok(estimate) + } + + pub(super) fn inventory_entry_from_estimate( + &self, + family_id: u64, + realm: &str, + area: &str, + resource: &str, + estimate: KvInventoryEstimate, + ) -> crate::control::admin::KvResourceInventoryEntry { + let resource_key = KvResourceLockKey::new(family_id, realm, area, resource); + let (read_latency, write_latency) = self.latency_snapshots(&resource_key); + crate::control::admin::KvResourceInventoryEntry { + route_family: family_id, + realm: realm.to_string(), + area: area.to_string(), + resource: resource.to_string(), + estimated_record_count: estimate.estimated_record_count, + estimated_storage_bytes: estimate.estimated_storage_bytes, + estimate_complete: estimate.estimate_complete, + read_latency_avg_ms: read_latency.avg_ms, + read_latency_p95_ms: read_latency.p95_ms, + write_latency_avg_ms: write_latency.avg_ms, + write_latency_p95_ms: write_latency.p95_ms, + transactions_active: self.active_transactions_for_resource(&resource_key), + } + } +} diff --git a/src/domains/kv/sink/admin/mod.rs b/src/domains/kv/sink/admin/mod.rs new file mode 100644 index 00000000..057ecfcb --- /dev/null +++ b/src/domains/kv/sink/admin/mod.rs @@ -0,0 +1,126 @@ +//! Public admin façade and crate-internal storage-backed implementations. + +use super::state::KvDomainSink; +use crate::runtime::routing::RouteFamily; + +mod inventory; +mod scans; +mod values; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AdminKvCommittedPair { + pub key: Vec, + pub value: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AdminKvPrefixScanResult { + pub items: Vec, + pub has_more: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AdminKvRowsResult { + pub items: Vec, + pub next_cursor: Option>, + pub has_more: bool, +} + +pub struct AdminKvRowsRequest<'a> { + pub route_family: RouteFamily, + pub realm: &'a str, + pub area: &'a str, + pub resource: &'a str, + pub starts_with: &'a [u8], + pub cursor: Option<&'a [u8]>, + pub limit: usize, +} + +impl KvDomainSink { + #[cfg(test)] + /// Read one family directly for storage-backed admin regression tests. + pub(super) fn admin_inventory_for_family_for_tests( + &self, + family_id: u64, + ) -> Result, String> { + self.state.runtime().admin_inventory_for_family(family_id) + } + + /// Build an admin inventory snapshot for the requested route family scope. + /// + /// # Errors + /// Returns an error when the underlying storage inventory scan fails. + pub fn admin_inventory( + &self, + family: Option, + ) -> Result, String> { + self.state.runtime().admin_inventory(family) + } + + /// Read one admin inventory entry for a specific KV resource. + /// + /// # Errors + /// Returns an error when storage reads, estimate refreshes, or estimate decoding fails. + pub fn admin_inventory_resource( + &self, + route_family: RouteFamily, + realm: &str, + area: &str, + resource: &str, + ) -> Result, String> { + self.state + .runtime() + .admin_inventory_resource(route_family, realm, area, resource) + } + + /// Read one committed KV value directly from storage for admin inspection. + /// + /// # Errors + /// Returns an error when the storage transaction or read fails. + pub fn admin_get_committed_value( + &self, + route_family: RouteFamily, + realm: &str, + area: &str, + resource: &str, + key: &[u8], + ) -> Result>, String> { + self.state + .runtime() + .admin_get_committed_value(route_family, realm, area, resource, key) + } + + /// Scan a committed KV prefix directly from storage for admin inspection. + /// + /// # Errors + /// Returns an error when the storage transaction or scan fails. + pub fn admin_scan_committed_prefix( + &self, + route_family: RouteFamily, + realm: &str, + area: &str, + resource: &str, + key_prefix: &[u8], + limit: usize, + ) -> Result { + self.state.runtime().admin_scan_committed_prefix( + route_family, + realm, + area, + resource, + key_prefix, + limit, + ) + } + + /// Scan committed KV rows with an optional pagination cursor. + /// + /// # Errors + /// Returns an error when cursor validation fails or the storage scan fails. + pub fn admin_scan_committed_rows( + &self, + request: &AdminKvRowsRequest<'_>, + ) -> Result { + self.state.runtime().admin_scan_committed_rows(request) + } +} diff --git a/src/domains/kv/sink/admin/scans.rs b/src/domains/kv/sink/admin/scans.rs new file mode 100644 index 00000000..63ce579b --- /dev/null +++ b/src/domains/kv/sink/admin/scans.rs @@ -0,0 +1,133 @@ +//! Storage-backed admin prefix and paginated row scans. + +use super::super::locks::KvResourceLockKey; +use super::super::state::KvDomainRuntime; +use super::{AdminKvCommittedPair, AdminKvPrefixScanResult, AdminKvRowsRequest, AdminKvRowsResult}; +use crate::domains::kv::KvActor; +use bytes::Bytes; + +impl KvDomainRuntime<'_> { + pub(super) fn admin_scan_committed_prefix( + &self, + route_family: crate::runtime::routing::RouteFamily, + realm: &str, + area: &str, + resource: &str, + key_prefix: &[u8], + limit: usize, + ) -> Result { + let started_at = std::time::Instant::now(); + let column_family = KvActor::resolve_column_family(route_family)?; + let tx = self + .core + .store + .begin_tx(column_family, cntryl_midge::TransactionMode::ReadOnly) + .map_err(|error| error.to_string())?; + let resource_prefix = KvActor::realm_resource_prefix(realm, area, resource); + let scoped_prefix = KvActor::encode_scoped_key(&resource_prefix, key_prefix); + let mut rows = Self::scan_scoped_prefix( + &tx, + &resource_prefix, + &scoped_prefix, + &scoped_prefix, + limit.saturating_add(1), + )?; + + let has_more = rows.len() > limit; + rows.truncate(limit); + self.record_read_latency( + &KvResourceLockKey::new(route_family.as_u64(), realm, area, resource), + started_at, + ); + Ok(AdminKvPrefixScanResult { + items: rows, + has_more, + }) + } + + pub(super) fn admin_scan_committed_rows( + &self, + request: &AdminKvRowsRequest<'_>, + ) -> Result { + let started_at = std::time::Instant::now(); + if let Some(cursor) = request.cursor { + if !cursor.starts_with(request.starts_with) { + return Err("cursor must start with starts_with prefix".to_string()); + } + } + + let column_family = KvActor::resolve_column_family(request.route_family)?; + let tx = self + .core + .store + .begin_tx(column_family, cntryl_midge::TransactionMode::ReadOnly) + .map_err(|error| error.to_string())?; + let resource_prefix = + KvActor::realm_resource_prefix(request.realm, request.area, request.resource); + let scoped_prefix = KvActor::encode_scoped_key(&resource_prefix, request.starts_with); + let scoped_start = request.cursor.map_or_else( + || scoped_prefix.clone(), + |cursor| KvActor::encode_scoped_key(&resource_prefix, cursor), + ); + let mut rows = Self::scan_scoped_prefix( + &tx, + &resource_prefix, + &scoped_prefix, + &scoped_start, + request.limit.saturating_add(1), + )?; + rows.retain(|item| { + request + .cursor + .is_none_or(|cursor| item.key.as_slice() > cursor) + }); + + let has_more = rows.len() > request.limit; + rows.truncate(request.limit); + let next_cursor = if has_more { + rows.last().map(|item| item.key.clone()) + } else { + None + }; + self.record_read_latency( + &KvResourceLockKey::new( + request.route_family.as_u64(), + request.realm, + request.area, + request.resource, + ), + started_at, + ); + Ok(AdminKvRowsResult { + items: rows, + next_cursor, + has_more, + }) + } + + pub(super) fn scan_scoped_prefix( + tx: &cntryl_midge::Transaction, + resource_prefix: &[u8], + scoped_prefix: &[u8], + scoped_start: &[u8], + limit: usize, + ) -> Result, String> { + let query = cntryl_midge::Query::new() + .prefix(Bytes::copy_from_slice(scoped_prefix)) + .start_key(Bytes::copy_from_slice(scoped_start)) + .end_key(Bytes::from(KvActor::prefix_range_end(scoped_prefix))) + .limit(limit); + let iterator = tx.scan(&query).map_err(|error| error.to_string())?; + let mut rows = Vec::new(); + for entry in iterator { + let (scoped_key, value) = entry.map_err(|error| error.to_string())?; + if let Some(user_key) = KvActor::strip_scoped_prefix(resource_prefix, &scoped_key) { + rows.push(AdminKvCommittedPair { + key: user_key, + value: value.to_vec(), + }); + } + } + Ok(rows) + } +} diff --git a/src/domains/kv/sink/admin/values.rs b/src/domains/kv/sink/admin/values.rs new file mode 100644 index 00000000..1e6621c9 --- /dev/null +++ b/src/domains/kv/sink/admin/values.rs @@ -0,0 +1,35 @@ +//! Direct committed-value inspection for the admin façade. + +use super::super::locks::KvResourceLockKey; +use super::super::state::KvDomainRuntime; +use crate::domains::kv::KvActor; + +impl KvDomainRuntime<'_> { + pub(super) fn admin_get_committed_value( + &self, + route_family: crate::runtime::routing::RouteFamily, + realm: &str, + area: &str, + resource: &str, + key: &[u8], + ) -> Result>, String> { + let started_at = std::time::Instant::now(); + let column_family = KvActor::resolve_column_family(route_family)?; + let tx = self + .core + .store + .begin_tx(column_family, cntryl_midge::TransactionMode::ReadOnly) + .map_err(|error| error.to_string())?; + let prefix = KvActor::realm_resource_prefix(realm, area, resource); + let scoped_key = KvActor::encode_scoped_key(&prefix, key); + let value = tx + .get(&scoped_key) + .map(|value| value.map(|value| value.as_ref().to_vec())) + .map_err(|error| error.to_string())?; + self.record_read_latency( + &KvResourceLockKey::new(route_family.as_u64(), realm, area, resource), + started_at, + ); + Ok(value) + } +} diff --git a/src/domains/kv/sink/cleanup.rs b/src/domains/kv/sink/cleanup.rs new file mode 100644 index 00000000..7f8d6e06 --- /dev/null +++ b/src/domains/kv/sink/cleanup.rs @@ -0,0 +1,46 @@ +//! Disconnect cleanup and stale queued-session rejection state. + +use super::state::KvDomainRuntime; +use crate::runtime::Envelope; + +impl KvDomainRuntime<'_> { + pub(super) fn handle_cleanup_envelope(&self, envelope: &Envelope) -> bool { + if let Some(cleanup) = envelope.payload::() { + self.cleanup_session(cleanup.session_id); + return true; + } + false + } + + pub(super) fn is_cleaned_up_session(&self, session_id: u64) -> bool { + self.core.cleaned_up_sessions.lock().contains(session_id) + } + + /// Remove all live KV state owned by a disconnected session. + pub(super) fn cleanup_session(&self, session_id: u64) { + // Mark first so an older normal-lane request that cleanup jumped over + // cannot recreate any of the state removed below. + self.core.cleaned_up_sessions.lock().mark(session_id); + self.core.actors.lock().remove(&session_id); + self.core + .resource_locks + .lock() + .retain(|_, owner| owner.session_id != session_id); + + { + let mut watch_registries = self.core.watch_registries.lock(); + for registry in watch_registries.values_mut() { + registry.remove_session(session_id); + } + watch_registries.retain(|_, registry| !registry.is_empty()); + } + + tracing::debug!( + domain = "kv", + session = session_id, + "All KV transactions, resource locks, watches, and admin state released for session" + ); + self.core.projection.remove_session_transactions(session_id); + self.refresh_metrics_gauges(); + } +} diff --git a/src/domains/kv/sink/commands.rs b/src/domains/kv/sink/commands.rs new file mode 100644 index 00000000..5248fe0e --- /dev/null +++ b/src/domains/kv/sink/commands.rs @@ -0,0 +1,68 @@ +//! Managed-actor command protocol and synchronous public controls. + +#[cfg(test)] +use super::locks::KvResourceLockKey; +use super::state::KvDomainSink; +use std::time::Duration; + +pub(super) enum KvDomainCommand { + Deliver(crate::runtime::Envelope), + CleanupSession(u64, crossbeam_channel::Sender<()>), + ReadActiveTransactionCount(crossbeam_channel::Sender), + #[cfg(test)] + SyncAdminSnapshot(crossbeam_channel::Sender<()>), + #[cfg(test)] + ReadLatencySnapshots( + KvResourceLockKey, + crossbeam_channel::Sender<( + crate::control::admin::KvLatencySnapshot, + crate::control::admin::KvLatencySnapshot, + )>, + ), + #[cfg(test)] + /// Ask the mailbox actor to apply its configured BEGIN write policy. + ApplyWriteOptions( + crate::domains::kv::KvMessage, + crossbeam_channel::Sender, + ), + #[cfg(test)] + PanicForTests, +} + +impl KvDomainSink { + pub(super) fn request_actor( + &self, + operation: &'static str, + build_command: impl FnOnce(crossbeam_channel::Sender) -> KvDomainCommand, + ) -> Option { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = self.actor.try_send_high_priority(build_command(reply_tx)) { + tracing::warn!(domain = "kv", operation, error = %error, "KV actor command enqueue failed"); + return None; + } + + match reply_rx.recv_timeout(Duration::from_secs(1)) { + Ok(reply) => Some(reply), + Err(error) => { + tracing::warn!(domain = "kv", operation, error = %error, "KV actor command reply failed"); + None + } + } + } + + /// Remove all live state owned by a disconnected session. + pub fn cleanup_session(&self, session_id: u64) { + let _ = self.request_actor("cleanup_session", |reply| { + KvDomainCommand::CleanupSession(session_id, reply) + }); + } + + /// Return the number of live KV transactions, or zero if the actor does not reply. + #[must_use] + pub fn active_transaction_count(&self) -> usize { + self.request_actor("active_transaction_count", |reply| { + KvDomainCommand::ReadActiveTransactionCount(reply) + }) + .unwrap_or_default() + } +} diff --git a/src/domains/kv/sink/delivery.rs b/src/domains/kv/sink/delivery.rs new file mode 100644 index 00000000..9ffd5c39 --- /dev/null +++ b/src/domains/kv/sink/delivery.rs @@ -0,0 +1,135 @@ +//! Request delivery, lifecycle rejection, parsing, and dispatch selection. + +use super::state::KvDomainRuntime; +use crate::domains::kv::{KvClientFrame, KvClientRequest}; +use crate::runtime::{DeliveryError, Envelope}; +use std::sync::atomic::Ordering; + +impl KvDomainRuntime<'_> { + pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { + if self.handle_cleanup_envelope(envelope) { + return Ok(()); + } + self.ensure_active()?; + Self::log_delivery(envelope); + + let request = Self::extract_request(envelope)?; + let meta = request.meta; + let request_started = self.record_request_start(); + if !Self::valid_request_envelope(envelope, meta) { + let response = Self::error_response("route family mismatch"); + let response_meta = Self::response_meta_for_source(envelope, meta); + self.route_kv_response(envelope, response_meta, &response, request_started)?; + return Ok(()); + } + + // This request was already queued (on the normal lane) before this + // session's disconnect cleanup ran (on the high-priority lane) and + // jumped ahead of it. Reject rather than silently recreating + // per-session state -- an actor and, for a write BEGIN, a resource + // lock -- for a session that is already gone and will never be + // cleaned up again. + if self.is_cleaned_up_session(meta.session_id) { + let response = Self::error_response("session already closed"); + let response_meta = Self::response_meta_for_source(envelope, meta); + self.route_kv_response(envelope, response_meta, &response, request_started)?; + return Ok(()); + } + + let operation_started = Self::record_operation_start(); + let Some(parsed_frame) = + self.parse_request_frame(envelope, meta, request.frame, request_started) + else { + return Ok(()); + }; + + match parsed_frame { + KvClientFrame::Sub(sub_msg) => { + self.handle_subscription_frame(envelope, meta, request_started, sub_msg) + } + KvClientFrame::Op(kv_message) => self.handle_actor_operation_frame( + envelope, + meta, + request_started, + operation_started, + kv_message, + ), + } + } + + fn ensure_active(&self) -> Result<(), DeliveryError> { + if !self.active.load(Ordering::Relaxed) { + return Err(DeliveryError::ActorStopped); + } + + Ok(()) + } + + fn log_delivery(envelope: &Envelope) { + tracing::debug!( + domain = "kv", + destination = %envelope.destination(), + source = ?envelope.source(), + "KV domain sink: received envelope" + ); + } + + fn extract_request(envelope: &Envelope) -> Result { + Self::request_from_envelope(envelope).ok_or_else(|| { + tracing::warn!( + domain = "kv", + destination = ?envelope.destination(), + "Envelope payload was not KvClientRequest" + ); + DeliveryError::ActorStopped + }) + } + + fn record_operation_start() -> std::time::Instant { + std::time::Instant::now() + } + + fn record_request_start(&self) -> std::time::Instant { + if let Some(metrics) = self.core.metrics.as_ref() { + metrics.record_request_start() + } else { + crate::observability::counter_inc(crate::domains::kv::metrics::METRIC_REQUESTS_TOTAL); + std::time::Instant::now() + } + } + + fn parse_request_frame( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + frame: Result, + request_started: std::time::Instant, + ) -> Option { + let parsed_frame = match frame { + Ok(msg) => msg, + Err(e) => { + tracing::warn!( + domain = "kv", + session = meta.session_id, + msg_type = meta.message_type, + error = %e, + "Failed to parse KV message" + ); + let response = Self::error_response(&e); + let response_meta = Self::response_meta_for_source(envelope, meta); + let _ = self.route_kv_response(envelope, response_meta, &response, request_started); + return None; + } + }; + + tracing::debug!( + domain = "kv", + session = meta.session_id, + channel = ?meta.channel, + msg_type = meta.message_type, + "Parsed KV message successfully" + ); + + Some(parsed_frame) + } +} diff --git a/src/domains/kv/sink/domain_sink_impl.rs b/src/domains/kv/sink/domain_sink_impl.rs deleted file mode 100644 index c4a07893..00000000 --- a/src/domains/kv/sink/domain_sink_impl.rs +++ /dev/null @@ -1,752 +0,0 @@ -#[cfg(test)] -use super::model::Utc; -use super::model::{ - AdminKvPrefixScanResult, AdminKvRowsRequest, AdminKvRowsResult, Arc, AtomicBool, HashMap, - KvAdminTransactionUpdate, KvDomainActor, KvDomainCommand, KvDomainCore, KvDomainRuntime, - KvDomainSink, KvDomainState, KvResourceLockKey, Mutex, Ordering, Router, - ADMIN_INVENTORY_REFRESH_LIMIT, -}; -use crate::domains::kv::KvActor; -use crate::runtime::routing::{Route, RouteAddress, RouteFamily}; - -mod admin_inventory; -mod routing; -#[cfg(test)] -mod test_channels; - -impl KvDomainState { - fn new( - store: Arc, - router: Arc, - admin_read_model: Arc, - ) -> Self { - Self { - core: KvDomainCore { - store, - actors: Arc::new(Mutex::new(HashMap::new())), - resource_locks: Mutex::new(HashMap::new()), - watch_actors: Mutex::new(HashMap::new()), - router, - projection: crate::domains::kv::projection::KvAdminProjection::new( - admin_read_model, - ), - metrics: None, - sync_write_options: cntryl_midge::WriteOptions::sync(), - buffered_write_options: cntryl_midge::WriteOptions::buffered(), - idle_transaction_ttl: std::time::Duration::from_mins(5), - }, - active: AtomicBool::new(true), - } - } - - pub(super) fn runtime(&self) -> KvDomainRuntime<'_> { - KvDomainRuntime { - core: &self.core, - active: &self.active, - } - } -} - -impl KvDomainActor { - pub(super) fn new(state: Arc) -> Self { - Self { state } - } - - pub(super) fn route_address() -> RouteAddress { - RouteAddress::new(RouteFamily::new(0), Route::new("internal://domain/kv")) - } -} - -impl KvDomainSink { - pub fn new( - store: Arc, - router: Arc, - admin_read_model: Arc, - ) -> Self { - let state = Arc::new(KvDomainState::new(store, router, admin_read_model)); - let actor = Self::spawn_actor(state.clone()); - Self { state, actor } - } - - fn spawn_actor(state: Arc) -> crate::runtime::ManagedActor { - let router = state.core.router.clone(); - crate::runtime::ManagedActor::spawn_fail_closed( - router, - KvDomainActor::route_address(), - move || KvDomainActor::new(state.clone()), - crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, - ) - } - - fn rebuild_actor(&mut self) { - self.actor.stop(); - self.actor = Self::spawn_actor(self.state.clone()); - } - - fn state_for_builder(&mut self) -> &mut KvDomainState { - Arc::get_mut(&mut self.state).expect("KV sink builders must run before sharing the sink") - } - - #[must_use] - pub fn with_sync_write_options(self, write_options: cntryl_midge::WriteOptions) -> Self { - let buffered_write_options = - if write_options.is_cloud_async() || write_options.is_cloud_strict() { - cntryl_midge::WriteOptions::cloud_async() - } else { - cntryl_midge::WriteOptions::buffered() - }; - self.with_write_options(write_options, buffered_write_options) - } - - #[must_use] - pub fn with_write_options( - mut self, - sync_write_options: cntryl_midge::WriteOptions, - buffered_write_options: cntryl_midge::WriteOptions, - ) -> Self { - self.actor.stop(); - let core = &mut self.state_for_builder().core; - core.sync_write_options = sync_write_options; - core.buffered_write_options = buffered_write_options; - self.rebuild_actor(); - self - } - - #[must_use] - pub fn with_idle_transaction_ttl(mut self, ttl: std::time::Duration) -> Self { - self.actor.stop(); - self.state_for_builder().core.idle_transaction_ttl = ttl; - self.rebuild_actor(); - self - } - - #[must_use] - pub fn with_metrics( - mut self, - collector: crate::observability::metrics::MetricsCollector, - ) -> Self { - self.actor.stop(); - let state = self.state_for_builder(); - state.core.metrics = Some(crate::domains::kv::KvMetrics::new(collector)); - state.runtime().refresh_metrics_gauges(); - self.rebuild_actor(); - self - } - - pub fn stop(&self) { - self.state.active.store(false, Ordering::Relaxed); - self.actor.stop(); - } - - pub(crate) fn actor_health_snapshot(&self) -> crate::runtime::ManagedActorHealthSnapshot { - self.actor.health_snapshot() - } - - #[cfg(test)] - pub(super) fn is_actor_running(&self) -> bool { - self.actor.is_running() - } - - #[cfg(test)] - pub(crate) fn mark_actor_permanently_failed_for_tests(&self) { - self.actor.mark_permanently_failed_for_tests(); - } - - #[cfg(test)] - pub(crate) fn panic_actor_for_tests(&self) { - let _ = self - .actor - .try_send_high_priority(KvDomainCommand::PanicForTests); - } - - /// Build an admin inventory snapshot for the requested route family scope. - /// - /// # Errors - /// - /// Returns an error when the underlying storage inventory scan fails. - pub fn admin_inventory( - &self, - family: Option, - ) -> Result, String> { - self.state.runtime().admin_inventory(family) - } - - /// Read one admin inventory entry for a specific KV resource. - /// - /// # Errors - /// - /// Returns an error when storage reads, estimate refreshes, or estimate - /// decoding fails. - pub fn admin_inventory_resource( - &self, - route_family: crate::runtime::routing::RouteFamily, - realm: &str, - area: &str, - resource: &str, - ) -> Result, String> { - self.state - .runtime() - .admin_inventory_resource(route_family, realm, area, resource) - } - - /// Read one committed KV value directly from storage for admin inspection. - /// - /// # Errors - /// - /// Returns an error when the storage transaction or read fails. - pub fn admin_get_committed_value( - &self, - route_family: crate::runtime::routing::RouteFamily, - realm: &str, - area: &str, - resource: &str, - key: &[u8], - ) -> Result>, String> { - self.state - .runtime() - .admin_get_committed_value(route_family, realm, area, resource, key) - } - - /// Scan a committed KV prefix directly from storage for admin inspection. - /// - /// # Errors - /// - /// Returns an error when the storage transaction or scan fails. - pub fn admin_scan_committed_prefix( - &self, - route_family: crate::runtime::routing::RouteFamily, - realm: &str, - area: &str, - resource: &str, - key_prefix: &[u8], - limit: usize, - ) -> Result { - self.state.runtime().admin_scan_committed_prefix( - route_family, - realm, - area, - resource, - key_prefix, - limit, - ) - } - - /// Scan committed KV rows with an optional pagination cursor. - /// - /// # Errors - /// - /// Returns an error when cursor validation fails or the storage scan fails. - pub fn admin_scan_committed_rows( - &self, - request: &AdminKvRowsRequest<'_>, - ) -> Result { - self.state.runtime().admin_scan_committed_rows(request) - } -} - -impl KvDomainRuntime<'_> { - /// Build an admin inventory snapshot for the requested route family scope. - /// - /// # Errors - /// - /// Returns an error when the underlying storage inventory scan fails. - pub fn admin_inventory( - &self, - family: Option, - ) -> Result, String> { - let families = if let Some(family) = family { - vec![family.id()] - } else { - self.core - .store - .list_column_families() - .map_err(|error| error.to_string())? - .into_iter() - .map(|handle| handle.id()) - .filter(|family_id| *family_id != 0) - .collect::>() - }; - - let mut entries = Vec::new(); - for family_id in families { - entries.extend(self.admin_inventory_for_family(u64::from(family_id))?); - } - entries.sort_by(|left, right| { - ( - left.route_family, - left.realm.as_str(), - left.area.as_str(), - left.resource.as_str(), - ) - .cmp(&( - right.route_family, - right.realm.as_str(), - right.area.as_str(), - right.resource.as_str(), - )) - }); - Ok(entries) - } - - /// Read one admin inventory entry for a specific KV resource. - /// - /// # Errors - /// - /// Returns an error when storage reads, estimate refreshes, or estimate - /// decoding fails. - pub fn admin_inventory_resource( - &self, - route_family: crate::runtime::routing::RouteFamily, - realm: &str, - area: &str, - resource: &str, - ) -> Result, String> { - let family_id = route_family.as_u64(); - let column_family = KvActor::resolve_column_family(route_family, resource)?; - let key = KvActor::inventory_metadata_key(realm, area, resource); - let tx = self - .core - .store - .begin_tx(column_family, cntryl_midge::TransactionMode::ReadOnly) - .map_err(|error| error.to_string())?; - - let estimate = if let Some(value) = tx.get(&key).map_err(|error| error.to_string())? { - KvActor::decode_inventory_estimate(&value)? - } else { - let refreshed = - self.refresh_inventory_estimate(family_id, realm, area, resource, false)?; - if refreshed.estimated_record_count == 0 && refreshed.estimate_complete { - return Ok(None); - } - refreshed - }; - let estimate = if estimate.estimate_complete { - estimate - } else { - self.refresh_inventory_estimate(family_id, realm, area, resource, true)? - }; - - Ok(Some(self.inventory_entry_from_estimate( - family_id, realm, area, resource, estimate, - ))) - } - - /// Read one committed KV value directly from storage for admin inspection. - /// - /// # Errors - /// - /// Returns an error when the storage transaction or read fails. - pub fn admin_get_committed_value( - &self, - route_family: crate::runtime::routing::RouteFamily, - realm: &str, - area: &str, - resource: &str, - key: &[u8], - ) -> Result>, String> { - let started_at = std::time::Instant::now(); - let column_family = KvActor::resolve_column_family(route_family, resource)?; - let tx = self - .core - .store - .begin_tx(column_family, cntryl_midge::TransactionMode::ReadOnly) - .map_err(|error| error.to_string())?; - let prefix = KvActor::realm_resource_prefix(realm, area, resource); - let scoped_key = KvActor::encode_scoped_key(&prefix, key); - let value = tx - .get(&scoped_key) - .map(|value| value.map(|value| value.as_ref().to_vec())) - .map_err(|error| error.to_string())?; - self.record_read_latency( - &KvResourceLockKey::new(route_family.as_u64(), realm, area, resource), - started_at, - ); - Ok(value) - } - - /// Scan a committed KV prefix directly from storage for admin inspection. - /// - /// # Errors - /// - /// Returns an error when the storage transaction or scan fails. - pub fn admin_scan_committed_prefix( - &self, - route_family: crate::runtime::routing::RouteFamily, - realm: &str, - area: &str, - resource: &str, - key_prefix: &[u8], - limit: usize, - ) -> Result { - let started_at = std::time::Instant::now(); - let column_family = KvActor::resolve_column_family(route_family, resource)?; - let tx = self - .core - .store - .begin_tx(column_family, cntryl_midge::TransactionMode::ReadOnly) - .map_err(|error| error.to_string())?; - let resource_prefix = KvActor::realm_resource_prefix(realm, area, resource); - let scoped_prefix = KvActor::encode_scoped_key(&resource_prefix, key_prefix); - let mut rows = Self::scan_scoped_prefix( - &tx, - &resource_prefix, - &scoped_prefix, - &scoped_prefix, - limit.saturating_add(1), - )?; - - let has_more = rows.len() > limit; - rows.truncate(limit); - self.record_read_latency( - &KvResourceLockKey::new(route_family.as_u64(), realm, area, resource), - started_at, - ); - Ok((rows, has_more)) - } - - /// Scan committed KV rows with an optional pagination cursor. - /// - /// # Errors - /// - /// Returns an error when cursor validation fails or the storage scan fails. - pub fn admin_scan_committed_rows( - &self, - request: &AdminKvRowsRequest<'_>, - ) -> Result { - let started_at = std::time::Instant::now(); - if let Some(cursor) = request.cursor { - if !cursor.starts_with(request.starts_with) { - return Err("cursor must start with starts_with prefix".to_string()); - } - } - - let column_family = KvActor::resolve_column_family(request.route_family, request.resource)?; - let tx = self - .core - .store - .begin_tx(column_family, cntryl_midge::TransactionMode::ReadOnly) - .map_err(|error| error.to_string())?; - let resource_prefix = - KvActor::realm_resource_prefix(request.realm, request.area, request.resource); - let scoped_prefix = KvActor::encode_scoped_key(&resource_prefix, request.starts_with); - let scoped_start = request.cursor.map_or_else( - || scoped_prefix.clone(), - |cursor| KvActor::encode_scoped_key(&resource_prefix, cursor), - ); - let mut rows = Self::scan_scoped_prefix( - &tx, - &resource_prefix, - &scoped_prefix, - &scoped_start, - request.limit.saturating_add(1), - )?; - rows.retain(|(user_key, _)| { - request - .cursor - .is_none_or(|cursor| user_key.as_slice() > cursor) - }); - - let has_more = rows.len() > request.limit; - rows.truncate(request.limit); - let next_cursor = if has_more { - rows.last().map(|(key, _)| key.clone()) - } else { - None - }; - self.record_read_latency( - &KvResourceLockKey::new( - request.route_family.as_u64(), - request.realm, - request.area, - request.resource, - ), - started_at, - ); - Ok((rows, next_cursor, has_more)) - } - - pub(super) fn admin_inventory_for_family( - &self, - family_id: u64, - ) -> Result, String> { - let route_family = crate::runtime::routing::RouteFamily::try_from(family_id) - .map_err(|_| format!("invalid route family ID: {family_id}"))?; - let column_family = KvActor::resolve_column_family(route_family, "")?; - let tx = self - .core - .store - .begin_tx(column_family, cntryl_midge::TransactionMode::ReadOnly) - .map_err(|error| error.to_string())?; - let mut iterator = tx - .scan(&cntryl_midge::Query::new()) - .map_err(|error| error.to_string())?; - let mut discovered = Vec::new(); - - for entry in iterator.by_ref() { - let (key, value) = entry.map_err(|error| error.to_string())?; - let Some((realm, area, resource)) = KvActor::parse_inventory_metadata_key(&key) else { - continue; - }; - let estimate = KvActor::decode_inventory_estimate(&value)?; - discovered.push((realm, area, resource, estimate)); - } - - drop(iterator); - drop(tx); - - discovered - .into_iter() - .map(|(realm, area, resource, estimate)| { - let estimate = if estimate.estimate_complete { - estimate - } else { - self.refresh_inventory_estimate(family_id, &realm, &area, &resource, true)? - }; - Ok(self - .inventory_entry_from_estimate(family_id, &realm, &area, &resource, estimate)) - }) - .collect() - } - - pub(super) fn refresh_inventory_estimate( - &self, - family_id: u64, - realm: &str, - area: &str, - resource: &str, - persist_empty: bool, - ) -> Result { - let route_family = crate::runtime::routing::RouteFamily::try_from(family_id) - .map_err(|_| format!("invalid route family ID: {family_id}"))?; - let column_family = KvActor::resolve_column_family(route_family, resource)?; - let read_tx = self - .core - .store - .begin_tx(column_family, cntryl_midge::TransactionMode::ReadOnly) - .map_err(|error| error.to_string())?; - let resource_prefix = KvActor::realm_resource_prefix(realm, area, resource); - let mut rows = Self::scan_scoped_prefix( - &read_tx, - &resource_prefix, - &resource_prefix, - &resource_prefix, - ADMIN_INVENTORY_REFRESH_LIMIT.saturating_add(1), - )?; - let has_more = rows.len() > ADMIN_INVENTORY_REFRESH_LIMIT; - rows.truncate(ADMIN_INVENTORY_REFRESH_LIMIT); - let count = u64::try_from(rows.len()).unwrap_or(u64::MAX); - let storage_bytes = rows.iter().fold(0u64, |total, (key, value)| { - total - .saturating_add(key.len() as u64) - .saturating_add(value.len() as u64) - }); - - let estimate_complete = !has_more; - let estimate = crate::domains::kv::actor::KvInventoryEstimate { - estimated_record_count: count, - estimated_storage_bytes: storage_bytes, - estimate_complete, - }; - - drop(read_tx); - - if persist_empty || estimate.estimated_record_count > 0 || !estimate.estimate_complete { - let mut write_tx = self - .core - .store - .begin_tx(column_family, cntryl_midge::TransactionMode::ReadWrite) - .map_err(|error| error.to_string())?; - write_tx - .put( - KvActor::inventory_metadata_key(realm, area, resource), - KvActor::encode_inventory_estimate(estimate), - None, - ) - .map_err(|error| error.to_string())?; - write_tx - .commit(self.core.sync_write_options) - .map_err(|error| error.to_string())?; - } - - Ok(estimate) - } - - pub(super) fn inventory_entry_from_estimate( - &self, - family_id: u64, - realm: &str, - area: &str, - resource: &str, - estimate: crate::domains::kv::actor::KvInventoryEstimate, - ) -> crate::control::admin::KvResourceInventoryEntry { - let resource_key = KvResourceLockKey::new(family_id, realm, area, resource); - let (read_latency, write_latency) = self.latency_snapshots(&resource_key); - crate::control::admin::KvResourceInventoryEntry { - route_family: family_id, - realm: realm.to_string(), - area: area.to_string(), - resource: resource.to_string(), - estimated_record_count: estimate.estimated_record_count, - estimated_storage_bytes: estimate.estimated_storage_bytes, - estimate_complete: estimate.estimate_complete, - read_latency_avg_ms: read_latency.avg_ms, - read_latency_p95_ms: read_latency.p95_ms, - write_latency_avg_ms: write_latency.avg_ms, - write_latency_p95_ms: write_latency.p95_ms, - transactions_active: self.active_transactions_for_resource(&resource_key), - } - } - - #[cfg(test)] - pub(super) fn sync_admin_snapshot(&self) { - let started_at = Utc::now().to_rfc3339(); - let actors: Vec<_> = self - .core - .actors - .lock() - .iter() - .map(|(session_id, actor)| (*session_id, actor.clone())) - .collect(); - let transactions = actors - .iter() - .flat_map(|(session_id, actor)| { - actor.lock().active_transaction_scopes().into_iter().map( - |(tx_id, family_id, realm, area, resource)| { - crate::control::admin::KvTransaction::snapshot( - family_id, - tx_id, - *session_id, - &realm, - &area, - &resource, - &started_at, - ) - }, - ) - }) - .collect(); - self.core.projection.mark_dirty(); - self.core.projection.refresh_if_dirty(|| transactions); - self.refresh_metrics_gauges(); - } - - pub(super) fn apply_admin_transaction_update(&self, update: KvAdminTransactionUpdate) { - match update { - KvAdminTransactionUpdate::None => return, - KvAdminTransactionUpdate::Upsert(transaction) => { - self.core.projection.upsert_transaction(transaction); - } - KvAdminTransactionUpdate::Remove { session_id, tx_id } => { - self.core.projection.remove_transaction(session_id, tx_id); - } - } - self.refresh_metrics_gauges(); - } - - pub(super) fn refresh_metrics_gauges(&self) { - if let Some(metrics) = &self.core.metrics { - metrics.set_active_transactions(self.active_transaction_count()); - metrics.set_subscription_count(self.subscription_count()); - } - } - - pub(super) fn subscription_count(&self) -> usize { - self.core - .watch_actors - .lock() - .values() - .map(crate::domains::kv::watch::KvWatchActor::subscription_count) - .sum() - } - - pub(super) fn active_transactions_for_resource( - &self, - resource_key: &KvResourceLockKey, - ) -> usize { - self.core.projection.active_transactions_for_resource( - resource_key.family_id, - &resource_key.realm, - &resource_key.area, - &resource_key.resource, - ) - } - - pub(super) fn conflicting_session_for_resource( - &self, - session_id: u64, - resource_key: &KvResourceLockKey, - ) -> Option { - self.core - .resource_locks - .lock() - .get(resource_key) - .filter(|owner| owner.session_id != session_id) - .map(|owner| owner.session_id) - } - - pub(super) fn session_holds_resource_write_lock( - &self, - session_id: u64, - resource_key: &KvResourceLockKey, - ) -> bool { - self.core - .resource_locks - .lock() - .get(resource_key) - .is_some_and(|owner| owner.session_id == session_id) - } - - pub(super) fn latency_snapshots( - &self, - resource_key: &KvResourceLockKey, - ) -> ( - crate::control::admin::KvLatencySnapshot, - crate::control::admin::KvLatencySnapshot, - ) { - self.core.projection.latency_snapshots(resource_key) - } - - pub(super) fn record_read_latency( - &self, - resource_key: &KvResourceLockKey, - started_at: std::time::Instant, - ) { - self.core - .projection - .record_read_latency(resource_key, started_at.elapsed().as_secs_f64() * 1000.0); - } - - pub(super) fn record_write_latency( - &self, - resource_key: &KvResourceLockKey, - started_at: std::time::Instant, - ) { - self.core - .projection - .record_write_latency(resource_key, started_at.elapsed().as_secs_f64() * 1000.0); - } - - pub(super) fn resource_key_for_tx( - &self, - session_id: u64, - tx_id: u64, - ) -> Option { - let actor = self.core.actors.lock().get(&session_id).cloned(); - actor - .and_then(|actor| actor.lock().resource_scope_for_tx(tx_id)) - .map(|(family_id, realm, area, resource)| { - KvResourceLockKey::new(family_id, &realm, &area, &resource) - }) - } - - #[cfg(test)] - pub(super) fn session_inbox_address( - family_id: crate::runtime::routing::RouteFamily, - session_id: u64, - ) -> crate::runtime::routing::RouteAddress { - crate::runtime::routing::RouteAddress::new( - family_id, - crate::runtime::routing::Route::new(format!("inbox://session/{session_id}")), - ) - } -} diff --git a/src/domains/kv/sink/domain_sink_impl/admin_inventory.rs b/src/domains/kv/sink/domain_sink_impl/admin_inventory.rs deleted file mode 100644 index ba34f548..00000000 --- a/src/domains/kv/sink/domain_sink_impl/admin_inventory.rs +++ /dev/null @@ -1,32 +0,0 @@ -use super::KvDomainRuntime; -use crate::domains::kv::sink::AdminKvCommittedPair; -use bytes::Bytes; - -impl KvDomainRuntime<'_> { - pub(super) fn scan_scoped_prefix( - tx: &cntryl_midge::Transaction, - resource_prefix: &[u8], - scoped_prefix: &[u8], - scoped_start: &[u8], - limit: usize, - ) -> Result, String> { - let query = cntryl_midge::Query::new() - .prefix(Bytes::copy_from_slice(scoped_prefix)) - .start_key(Bytes::copy_from_slice(scoped_start)) - .end_key(Bytes::from(crate::domains::kv::KvActor::prefix_range_end( - scoped_prefix, - ))) - .limit(limit); - let iterator = tx.scan(&query).map_err(|error| error.to_string())?; - let mut rows = Vec::new(); - for entry in iterator { - let (scoped_key, value) = entry.map_err(|error| error.to_string())?; - if let Some(user_key) = - crate::domains::kv::KvActor::strip_scoped_prefix(resource_prefix, &scoped_key) - { - rows.push((user_key, value.to_vec())); - } - } - Ok(rows) - } -} diff --git a/src/domains/kv/sink/domain_sink_impl/routing.rs b/src/domains/kv/sink/domain_sink_impl/routing.rs deleted file mode 100644 index df0f7a44..00000000 --- a/src/domains/kv/sink/domain_sink_impl/routing.rs +++ /dev/null @@ -1,236 +0,0 @@ -use super::super::model::{DeliveryError, Envelope, KvDomainRuntime, KvResourceLockKey}; -#[cfg(test)] -use super::test_channels::test_protocol_channel_from_client; -#[cfg(test)] -use crate::dispatch::protocol::frame_context::FrameContext; - -impl KvDomainRuntime<'_> { - fn kv_route_for_lock(resource_key: &KvResourceLockKey) -> crate::runtime::routing::Route { - crate::runtime::routing::Route::new(format!( - "kv://{}/{}/{}", - resource_key.realm, resource_key.area, resource_key.resource - )) - } - - fn route_kv_notify_to_subscription( - &self, - session_id: u64, - subscription_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - route: &crate::runtime::routing::Route, - mutation_count: u64, - ) { - #[cfg(test)] - { - let payload = crate::dispatch::protocol::kv::encode_notify( - subscription_id, - route, - crate::domains::kv::KvNotification { mutation_count }, - ); - let notify_ctx = FrameContext::new( - session_id, - crate::dispatch::protocol::frame::ChannelId::Sub, - crate::dispatch::protocol::tlv::MessageType::new( - crate::dispatch::protocol::kv::msg_type::NOTIFY, - ), - bytes::Bytes::from(payload), - *subscriber.family(), - ); - let notify_envelope = Envelope::new(subscriber.clone(), notify_ctx); - if self.core.router.route(notify_envelope).is_err() { - crate::observability::counter_inc( - crate::domains::kv::metrics::METRIC_NOTIFY_DROPS_TOTAL, - ); - } - } - - #[cfg(not(test))] - { - let notification = crate::domains::kv::KvClientNotification::new( - session_id, - *subscriber.family(), - subscription_id, - route.clone(), - crate::domains::kv::KvNotification { mutation_count }, - ); - let notify_envelope = Envelope::new(subscriber.clone(), notification); - if self.core.router.route(notify_envelope).is_err() { - crate::observability::counter_inc( - crate::domains::kv::metrics::METRIC_NOTIFY_DROPS_TOTAL, - ); - } - } - } - - pub(in crate::domains::kv::sink) fn route_kv_notification( - &self, - resource_key: &KvResourceLockKey, - mutation_count: u64, - ) { - let (route, watch_targets) = { - let watch_actors = self.core.watch_actors.lock(); - let Some(actor) = watch_actors.get(&resource_key.family_id) else { - return; - }; - let route = Self::kv_route_for_lock(resource_key); - let watch_targets = actor.matching_targets(&route); - (route, watch_targets) - }; - for target in watch_targets { - self.route_kv_notify_to_subscription( - target.session_id, - target.subscription_id, - &target.subscriber, - &route, - mutation_count, - ); - } - } - - pub(in crate::domains::kv::sink) fn route_kv_response( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - response: &crate::domains::kv::KvResponse, - request_started: Option, - ) -> Result<(), DeliveryError> { - #[cfg(test)] - let response_ctx = { - let response_bytes = crate::dispatch::protocol::kv::encode_response(response); - tracing::trace!( - domain = "kv", - session = meta.session_id, - response_len = response_bytes.len(), - "KV response encoded" - ); - - FrameContext::new( - meta.session_id, - test_protocol_channel_from_client(meta.channel), - crate::dispatch::protocol::tlv::MessageType::new(meta.message_type), - bytes::Bytes::from(response_bytes), - meta.route_family, - ) - }; - - #[cfg(not(test))] - let response_ctx = crate::domains::kv::KvClientResponse::new(meta, response.clone()); - - let Some(response_envelope) = envelope.try_reply_to(response_ctx) else { - if let (Some(metrics), Some(started_at)) = (self.core.metrics.as_ref(), request_started) - { - if matches!(response, crate::domains::kv::KvResponse::Error { .. }) { - metrics.record_failure(started_at); - } else { - metrics.record_success(started_at); - } - } - tracing::warn!( - domain = "kv", - session = meta.session_id, - "Cannot route response: envelope has no source address" - ); - return Ok(()); - }; - - match self.core.router.route(response_envelope) { - Ok(()) => { - if let (Some(metrics), Some(started_at)) = - (self.core.metrics.as_ref(), request_started) - { - if matches!(response, crate::domains::kv::KvResponse::Error { .. }) { - metrics.record_failure(started_at); - } else { - metrics.record_success(started_at); - } - } - tracing::debug!( - domain = "kv", - session = meta.session_id, - "KV message handled and response routed" - ); - Ok(()) - } - Err(error) => { - if let (Some(metrics), Some(started_at)) = - (self.core.metrics.as_ref(), request_started) - { - metrics.record_failure(started_at); - } - tracing::warn!( - domain = "kv", - session = meta.session_id, - error = ?error, - "Failed to route response" - ); - // Preserve why delivery failed. Reporting backpressure as a - // stopped actor discards the occupancy the caller needs to tell - // a transient full mailbox from a dead one. - Err(match error { - crate::runtime::RouteError::DeliveryFailed(_, delivery_error) => delivery_error, - crate::runtime::RouteError::RouteNotFound(_) => DeliveryError::ActorStopped, - }) - } - } - } - - /// Remove all live KV transaction state owned by a disconnected session. - /// - /// This is the authoritative boundary for session-scoped cleanup: open - /// transactions are dropped, resource locks are released, and the admin read - /// model is refreshed so no durable recovery is implied. - pub(in crate::domains::kv::sink) fn cleanup_session(&self, session_id: u64) { - self.core.actors.lock().remove(&session_id); - self.core - .resource_locks - .lock() - .retain(|_, owner| owner.session_id != session_id); - - { - let mut watch_actors = self.core.watch_actors.lock(); - for actor in watch_actors.values_mut() { - actor.remove_session(session_id); - } - watch_actors.retain(|_, actor| !actor.is_empty()); - } - - tracing::debug!( - domain = "kv", - session = session_id, - "All KV transactions and resource locks released for session (disconnect cleanup)" - ); - self.core.projection.remove_session_transactions(session_id); - self.refresh_metrics_gauges(); - } - - pub(in crate::domains::kv::sink) fn active_transaction_count(&self) -> usize { - self.core.projection.active_transaction_count() - } - - pub(in crate::domains::kv::sink) fn apply_write_options( - &self, - message: crate::domains::kv::KvMessage, - ) -> crate::domains::kv::KvMessage { - match message { - crate::domains::kv::KvMessage::Begin { - scope, - mode, - write_options, - } if write_options.is_sync() - || write_options == cntryl_midge::WriteOptions::buffered() => - { - let write_options = if write_options.is_sync() { - self.core.sync_write_options - } else { - self.core.buffered_write_options - }; - crate::domains::kv::KvMessage::Begin { - scope, - mode, - write_options, - } - } - message => message, - } - } -} diff --git a/src/domains/kv/sink/domain_sink_impl/test_channels.rs b/src/domains/kv/sink/domain_sink_impl/test_channels.rs deleted file mode 100644 index 69c75c6e..00000000 --- a/src/domains/kv/sink/domain_sink_impl/test_channels.rs +++ /dev/null @@ -1,16 +0,0 @@ -pub(super) fn test_protocol_channel_from_client( - channel: crate::runtime::ClientChannel, -) -> crate::dispatch::protocol::frame::ChannelId { - match channel { - crate::runtime::ClientChannel::Control => { - crate::dispatch::protocol::frame::ChannelId::Control - } - crate::runtime::ClientChannel::Pub => crate::dispatch::protocol::frame::ChannelId::Pub, - crate::runtime::ClientChannel::Sub => crate::dispatch::protocol::frame::ChannelId::Sub, - crate::runtime::ClientChannel::Rpc => crate::dispatch::protocol::frame::ChannelId::Rpc, - crate::runtime::ClientChannel::Lease => crate::dispatch::protocol::frame::ChannelId::Lease, - crate::runtime::ClientChannel::Internal => { - crate::dispatch::protocol::frame::ChannelId::Internal - } - } -} diff --git a/src/domains/kv/sink/lifecycle.rs b/src/domains/kv/sink/lifecycle.rs new file mode 100644 index 00000000..9df76948 --- /dev/null +++ b/src/domains/kv/sink/lifecycle.rs @@ -0,0 +1,180 @@ +//! Sink construction, pre-registration configuration, and actor lifecycle. + +use super::state::{ + KvDomainCore, KvDomainMailboxActor, KvDomainRuntime, KvDomainSink, KvDomainState, +}; +use crate::runtime::routing::{Route, RouteAddress, RouteFamily}; +use crate::runtime::{ManagedActor, Router}; +use parking_lot::Mutex; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use super::commands::KvDomainCommand; + +impl KvDomainState { + #[must_use] + fn new( + store: Arc, + router: Arc, + admin_read_model: Arc, + ) -> Self { + Self { + core: KvDomainCore { + store, + actors: Arc::new(Mutex::new(HashMap::new())), + resource_locks: Mutex::new(HashMap::new()), + watch_registries: Mutex::new(HashMap::new()), + cleaned_up_sessions: Mutex::new(crate::runtime::CleanedUpSessions::new( + crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, + )), + router, + projection: crate::domains::kv::admin_projection::KvAdminProjection::new( + admin_read_model, + ), + metrics: None, + sync_write_options: cntryl_midge::WriteOptions::sync(), + buffered_write_options: cntryl_midge::WriteOptions::buffered(), + idle_transaction_ttl: std::time::Duration::from_mins(5), + }, + active: AtomicBool::new(true), + } + } + + pub(super) fn runtime(&self) -> KvDomainRuntime<'_> { + KvDomainRuntime { + core: &self.core, + active: &self.active, + } + } +} + +impl KvDomainMailboxActor { + #[must_use] + pub(super) fn new(state: Arc) -> Self { + Self { state } + } + + pub(super) fn route_address() -> RouteAddress { + RouteAddress::new(RouteFamily::new(0), Route::new("internal://domain/kv")) + } +} + +impl KvDomainSink { + #[must_use] + pub fn new( + store: Arc, + router: Arc, + admin_read_model: Arc, + ) -> Self { + let state = Arc::new(KvDomainState::new(store, router, admin_read_model)); + let actor = Self::spawn_actor(state.clone()); + Self { state, actor } + } + + fn spawn_actor(state: Arc) -> ManagedActor { + let router = state.core.router.clone(); + crate::runtime::ManagedActor::spawn_fail_closed( + router, + KvDomainMailboxActor::route_address(), + move || KvDomainMailboxActor::new(state.clone()), + crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, + ) + } + + fn rebuild_actor(&mut self) { + self.actor.stop(); + self.actor = Self::spawn_actor(self.state.clone()); + } + + fn state_for_builder(&mut self) -> &mut KvDomainState { + Arc::get_mut(&mut self.state).expect("KV sink builders must run before sharing the sink") + } + + #[must_use] + /// Configure the sync policy before registering or sharing this sink. + /// + /// Like every consuming `with_*` method here, this updates private state + /// and rebuilds the sink's managed actor before returning the new value. + pub fn with_sync_write_options(self, write_options: cntryl_midge::WriteOptions) -> Self { + let buffered_write_options = + if write_options.is_cloud_async() || write_options.is_cloud_strict() { + cntryl_midge::WriteOptions::cloud_async() + } else { + cntryl_midge::WriteOptions::buffered() + }; + self.with_write_options(write_options, buffered_write_options) + } + + #[must_use] + /// Configure sync and buffered policies before registering or sharing this sink. + /// + /// This consuming method rebuilds the sink's private managed actor. + pub fn with_write_options( + mut self, + sync_write_options: cntryl_midge::WriteOptions, + buffered_write_options: cntryl_midge::WriteOptions, + ) -> Self { + self.actor.stop(); + let core = &mut self.state_for_builder().core; + core.sync_write_options = sync_write_options; + core.buffered_write_options = buffered_write_options; + self.rebuild_actor(); + self + } + + #[must_use] + /// Configure idle transaction expiry before registering or sharing this sink. + /// + /// This consuming method rebuilds the sink's private managed actor. + pub fn with_idle_transaction_ttl(mut self, ttl: std::time::Duration) -> Self { + self.actor.stop(); + self.state_for_builder().core.idle_transaction_ttl = ttl; + self.rebuild_actor(); + self + } + + #[must_use] + /// Configure the KV metrics collector before registering or sharing this sink. + /// + /// This consuming method rebuilds the sink's private managed actor. + pub fn with_metrics( + mut self, + collector: crate::observability::metrics::MetricsCollector, + ) -> Self { + self.actor.stop(); + let state = self.state_for_builder(); + state.core.metrics = Some(crate::domains::kv::metrics::KvMetrics::new(collector)); + state.runtime().refresh_metrics_gauges(); + self.rebuild_actor(); + self + } + + pub fn stop(&self) { + self.state.active.store(false, Ordering::Relaxed); + self.actor.stop(); + } + + pub(crate) fn actor_health_snapshot(&self) -> crate::runtime::ManagedActorHealthSnapshot { + self.actor.health_snapshot() + } + + #[cfg(test)] + pub(super) fn is_actor_running(&self) -> bool { + self.actor.is_running() + } + + #[cfg(test)] + /// Mark the mailbox actor permanently failed without delivering a panic. + pub(crate) fn mark_actor_permanently_failed_for_tests(&self) { + self.actor.mark_permanently_failed_for_tests(); + } + + #[cfg(test)] + /// Trigger the mailbox actor's fail-closed panic path. + pub(crate) fn panic_actor_for_tests(&self) { + let _ = self + .actor + .try_send_high_priority(KvDomainCommand::PanicForTests); + } +} diff --git a/src/domains/kv/sink/locks.rs b/src/domains/kv/sink/locks.rs new file mode 100644 index 00000000..2ed20932 --- /dev/null +++ b/src/domains/kv/sink/locks.rs @@ -0,0 +1,164 @@ +//! Live resource-lock identities and ownership coordination. + +use super::state::KvDomainRuntime; +use crate::domains::kv::{KvActor, KvMessage}; +use parking_lot::Mutex; +use std::sync::Arc; + +/// Identifies the in-memory write lock owner for a single resource scope. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub(crate) struct KvResourceLockKey { + pub(super) family_id: u64, + pub(super) realm: String, + pub(super) area: String, + pub(super) resource: String, +} + +impl KvResourceLockKey { + #[must_use] + pub(crate) fn new(family_id: u64, realm: &str, area: &str, resource: &str) -> Self { + Self { + family_id, + realm: realm.to_string(), + area: area.to_string(), + resource: resource.to_string(), + } + } + + #[must_use] + pub(super) fn from_scope(scope: &crate::domains::kv::KvResourceScope) -> Self { + Self::new( + scope.route_family.as_u64(), + &scope.realm, + &scope.area, + &scope.resource, + ) + } +} + +#[derive(Clone, Copy)] +pub(super) struct KvResourceLockOwner { + /// Session that owns the active write transaction lock. + pub(super) session_id: u64, + /// Active transaction id that currently holds the lock. + pub(super) tx_id: u64, + /// Last request activity used for idle lock expiry. + pub(super) last_activity: std::time::Instant, +} + +struct KvTransactionLock { + tx_id: u64, + resource_key: KvResourceLockKey, +} + +impl KvDomainRuntime<'_> { + pub(super) fn expire_idle_transactions_for_session(&self, session_id: u64) { + let actor = self.core.actors.lock().get(&session_id).cloned(); + if let Some(actor) = actor { + self.remove_expired_transactions(session_id, &actor); + } + } + + fn remove_expired_transactions(&self, session_id: u64, actor: &Arc>) { + for tx_id in actor + .lock() + .expire_idle_transactions(self.core.idle_transaction_ttl) + { + self.core + .resource_locks + .lock() + .retain(|_, owner| owner.session_id != session_id || owner.tx_id != tx_id); + self.core.projection.remove_transaction(session_id, tx_id); + } + } + + pub(super) fn expire_resource_lock_if_idle(&self, resource_key: &KvResourceLockKey) { + let owner = self.core.resource_locks.lock().get(resource_key).copied(); + let Some(owner) = + owner.filter(|owner| owner.last_activity.elapsed() >= self.core.idle_transaction_ttl) + else { + return; + }; + let actor = self.core.actors.lock().get(&owner.session_id).cloned(); + if let Some(actor) = actor { + actor.lock().rollback_transaction(owner.tx_id); + } + self.core.resource_locks.lock().remove(resource_key); + self.core + .projection + .remove_transaction(owner.session_id, owner.tx_id); + } + + fn transaction_lock(message: &crate::domains::kv::KvMessage) -> Option { + let (tx_id, scope) = match message { + KvMessage::Begin { .. } => return None, + KvMessage::Commit { tx_id, scope } + | KvMessage::Rollback { tx_id, scope } + | KvMessage::Get { tx_id, scope, .. } + | KvMessage::Put { tx_id, scope, .. } + | KvMessage::Insert { tx_id, scope, .. } + | KvMessage::Delete { tx_id, scope, .. } + | KvMessage::DeleteRange { tx_id, scope, .. } + | KvMessage::Scan { tx_id, scope, .. } => (*tx_id, scope), + }; + Some(KvTransactionLock { + tx_id, + resource_key: KvResourceLockKey::from_scope(scope), + }) + } + + pub(super) fn touch_resource_lock( + &self, + session_id: u64, + message: &crate::domains::kv::KvMessage, + ) { + let Some(transaction_lock) = Self::transaction_lock(message) else { + return; + }; + let mut locks = self.core.resource_locks.lock(); + if let Some(owner) = locks.get_mut(&transaction_lock.resource_key) { + if owner.session_id == session_id && owner.tx_id == transaction_lock.tx_id { + owner.last_activity = std::time::Instant::now(); + } + } + } + + pub(super) fn conflicting_session_for_resource( + &self, + session_id: u64, + resource_key: &KvResourceLockKey, + ) -> Option { + self.core + .resource_locks + .lock() + .get(resource_key) + .filter(|owner| owner.session_id != session_id) + .map(|owner| owner.session_id) + } + + /// Returns true if this session currently holds the write lock for the resource. + /// + /// The lock table tracks write transactions only. + pub(super) fn session_holds_resource_lock( + &self, + session_id: u64, + resource_key: &KvResourceLockKey, + ) -> bool { + self.core + .resource_locks + .lock() + .get(resource_key) + .is_some_and(|owner| owner.session_id == session_id) + } + + pub(super) fn resource_key_for_tx( + &self, + session_id: u64, + tx_id: u64, + ) -> Option { + let actor = self.core.actors.lock().get(&session_id).cloned(); + actor + .and_then(|actor| actor.lock().resource_scope_for_tx(tx_id)) + .map(|scope| KvResourceLockKey::from_scope(&scope)) + } +} diff --git a/src/domains/kv/sink/mailbox.rs b/src/domains/kv/sink/mailbox.rs new file mode 100644 index 00000000..a7eb01de --- /dev/null +++ b/src/domains/kv/sink/mailbox.rs @@ -0,0 +1,54 @@ +//! Mailbox boundary for the managed KV domain actor. + +use super::commands::KvDomainCommand; +use super::state::{KvDomainMailboxActor, KvDomainSink}; +use crate::runtime::{Actor, Context, DeliveryError, Envelope, MailboxSink}; + +impl MailboxSink for KvDomainSink { + fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.actor.try_send(KvDomainCommand::Deliver(envelope)) + } + + fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.actor + .try_send_high_priority(KvDomainCommand::Deliver(envelope)) + } +} + +impl Actor for KvDomainMailboxActor { + type Message = KvDomainCommand; + + fn receive(&mut self, msg: Self::Message, _ctx: &mut Context) { + match msg { + KvDomainCommand::Deliver(envelope) => { + if let Err(error) = self.state.runtime().deliver_envelope(&envelope) { + tracing::warn!(domain = "kv", error = %error, "KV actor delivery failed"); + } + } + KvDomainCommand::CleanupSession(session_id, reply) => { + self.state.runtime().cleanup_session(session_id); + let _ = reply.send(()); + } + KvDomainCommand::ReadActiveTransactionCount(reply) => { + let _ = reply.send(self.state.runtime().active_transaction_count()); + } + #[cfg(test)] + KvDomainCommand::SyncAdminSnapshot(reply) => { + self.state.runtime().sync_admin_snapshot(); + let _ = reply.send(()); + } + #[cfg(test)] + KvDomainCommand::ReadLatencySnapshots(resource_key, reply) => { + let _ = reply.send(self.state.runtime().latency_snapshots(&resource_key)); + } + #[cfg(test)] + KvDomainCommand::ApplyWriteOptions(message, reply) => { + let _ = reply.send(self.state.runtime().apply_write_options(message)); + } + #[cfg(test)] + KvDomainCommand::PanicForTests => { + panic!("test KV domain actor panic"); + } + } + } +} diff --git a/src/domains/kv/sink/mailbox_sink_impl.rs b/src/domains/kv/sink/mailbox_sink_impl.rs deleted file mode 100644 index 246df647..00000000 --- a/src/domains/kv/sink/mailbox_sink_impl.rs +++ /dev/null @@ -1,812 +0,0 @@ -use super::model::{ - Arc, DeliveryError, Envelope, KvAdminTransactionUpdate, KvClientFrame, KvClientRequest, - KvDomainActor, KvDomainCommand, KvDomainRuntime, KvDomainSink, KvOperationOutcome, - KvResourceLockKey, KvResourceLockOwner, MailboxSink, Mutex, Ordering, -}; -#[cfg(test)] -use crate::dispatch::protocol::frame_context::FrameContext; -use crate::domains::kv::{KvActor, KvError, KvResponse}; -use crate::runtime::{Actor, Context}; - -impl MailboxSink for KvDomainSink { - fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { - self.actor.try_send(KvDomainCommand::Deliver(envelope)) - } - - fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { - self.actor - .try_send_high_priority(KvDomainCommand::Deliver(envelope)) - } -} - -impl Actor for KvDomainActor { - type Message = KvDomainCommand; - - fn receive(&mut self, msg: Self::Message, _ctx: &mut Context) { - match msg { - KvDomainCommand::Deliver(envelope) => { - if let Err(error) = self.state.runtime().deliver_envelope(&envelope) { - tracing::warn!(domain = "kv", error = %error, "KV actor delivery failed"); - } - } - KvDomainCommand::CleanupSession(session_id, reply) => { - self.state.runtime().cleanup_session(session_id); - let _ = reply.send(()); - } - KvDomainCommand::ReadActiveTransactionCount(reply) => { - let _ = reply.send(self.state.runtime().active_transaction_count()); - } - #[cfg(test)] - KvDomainCommand::SyncAdminSnapshot(reply) => { - self.state.runtime().sync_admin_snapshot(); - let _ = reply.send(()); - } - #[cfg(test)] - KvDomainCommand::ReadLatencySnapshots(resource_key, reply) => { - let _ = reply.send(self.state.runtime().latency_snapshots(&resource_key)); - } - #[cfg(test)] - KvDomainCommand::ApplyWriteOptions(message, reply) => { - let _ = reply.send(self.state.runtime().apply_write_options(message)); - } - #[cfg(test)] - KvDomainCommand::PanicForTests => { - panic!("test KV domain actor panic"); - } - } - } -} - -impl KvDomainRuntime<'_> { - pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { - if self.handle_cleanup_envelope(envelope) { - return Ok(()); - } - self.ensure_active()?; - Self::log_delivery(envelope); - - let request = Self::extract_request(envelope)?; - let meta = request.meta; - let request_started = self.record_request_start(); - if !Self::valid_request_envelope(envelope, meta) { - let response = Self::error_response("route family mismatch"); - let response_meta = Self::response_meta_for_source(envelope, meta); - self.route_kv_response(envelope, response_meta, &response, request_started)?; - return Ok(()); - } - - let operation_started = Self::record_operation_start(); - let Some(parsed_frame) = - self.parse_request_frame(envelope, meta, request.frame, request_started) - else { - return Ok(()); - }; - - match parsed_frame { - KvClientFrame::Sub(sub_msg) => { - self.handle_subscription_frame(envelope, meta, request_started, sub_msg) - } - KvClientFrame::Op(kv_message) => self.handle_actor_operation_frame( - envelope, - meta, - request_started, - operation_started, - kv_message, - ), - } - } - - fn handle_cleanup_envelope(&self, envelope: &Envelope) -> bool { - if let Some(cleanup) = envelope.payload::() { - self.cleanup_session(cleanup.session_id); - return true; - } - - false - } - - fn ensure_active(&self) -> Result<(), DeliveryError> { - if !self.active.load(Ordering::Relaxed) { - return Err(DeliveryError::ActorStopped); - } - - Ok(()) - } - - fn log_delivery(envelope: &Envelope) { - tracing::debug!( - domain = "kv", - destination = %envelope.destination(), - source = ?envelope.source(), - "KV domain sink: received envelope" - ); - } - - fn extract_request(envelope: &Envelope) -> Result { - Self::request_from_envelope(envelope).ok_or_else(|| { - tracing::warn!( - domain = "kv", - destination = ?envelope.destination(), - "Envelope payload was not KvClientRequest" - ); - DeliveryError::ActorStopped - }) - } - - fn record_operation_start() -> std::time::Instant { - std::time::Instant::now() - } - - fn record_request_start(&self) -> Option { - self.core - .metrics - .as_ref() - .map(super::super::metrics::KvMetrics::record_request_start) - } - - fn parse_request_frame( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - frame: Result, - request_started: Option, - ) -> Option { - let parsed_frame = match frame { - Ok(msg) => msg, - Err(e) => { - tracing::warn!( - domain = "kv", - session = meta.session_id, - msg_type = meta.message_type, - error = %e, - "Failed to parse KV message" - ); - let response = Self::error_response(&e); - let response_meta = Self::response_meta_for_source(envelope, meta); - let _ = self.route_kv_response(envelope, response_meta, &response, request_started); - return None; - } - }; - - tracing::debug!( - domain = "kv", - session = meta.session_id, - channel = ?meta.channel, - msg_type = meta.message_type, - "Parsed KV message successfully" - ); - - Some(parsed_frame) - } - - fn handle_subscription_frame( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - request_started: Option, - sub_msg: crate::domains::kv::KvSubscriptionMessage, - ) -> Result<(), DeliveryError> { - let response = match sub_msg { - crate::domains::kv::KvSubscriptionMessage::Subscribe { - family_id, - pattern, - session_id, - subscriber, - } => self - .handle_kv_subscribe(envelope, meta, family_id, &pattern, session_id, subscriber), - crate::domains::kv::KvSubscriptionMessage::Unsubscribe { - family_id, - pattern, - session_id, - subscriber, - } => self.handle_kv_unsubscribe( - envelope, - meta, - family_id, - &pattern, - session_id, - &subscriber, - ), - }; - - self.refresh_metrics_gauges(); - self.route_kv_response(envelope, meta, &response, request_started) - } - - fn handle_kv_subscribe( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - family_id: crate::runtime::routing::RouteFamily, - pattern: &crate::runtime::routing::Route, - session_id: u64, - subscriber: crate::runtime::routing::RouteAddress, - ) -> KvResponse { - if Self::valid_subscription_request(envelope, meta, family_id, session_id, &subscriber) { - let compiled = match Self::compile_kv_subscription_pattern(pattern) { - Ok(compiled) => compiled, - Err(response) => return response, - }; - let subscription_id = { - let mut watch_actors = self.core.watch_actors.lock(); - let actor = watch_actors - .entry(family_id.as_u64()) - .or_insert_with(|| crate::domains::kv::watch::KvWatchActor::new(family_id)); - let subscription_id = actor.subscribe(session_id, compiled, subscriber); - if subscription_id.is_err() && actor.is_empty() { - watch_actors.remove(&family_id.as_u64()); - } - subscription_id - }; - subscription_id.map_or_else( - |error| KvResponse::Error { error }, - |subscription_id| KvResponse::SubscribeOk { subscription_id }, - ) - } else { - Self::error_response("route family mismatch") - } - } - - fn handle_kv_unsubscribe( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - family_id: crate::runtime::routing::RouteFamily, - pattern: &crate::runtime::routing::Route, - session_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - ) -> KvResponse { - if Self::valid_subscription_request(envelope, meta, family_id, session_id, subscriber) { - if let Err(response) = Self::compile_kv_subscription_pattern(pattern) { - return response; - } - let mut watch_actors = self.core.watch_actors.lock(); - let remove_family = if let Some(actor) = watch_actors.get_mut(&family_id.as_u64()) { - actor.unsubscribe(session_id, pattern.as_str()); - actor.is_empty() - } else { - false - }; - if remove_family { - watch_actors.remove(&family_id.as_u64()); - } - KvResponse::UnsubscribeOk - } else { - Self::error_response("route family mismatch") - } - } - - fn compile_kv_subscription_pattern( - pattern: &crate::runtime::routing::Route, - ) -> Result { - crate::runtime::DomainKind::Kv - .descriptor() - .compile_registration_pattern(pattern.as_str()) - .map_err(|error| KvResponse::Error { - error: KvError::InvalidSubscriptionPattern(error), - }) - } - - fn handle_actor_operation_frame( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - request_started: Option, - operation_started: std::time::Instant, - kv_message: crate::domains::kv::KvMessage, - ) -> Result<(), DeliveryError> { - use crate::domains::kv::{KvMessage, TxMode}; - if Self::kv_message_family(&kv_message) != meta.route_family { - let response = Self::error_response("route family mismatch"); - self.route_kv_response(envelope, meta, &response, request_started)?; - return Ok(()); - } - - let kv_message = self.apply_write_options(kv_message); - let session_id = meta.session_id; - let read_tx_id = match &kv_message { - KvMessage::Get { tx_id, .. } | KvMessage::Scan { tx_id, .. } => Some(*tx_id), - _ => None, - }; - let is_commit = matches!(&kv_message, KvMessage::Commit { .. }); - - if matches!( - &kv_message, - KvMessage::Begin { - mode: TxMode::ReadWrite, - .. - } - ) { - if let KvMessage::Begin { scope, .. } = &kv_message { - self.expire_resource_lock_if_idle(&KvResourceLockKey::new( - scope.route_family.as_u64(), - &scope.realm, - &scope.area, - &scope.resource, - )); - } - } else { - self.expire_idle_transactions_for_session(session_id); - } - - tracing::trace!( - domain = "kv", - session_id = session_id, - msg_type = meta.message_type, - "KV deliver: getting or creating actor for session" - ); - - self.touch_resource_lock(session_id, &kv_message); - let KvOperationOutcome { - response, - admin_update, - commit_notification, - } = self.dispatch_actor_operation(session_id, meta, kv_message); - if matches!( - &response, - KvResponse::Error { - error: KvError::InvalidTxId, - .. - } - ) { - crate::observability::counter_inc("fitz_kv_invalid_transaction_rejects_total"); - } - match (&response, read_tx_id, is_commit) { - (KvResponse::GetResult { .. } | KvResponse::ScanResult { .. }, Some(tx_id), _) => { - if let Some(resource_key) = self.resource_key_for_tx(session_id, tx_id) { - self.record_read_latency(&resource_key, operation_started); - } - } - (KvResponse::CommitOk, _, true) => { - if let Some((resource_key, _)) = commit_notification.as_ref() { - self.record_write_latency(resource_key, operation_started); - } - } - _ => {} - } - self.apply_admin_transaction_update(admin_update); - if let Some((resource_key, mutation_count)) = commit_notification { - self.route_kv_notification(&resource_key, mutation_count); - } - - tracing::debug!( - domain = "kv", - session = meta.session_id, - response = ?std::mem::discriminant(&response), - "KV actor returned response" - ); - - self.route_kv_response(envelope, meta, &response, request_started) - } - - fn dispatch_actor_operation( - &self, - session_id: u64, - meta: crate::runtime::ClientFrameMeta, - kv_message: crate::domains::kv::KvMessage, - ) -> KvOperationOutcome { - use crate::domains::kv::{KvMessage, TxMode}; - let write_scope = match &kv_message { - KvMessage::Begin { scope, mode, .. } if *mode == TxMode::ReadWrite => Some(( - scope.route_family.as_u64(), - scope.realm.clone(), - scope.area.clone(), - scope.resource.clone(), - )), - _ => None, - }; - if let Some((family_id, realm, area, resource)) = write_scope { - return self.handle_begin_read_write( - session_id, family_id, &realm, &area, &resource, kv_message, - ); - } - match kv_message { - message @ KvMessage::Commit { tx_id, .. } => { - self.handle_commit_frame(session_id, meta.route_family, tx_id, message) - } - message @ KvMessage::Rollback { tx_id, .. } => { - self.handle_rollback_frame(session_id, meta.route_family, tx_id, message) - } - message => self.handle_regular_operation_frame(session_id, meta.message_type, message), - } - } - - fn actor_for_session(&self, session_id: u64, context: &str) -> Arc> { - self.core - .actors - .lock() - .entry(session_id) - .or_insert_with(|| { - tracing::trace!( - domain = "kv", - session_id = session_id, - "Creating new KvActor instance ({context})" - ); - Arc::new(Mutex::new(KvActor::new(self.core.store.clone()))) - }) - .clone() - } - - fn expire_idle_transactions_for_session(&self, session_id: u64) { - let actor = self.core.actors.lock().get(&session_id).cloned(); - if let Some(actor) = actor { - self.remove_expired_transactions(session_id, &actor); - } - } - - fn remove_expired_transactions(&self, session_id: u64, actor: &Arc>) { - for tx_id in actor - .lock() - .expire_idle_transactions(self.core.idle_transaction_ttl) - { - self.core - .resource_locks - .lock() - .retain(|_, owner| owner.session_id != session_id || owner.tx_id != tx_id); - self.core.projection.remove_transaction(session_id, tx_id); - } - } - - fn expire_resource_lock_if_idle(&self, resource_key: &KvResourceLockKey) { - let owner = self.core.resource_locks.lock().get(resource_key).copied(); - let Some(owner) = - owner.filter(|owner| owner.last_activity.elapsed() >= self.core.idle_transaction_ttl) - else { - return; - }; - let actor = self.core.actors.lock().get(&owner.session_id).cloned(); - if let Some(actor) = actor { - actor.lock().rollback_transaction(owner.tx_id); - } - self.core.resource_locks.lock().remove(resource_key); - self.core - .projection - .remove_transaction(owner.session_id, owner.tx_id); - } - - fn transaction_resource( - message: &crate::domains::kv::KvMessage, - ) -> Option<(u64, KvResourceLockKey)> { - use crate::domains::kv::KvMessage; - let (tx_id, scope) = match message { - KvMessage::Begin { .. } => return None, - KvMessage::Commit { tx_id, scope } - | KvMessage::Rollback { tx_id, scope } - | KvMessage::Get { tx_id, scope, .. } - | KvMessage::Put { tx_id, scope, .. } - | KvMessage::Insert { tx_id, scope, .. } - | KvMessage::Delete { tx_id, scope, .. } - | KvMessage::DeleteRange { tx_id, scope, .. } - | KvMessage::Scan { tx_id, scope, .. } => (*tx_id, scope), - }; - Some(( - tx_id, - KvResourceLockKey::new( - scope.route_family.as_u64(), - &scope.realm, - &scope.area, - &scope.resource, - ), - )) - } - - fn touch_resource_lock(&self, session_id: u64, message: &crate::domains::kv::KvMessage) { - let Some((tx_id, resource_key)) = Self::transaction_resource(message) else { - return; - }; - let mut locks = self.core.resource_locks.lock(); - if let Some(owner) = locks.get_mut(&resource_key) { - if owner.session_id == session_id && owner.tx_id == tx_id { - owner.last_activity = std::time::Instant::now(); - } - } - } - - fn handle_begin_read_write( - &self, - session_id: u64, - family_id: u64, - realm: &str, - area: &str, - resource: &str, - kv_message: crate::domains::kv::KvMessage, - ) -> KvOperationOutcome { - let lock_key = KvResourceLockKey::new(family_id, realm, area, resource); - let held_by_same_session = self.session_holds_resource_write_lock(session_id, &lock_key); - if self - .conflicting_session_for_resource(session_id, &lock_key) - .is_some() - { - return KvOperationOutcome::new( - KvResponse::Error { - error: KvError::Conflict("resource locked by another session".to_string()), - }, - KvAdminTransactionUpdate::None, - None, - ); - } - if held_by_same_session { - return KvOperationOutcome::new( - KvResponse::Error { - error: KvError::Conflict( - "resource already has a read-write transaction for this session" - .to_string(), - ), - }, - KvAdminTransactionUpdate::None, - None, - ); - } - - let log_context = "BEGIN (ReadWrite, acquiring lock)"; - let actor = self.actor_for_session(session_id, "begin"); - let mut actor = actor.lock(); - tracing::trace!( - domain = "kv", - session_id = session_id, - "Calling actor.handle() for {log_context}" - ); - let response = actor.handle(kv_message); - if let KvResponse::BeginOk { tx_id } = response { - self.core.resource_locks.lock().insert( - lock_key, - KvResourceLockOwner { - session_id, - tx_id, - last_activity: std::time::Instant::now(), - }, - ); - tracing::trace!( - domain = "kv", - session_id = session_id, - tx_id = tx_id, - "BEGIN succeeded with actor-owned transaction scope" - ); - let transaction = crate::control::admin::KvTransaction::snapshot( - family_id, - tx_id, - session_id, - realm, - area, - resource, - &chrono::Utc::now().to_rfc3339(), - ); - KvOperationOutcome::new( - response, - KvAdminTransactionUpdate::Upsert(transaction), - None, - ) - } else { - KvOperationOutcome::new(response, KvAdminTransactionUpdate::None, None) - } - } - - fn handle_commit_frame( - &self, - session_id: u64, - route_family: crate::runtime::routing::RouteFamily, - tx_id: u64, - kv_message: crate::domains::kv::KvMessage, - ) -> KvOperationOutcome { - let actor = self.actor_for_session(session_id, "commit"); - let mut actor = actor.lock(); - tracing::trace!( - domain = "kv", - session_id = session_id, - tx_id = tx_id, - "Calling actor.handle() for COMMIT" - ); - let mutation_count = actor.mutation_count_for_tx(tx_id).unwrap_or(0); - let lock_key = - actor - .resource_scope_for_tx(tx_id) - .map(|(family_id, realm, area, resource)| { - KvResourceLockKey::new(family_id, &realm, &area, &resource) - }); - if lock_key - .as_ref() - .is_some_and(|key| key.family_id != route_family.as_u64()) - { - return KvOperationOutcome::new( - KvResponse::Error { - error: KvError::InvalidRequest("route family mismatch".to_string()), - }, - KvAdminTransactionUpdate::None, - None, - ); - } - let had_transaction = lock_key.is_some(); - let response = actor.handle(kv_message); - let admin_update = if had_transaction && actor.resource_scope_for_tx(tx_id).is_none() { - if let Some(lock_key) = &lock_key { - self.core.resource_locks.lock().remove(lock_key); - } - KvAdminTransactionUpdate::Remove { session_id, tx_id } - } else { - KvAdminTransactionUpdate::None - }; - if let KvResponse::CommitOk = response { - if let Some(lock_key) = lock_key { - let notify = (mutation_count > 0).then_some((lock_key, mutation_count)); - KvOperationOutcome::new(response, admin_update, notify) - } else { - KvOperationOutcome::new(response, admin_update, None) - } - } else { - crate::observability::counter_inc("fitz_kv_commits_failed_total"); - KvOperationOutcome::new(response, admin_update, None) - } - } - - fn handle_rollback_frame( - &self, - session_id: u64, - route_family: crate::runtime::routing::RouteFamily, - tx_id: u64, - kv_message: crate::domains::kv::KvMessage, - ) -> KvOperationOutcome { - let actor = self.actor_for_session(session_id, "rollback"); - let mut actor = actor.lock(); - tracing::trace!( - domain = "kv", - session_id = session_id, - tx_id = tx_id, - "Calling actor.handle() for ROLLBACK" - ); - let resource_scope = actor.resource_scope_for_tx(tx_id); - if resource_scope - .as_ref() - .is_some_and(|(family_id, _, _, _)| *family_id != route_family.as_u64()) - { - return KvOperationOutcome::new( - KvResponse::Error { - error: KvError::InvalidRequest("route family mismatch".to_string()), - }, - KvAdminTransactionUpdate::None, - None, - ); - } - let response = actor.handle(kv_message); - let admin_update = - if resource_scope.is_some() && actor.resource_scope_for_tx(tx_id).is_none() { - if let Some((family_id, realm, area, resource)) = &resource_scope { - self.core - .resource_locks - .lock() - .remove(&KvResourceLockKey::new(*family_id, realm, area, resource)); - } - KvAdminTransactionUpdate::Remove { session_id, tx_id } - } else { - KvAdminTransactionUpdate::None - }; - if let KvResponse::RollbackOk = response { - crate::observability::counter_inc("fitz_kv_rollbacks_total"); - KvOperationOutcome::new(response, admin_update, None) - } else { - KvOperationOutcome::new(response, admin_update, None) - } - } - - fn handle_regular_operation_frame( - &self, - session_id: u64, - message_type: u16, - kv_message: crate::domains::kv::KvMessage, - ) -> KvOperationOutcome { - let actor = self.actor_for_session(session_id, "other operation"); - let mut actor = actor.lock(); - tracing::trace!( - domain = "kv", - session_id = session_id, - msg_type = message_type, - "Calling actor.handle() for operation" - ); - KvOperationOutcome::new( - actor.handle(kv_message), - KvAdminTransactionUpdate::None, - None, - ) - } - - fn request_from_envelope(envelope: &Envelope) -> Option { - if let Some(request) = envelope.payload::() { - return Some(request.clone()); - } - - #[cfg(test)] - { - let frame_ctx = envelope.payload::()?.clone(); - let subscriber = envelope.source().cloned().unwrap_or_else(|| { - Self::session_inbox_address(frame_ctx.route_family, frame_ctx.session_id) - }); - let meta = crate::runtime::ClientFrameMeta::new( - frame_ctx.session_id, - test_client_channel_from_protocol(frame_ctx.channel_id), - frame_ctx.msg_type.as_u16(), - frame_ctx.route_family, - ); - let parsed = crate::dispatch::protocol::kv::parse_frame( - &frame_ctx, - &frame_ctx.payload, - frame_ctx.route_family, - frame_ctx.session_id, - subscriber, - ) - .map(|frame| match frame { - crate::dispatch::protocol::kv::ParsedKvFrame::Op(message) => { - KvClientFrame::Op(message) - } - crate::dispatch::protocol::kv::ParsedKvFrame::Sub(message) => { - KvClientFrame::Sub(message) - } - }); - Some(KvClientRequest::new(meta, parsed)) - } - - #[cfg(not(test))] - { - None - } - } - - fn valid_request_envelope(envelope: &Envelope, meta: crate::runtime::ClientFrameMeta) -> bool { - meta.route_family == *envelope.destination().family() - && envelope - .source() - .is_none_or(|source| *source.family() == meta.route_family) - } - - fn valid_subscription_request( - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - family_id: crate::runtime::routing::RouteFamily, - session_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - ) -> bool { - family_id == meta.route_family - && *subscriber.family() == family_id - && session_id == meta.session_id - && envelope.source().is_none_or(|source| source == subscriber) - } - - fn kv_message_family( - message: &crate::domains::kv::KvMessage, - ) -> crate::runtime::routing::RouteFamily { - message.scope().route_family - } - - fn error_response(reason: &str) -> KvResponse { - KvResponse::Error { - error: KvError::InvalidRequest(reason.to_string()), - } - } - - fn response_meta_for_source( - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - ) -> crate::runtime::ClientFrameMeta { - envelope.source().map_or(meta, |source| { - let mut response_meta = meta; - response_meta.route_family = *source.family(); - response_meta - }) - } -} - -#[cfg(test)] -fn test_client_channel_from_protocol( - channel: crate::dispatch::protocol::frame::ChannelId, -) -> crate::runtime::ClientChannel { - match channel { - crate::dispatch::protocol::frame::ChannelId::Control => { - crate::runtime::ClientChannel::Control - } - crate::dispatch::protocol::frame::ChannelId::Pub => crate::runtime::ClientChannel::Pub, - crate::dispatch::protocol::frame::ChannelId::Sub => crate::runtime::ClientChannel::Sub, - crate::dispatch::protocol::frame::ChannelId::Rpc => crate::runtime::ClientChannel::Rpc, - crate::dispatch::protocol::frame::ChannelId::Lease => crate::runtime::ClientChannel::Lease, - crate::dispatch::protocol::frame::ChannelId::Internal => { - crate::runtime::ClientChannel::Internal - } - } -} diff --git a/src/domains/kv/sink/mod.rs b/src/domains/kv/sink/mod.rs index e86f8ddd..6167b5c1 100644 --- a/src/domains/kv/sink/mod.rs +++ b/src/domains/kv/sink/mod.rs @@ -1,18 +1,27 @@ -mod actor_commands; -mod domain_sink_impl; -mod mailbox_sink_impl; -mod model; +//! KV runtime sink, admin façade, and behavior-focused internal modules. + +mod admin; +mod cleanup; +mod commands; +mod delivery; +mod lifecycle; +mod locks; +mod mailbox; +mod observability; +mod operations; +mod responses; +mod state; +mod subscriptions; #[cfg(test)] -mod test_actor_commands; +mod test_support; +mod transactions; +mod write_policy; -pub(crate) use model::KvResourceLockKey; -pub use model::{ +pub use admin::{ AdminKvCommittedPair, AdminKvPrefixScanResult, AdminKvRowsRequest, AdminKvRowsResult, - KvDomainSink, }; - -#[cfg(test)] -use model::*; +pub(super) use locks::KvResourceLockKey; +pub use state::KvDomainSink; #[cfg(test)] mod tests; diff --git a/src/domains/kv/sink/model.rs b/src/domains/kv/sink/model.rs deleted file mode 100644 index aab80b3a..00000000 --- a/src/domains/kv/sink/model.rs +++ /dev/null @@ -1,143 +0,0 @@ -// KV domain sink for session-scoped transaction dispatch. -// -// Committed KV writes flow straight to Midge and persist according to the -// `WriteOptions` selected when the transaction commits. Active `tx_id` -// handles, resource locks, and admin snapshot entries are separate live -// in-memory state owned by the current broker process. `cleanup_session` -// intentionally discards that state on disconnect, and broker restart clears -// it wholesale instead of attempting transaction recovery. - -#[cfg(test)] -pub(super) use crate::dispatch::protocol::frame_context::FrameContext; -pub(super) use crate::domains::kv::{KvClientFrame, KvClientRequest}; -pub(super) use crate::runtime::routing::RouteFamily; -pub(super) use crate::runtime::{DeliveryError, Envelope, MailboxSink, ManagedActor, Router}; -#[cfg(test)] -pub(super) use chrono::Utc; -pub(super) use parking_lot::Mutex; -pub(super) use std::collections::HashMap; -pub(super) use std::sync::atomic::{AtomicBool, Ordering}; -pub(super) use std::sync::Arc; - -pub type AdminKvCommittedPair = (Vec, Vec); -pub type AdminKvPrefixScanResult = (Vec, bool); -pub type AdminKvRowsResult = (Vec, Option>, bool); - -pub struct AdminKvRowsRequest<'a> { - pub route_family: RouteFamily, - pub realm: &'a str, - pub area: &'a str, - pub resource: &'a str, - pub starts_with: &'a [u8], - pub cursor: Option<&'a [u8]>, - pub limit: usize, -} - -pub(super) const ADMIN_INVENTORY_REFRESH_LIMIT: usize = 10_000; - -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub(crate) struct KvResourceLockKey { - pub(super) family_id: u64, - pub(super) realm: String, - pub(super) area: String, - pub(super) resource: String, -} - -impl KvResourceLockKey { - pub(crate) fn new(family_id: u64, realm: &str, area: &str, resource: &str) -> Self { - Self { - family_id, - realm: realm.to_string(), - area: area.to_string(), - resource: resource.to_string(), - } - } -} - -#[derive(Clone, Copy)] -pub(super) struct KvResourceLockOwner { - pub(super) session_id: u64, - pub(super) tx_id: u64, - pub(super) last_activity: std::time::Instant, -} - -pub(super) struct KvDomainCore { - pub(super) store: Arc, - pub(super) actors: Arc>>>>, - pub(super) resource_locks: Mutex>, - pub(super) watch_actors: Mutex>, - pub(super) router: Arc, - pub(super) projection: crate::domains::kv::projection::KvAdminProjection, - pub(super) metrics: Option, - pub(super) sync_write_options: cntryl_midge::WriteOptions, - pub(super) buffered_write_options: cntryl_midge::WriteOptions, - pub(super) idle_transaction_ttl: std::time::Duration, -} - -pub(super) struct KvDomainState { - pub(super) core: KvDomainCore, - pub(super) active: AtomicBool, -} - -pub(super) struct KvDomainRuntime<'a> { - pub(super) core: &'a KvDomainCore, - pub(super) active: &'a AtomicBool, -} - -pub(super) enum KvAdminTransactionUpdate { - None, - Upsert(crate::control::admin::KvTransaction), - Remove { session_id: u64, tx_id: u64 }, -} - -pub(super) struct KvOperationOutcome { - pub(super) response: crate::domains::kv::KvResponse, - pub(super) admin_update: KvAdminTransactionUpdate, - pub(super) commit_notification: Option<(KvResourceLockKey, u64)>, -} - -impl KvOperationOutcome { - pub(super) fn new( - response: crate::domains::kv::KvResponse, - admin_update: KvAdminTransactionUpdate, - commit_notification: Option<(KvResourceLockKey, u64)>, - ) -> Self { - Self { - response, - admin_update, - commit_notification, - } - } -} - -pub(super) enum KvDomainCommand { - Deliver(Envelope), - CleanupSession(u64, crossbeam_channel::Sender<()>), - ReadActiveTransactionCount(crossbeam_channel::Sender), - #[cfg(test)] - SyncAdminSnapshot(crossbeam_channel::Sender<()>), - #[cfg(test)] - ReadLatencySnapshots( - KvResourceLockKey, - crossbeam_channel::Sender<( - crate::control::admin::KvLatencySnapshot, - crate::control::admin::KvLatencySnapshot, - )>, - ), - #[cfg(test)] - ApplyWriteOptions( - crate::domains::kv::KvMessage, - crossbeam_channel::Sender, - ), - #[cfg(test)] - PanicForTests, -} - -pub(super) struct KvDomainActor { - pub(super) state: Arc, -} - -pub struct KvDomainSink { - pub(super) state: Arc, - pub(super) actor: ManagedActor, -} diff --git a/src/domains/kv/sink/observability.rs b/src/domains/kv/sink/observability.rs new file mode 100644 index 00000000..72c65883 --- /dev/null +++ b/src/domains/kv/sink/observability.rs @@ -0,0 +1,168 @@ +//! KV metrics, latency, and admin-projection updates. + +use super::locks::KvResourceLockKey; +use super::state::{KvAdminTransactionUpdate, KvDomainRuntime}; +#[cfg(test)] +use chrono::Utc; + +impl KvDomainRuntime<'_> { + pub(super) fn counter_inc(&self, name: &str) { + if let Some(metrics) = &self.core.metrics { + metrics.counter_inc(name); + } else { + crate::observability::counter_inc(name); + } + } + + pub(super) fn record_response_metrics( + &self, + response: &crate::domains::kv::KvResponse, + started_at: std::time::Instant, + ) { + self.record_request_metrics( + matches!(response, crate::domains::kv::KvResponse::Error { .. }), + started_at, + ); + } + + pub(super) fn record_request_metrics(&self, failed: bool, started_at: std::time::Instant) { + if let Some(metrics) = &self.core.metrics { + if failed { + metrics.record_failure(started_at); + } else { + metrics.record_success(started_at); + } + return; + } + + crate::observability::counter_inc(if failed { + crate::domains::kv::metrics::METRIC_FAILURE_TOTAL + } else { + crate::domains::kv::metrics::METRIC_SUCCESS_TOTAL + }); + let elapsed_ms = u64::try_from(started_at.elapsed().as_millis()).unwrap_or(u64::MAX); + crate::observability::metrics() + .histogram_observe_ms(crate::domains::kv::metrics::METRIC_LATENCY_MS, elapsed_ms); + } + + pub(super) fn active_transaction_count(&self) -> usize { + self.core.projection.active_transaction_count() + } + + #[cfg(test)] + pub(super) fn sync_admin_snapshot(&self) { + let started_at = Utc::now().to_rfc3339(); + let actors: Vec<_> = self + .core + .actors + .lock() + .iter() + .map(|(session_id, actor)| (*session_id, actor.clone())) + .collect(); + let transactions = actors + .iter() + .flat_map(|(session_id, actor)| { + actor + .lock() + .active_transaction_snapshots() + .into_iter() + .map(|snapshot| { + crate::control::admin::KvTransaction::snapshot( + snapshot.scope.route_family.as_u64(), + snapshot.tx_id, + *session_id, + &snapshot.scope.realm, + &snapshot.scope.area, + &snapshot.scope.resource, + &started_at, + ) + }) + }) + .collect(); + self.core.projection.mark_dirty(); + self.core.projection.refresh_if_dirty(|| transactions); + self.refresh_metrics_gauges(); + } + + pub(super) fn apply_admin_transaction_update(&self, update: KvAdminTransactionUpdate) { + match update { + KvAdminTransactionUpdate::None => return, + KvAdminTransactionUpdate::Upsert(transaction) => { + self.core.projection.upsert_transaction(transaction); + } + KvAdminTransactionUpdate::Remove { session_id, tx_id } => { + self.core.projection.remove_transaction(session_id, tx_id); + } + } + self.refresh_metrics_gauges(); + } + + pub(super) fn refresh_metrics_gauges(&self) { + if let Some(metrics) = &self.core.metrics { + metrics.set_active_transactions(self.active_transaction_count()); + metrics.set_subscription_count(self.subscription_count()); + } + } + + pub(super) fn subscription_count(&self) -> usize { + self.core + .watch_registries + .lock() + .values() + .map(crate::domains::kv::watch_registry::KvWatchRegistry::subscription_count) + .sum() + } + + pub(super) fn active_transactions_for_resource( + &self, + resource_key: &KvResourceLockKey, + ) -> usize { + self.core.projection.active_transactions_for_resource( + resource_key.family_id, + &resource_key.realm, + &resource_key.area, + &resource_key.resource, + ) + } + + pub(super) fn latency_snapshots( + &self, + resource_key: &KvResourceLockKey, + ) -> ( + crate::control::admin::KvLatencySnapshot, + crate::control::admin::KvLatencySnapshot, + ) { + self.core.projection.latency_snapshots(resource_key) + } + + pub(super) fn record_read_latency( + &self, + resource_key: &KvResourceLockKey, + started_at: std::time::Instant, + ) { + self.core + .projection + .record_read_latency(resource_key, started_at.elapsed().as_secs_f64() * 1000.0); + } + + pub(super) fn record_write_latency( + &self, + resource_key: &KvResourceLockKey, + started_at: std::time::Instant, + ) { + self.core + .projection + .record_write_latency(resource_key, started_at.elapsed().as_secs_f64() * 1000.0); + } + + #[cfg(test)] + pub(super) fn session_inbox_address( + family_id: crate::runtime::routing::RouteFamily, + session_id: u64, + ) -> crate::runtime::routing::RouteAddress { + crate::runtime::routing::RouteAddress::new( + family_id, + crate::runtime::routing::Route::new(format!("inbox://session/{session_id}")), + ) + } +} diff --git a/src/domains/kv/sink/operations.rs b/src/domains/kv/sink/operations.rs new file mode 100644 index 00000000..48de7bee --- /dev/null +++ b/src/domains/kv/sink/operations.rs @@ -0,0 +1,186 @@ +//! Actor lookup, operation dispatch, and request-envelope validation. + +use super::locks::KvResourceLockKey; +use super::state::KvDomainRuntime; +use super::state::{KvAdminTransactionUpdate, KvOperationOutcome}; +#[cfg(test)] +use crate::dispatch::protocol::frame_context::FrameContext; +use crate::domains::kv::KvActor; +#[cfg(test)] +use crate::domains::kv::KvClientFrame; +use crate::domains::kv::KvClientRequest; +use crate::domains::kv::{KvError, KvResponse}; +use crate::runtime::Envelope; +use parking_lot::Mutex; +use std::sync::Arc; + +impl KvDomainRuntime<'_> { + pub(super) fn dispatch_actor_operation( + &self, + session_id: u64, + meta: crate::runtime::ClientFrameMeta, + kv_message: crate::domains::kv::KvMessage, + ) -> KvOperationOutcome { + use crate::domains::kv::{KvMessage, TxMode}; + let write_lock = match &kv_message { + KvMessage::Begin { scope, mode, .. } if *mode == TxMode::ReadWrite => { + Some(KvResourceLockKey::from_scope(scope)) + } + _ => None, + }; + if let Some(lock_key) = write_lock { + return self.handle_begin_read_write(session_id, &lock_key, kv_message); + } + match kv_message { + message @ KvMessage::Commit { tx_id, .. } => { + self.handle_commit_frame(session_id, meta.route_family, tx_id, message) + } + message @ KvMessage::Rollback { tx_id, .. } => { + self.handle_rollback_frame(session_id, meta.route_family, tx_id, message) + } + message => self.handle_regular_operation_frame(session_id, meta.message_type, message), + } + } + + pub(super) fn actor_for_session(&self, session_id: u64, context: &str) -> Arc> { + self.core + .actors + .lock() + .entry(session_id) + .or_insert_with(|| { + tracing::trace!( + domain = "kv", + session_id = session_id, + "Creating new KvActor instance ({context})" + ); + Arc::new(Mutex::new(KvActor::new(self.core.store.clone()))) + }) + .clone() + } + + fn handle_regular_operation_frame( + &self, + session_id: u64, + message_type: u16, + kv_message: crate::domains::kv::KvMessage, + ) -> KvOperationOutcome { + let actor = self.actor_for_session(session_id, "other operation"); + let mut actor = actor.lock(); + tracing::trace!( + domain = "kv", + session_id = session_id, + msg_type = message_type, + "Calling actor.handle() for operation" + ); + KvOperationOutcome::new( + actor.handle(kv_message), + KvAdminTransactionUpdate::None, + None, + ) + } + + pub(super) fn request_from_envelope(envelope: &Envelope) -> Option { + if let Some(request) = envelope.payload::() { + return Some(request.clone()); + } + + #[cfg(test)] + { + let frame_ctx = envelope.payload::()?.clone(); + let subscriber = envelope.source().cloned().unwrap_or_else(|| { + Self::session_inbox_address(frame_ctx.route_family, frame_ctx.session_id) + }); + let meta = crate::runtime::ClientFrameMeta::new( + frame_ctx.session_id, + test_client_channel_from_protocol(frame_ctx.channel_id), + frame_ctx.msg_type.as_u16(), + frame_ctx.route_family, + ); + let parsed = crate::dispatch::protocol::kv::parse_frame( + &frame_ctx, + &frame_ctx.payload, + frame_ctx.route_family, + frame_ctx.session_id, + subscriber, + ) + .map(|frame| match frame { + crate::dispatch::protocol::kv::ParsedKvFrame::Op(message) => { + KvClientFrame::Op(message) + } + crate::dispatch::protocol::kv::ParsedKvFrame::Sub(message) => { + KvClientFrame::Sub(message) + } + }); + Some(KvClientRequest::new(meta, parsed)) + } + + #[cfg(not(test))] + { + None + } + } + + pub(super) fn valid_request_envelope( + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + ) -> bool { + meta.route_family == *envelope.destination().family() + && envelope + .source() + .is_none_or(|source| *source.family() == meta.route_family) + } + + pub(super) fn valid_subscription_request( + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + family_id: crate::runtime::routing::RouteFamily, + session_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + ) -> bool { + family_id == meta.route_family + && *subscriber.family() == family_id + && session_id == meta.session_id + && envelope.source().is_none_or(|source| source == subscriber) + } + + pub(super) fn kv_message_family( + message: &crate::domains::kv::KvMessage, + ) -> crate::runtime::routing::RouteFamily { + message.scope().route_family + } + + pub(super) fn error_response(reason: &str) -> KvResponse { + KvResponse::Error { + error: KvError::InvalidRequest(reason.to_string()), + } + } + + pub(super) fn response_meta_for_source( + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + ) -> crate::runtime::ClientFrameMeta { + envelope.source().map_or(meta, |source| { + let mut response_meta = meta; + response_meta.route_family = *source.family(); + response_meta + }) + } +} + +#[cfg(test)] +fn test_client_channel_from_protocol( + channel: crate::dispatch::protocol::frame::ChannelId, +) -> crate::runtime::ClientChannel { + match channel { + crate::dispatch::protocol::frame::ChannelId::Control => { + crate::runtime::ClientChannel::Control + } + crate::dispatch::protocol::frame::ChannelId::Pub => crate::runtime::ClientChannel::Pub, + crate::dispatch::protocol::frame::ChannelId::Sub => crate::runtime::ClientChannel::Sub, + crate::dispatch::protocol::frame::ChannelId::Rpc => crate::runtime::ClientChannel::Rpc, + crate::dispatch::protocol::frame::ChannelId::Lease => crate::runtime::ClientChannel::Lease, + crate::dispatch::protocol::frame::ChannelId::Internal => { + crate::runtime::ClientChannel::Internal + } + } +} diff --git a/src/domains/kv/sink/responses.rs b/src/domains/kv/sink/responses.rs new file mode 100644 index 00000000..1bfff1c7 --- /dev/null +++ b/src/domains/kv/sink/responses.rs @@ -0,0 +1,76 @@ +//! Response-envelope construction and routing. + +use super::state::KvDomainRuntime; +#[cfg(test)] +use crate::dispatch::protocol::frame_context::FrameContext; +use crate::runtime::{DeliveryError, Envelope}; + +impl KvDomainRuntime<'_> { + pub(super) fn route_kv_response( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + response: &crate::domains::kv::KvResponse, + request_started: std::time::Instant, + ) -> Result<(), DeliveryError> { + #[cfg(test)] + let response_ctx = { + let response_bytes = crate::dispatch::protocol::kv::encode_response(response); + tracing::trace!( + domain = "kv", + session = meta.session_id, + response_len = response_bytes.len(), + "KV response encoded" + ); + + FrameContext::new( + meta.session_id, + crate::protocol::test_support::channel_id_from_client(meta.channel), + crate::dispatch::protocol::tlv::MessageType::new(meta.message_type), + bytes::Bytes::from(response_bytes), + meta.route_family, + ) + }; + + #[cfg(not(test))] + let response_ctx = crate::domains::kv::KvClientResponse::new(meta, response.clone()); + + let Some(response_envelope) = envelope.try_reply_to(response_ctx) else { + self.record_response_metrics(response, request_started); + tracing::warn!( + domain = "kv", + session = meta.session_id, + "Cannot route response: envelope has no source address" + ); + return Ok(()); + }; + + match self.core.router.route(response_envelope) { + Ok(()) => { + self.record_response_metrics(response, request_started); + tracing::debug!( + domain = "kv", + session = meta.session_id, + "KV message handled and response routed" + ); + Ok(()) + } + Err(error) => { + self.record_request_metrics(true, request_started); + tracing::warn!( + domain = "kv", + session = meta.session_id, + error = ?error, + "Failed to route response" + ); + // Preserve why delivery failed. Reporting backpressure as a + // stopped actor discards the occupancy the caller needs to tell + // a transient full mailbox from a dead one. + Err(match error { + crate::runtime::RouteError::DeliveryFailed(_, delivery_error) => delivery_error, + crate::runtime::RouteError::RouteNotFound(_) => DeliveryError::ActorStopped, + }) + } + } + } +} diff --git a/src/domains/kv/sink/state.rs b/src/domains/kv/sink/state.rs new file mode 100644 index 00000000..13c29919 --- /dev/null +++ b/src/domains/kv/sink/state.rs @@ -0,0 +1,84 @@ +//! KV domain sink state for session-scoped transaction dispatch. +// +// Committed KV writes flow straight to Midge and persist according to the +// `WriteOptions` selected when the transaction commits. Active `tx_id` +// handles, resource locks, and admin snapshot entries are separate live +// in-memory state owned by the current broker process. `cleanup_session` +// intentionally discards that state on disconnect, and broker restart clears +// it wholesale instead of attempting transaction recovery. + +use super::commands::KvDomainCommand; +use super::locks::{KvResourceLockKey, KvResourceLockOwner}; +use crate::runtime::CleanedUpSessions; +use crate::runtime::{ManagedActor, Router}; +use parking_lot::Mutex; +use std::collections::HashMap; +use std::sync::atomic::AtomicBool; +use std::sync::Arc; + +pub(super) struct KvDomainCore { + pub(super) store: Arc, + pub(super) actors: Arc>>>>, + pub(super) resource_locks: Mutex>, + pub(super) watch_registries: + Mutex>, + pub(super) cleaned_up_sessions: Mutex, + pub(super) router: Arc, + pub(super) projection: crate::domains::kv::admin_projection::KvAdminProjection, + pub(super) metrics: Option, + pub(super) sync_write_options: cntryl_midge::WriteOptions, + pub(super) buffered_write_options: cntryl_midge::WriteOptions, + pub(super) idle_transaction_ttl: std::time::Duration, +} + +pub(super) struct KvDomainState { + pub(super) core: KvDomainCore, + pub(super) active: AtomicBool, +} + +pub(super) struct KvDomainRuntime<'a> { + pub(super) core: &'a KvDomainCore, + pub(super) active: &'a AtomicBool, +} + +pub(super) enum KvAdminTransactionUpdate { + None, + Upsert(crate::control::admin::KvTransaction), + Remove { session_id: u64, tx_id: u64 }, +} + +pub(super) struct KvOperationOutcome { + pub(super) response: crate::domains::kv::KvResponse, + pub(super) admin_update: KvAdminTransactionUpdate, + pub(super) commit_notification: Option, +} + +pub(super) struct KvCommitNotification { + pub(super) resource_key: KvResourceLockKey, + pub(super) mutation_count: u64, +} + +impl KvOperationOutcome { + #[must_use] + pub(super) fn new( + response: crate::domains::kv::KvResponse, + admin_update: KvAdminTransactionUpdate, + commit_notification: Option, + ) -> Self { + Self { + response, + admin_update, + commit_notification, + } + } +} + +/// Managed mailbox adapter that serializes access to the KV domain runtime. +pub(super) struct KvDomainMailboxActor { + pub(super) state: Arc, +} + +pub struct KvDomainSink { + pub(super) state: Arc, + pub(super) actor: ManagedActor, +} diff --git a/src/domains/kv/sink/subscriptions.rs b/src/domains/kv/sink/subscriptions.rs new file mode 100644 index 00000000..fe24a22b --- /dev/null +++ b/src/domains/kv/sink/subscriptions.rs @@ -0,0 +1,197 @@ +//! KV watch registration, removal, matching, and notification routing. + +use super::locks::KvResourceLockKey; +use super::state::KvDomainRuntime; +#[cfg(test)] +use crate::dispatch::protocol::frame_context::FrameContext; +use crate::domains::kv::{KvError, KvResponse}; +use crate::runtime::{DeliveryError, Envelope}; + +impl KvDomainRuntime<'_> { + pub(super) fn handle_subscription_frame( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + request_started: std::time::Instant, + sub_msg: crate::domains::kv::KvSubscriptionMessage, + ) -> Result<(), DeliveryError> { + let response = match sub_msg { + crate::domains::kv::KvSubscriptionMessage::Subscribe { + family_id, + pattern, + session_id, + subscriber, + } => self + .handle_kv_subscribe(envelope, meta, family_id, &pattern, session_id, subscriber), + crate::domains::kv::KvSubscriptionMessage::Unsubscribe { + family_id, + pattern, + session_id, + subscriber, + } => self.handle_kv_unsubscribe( + envelope, + meta, + family_id, + &pattern, + session_id, + &subscriber, + ), + }; + + self.refresh_metrics_gauges(); + self.route_kv_response(envelope, meta, &response, request_started) + } + + fn handle_kv_subscribe( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + family_id: crate::runtime::routing::RouteFamily, + pattern: &crate::runtime::routing::Route, + session_id: u64, + subscriber: crate::runtime::routing::RouteAddress, + ) -> KvResponse { + if Self::valid_subscription_request(envelope, meta, family_id, session_id, &subscriber) { + let compiled = match Self::compile_kv_subscription_pattern(pattern) { + Ok(compiled) => compiled, + Err(response) => return response, + }; + let subscription_id = { + let mut watch_registries = self.core.watch_registries.lock(); + let registry = watch_registries + .entry(family_id.as_u64()) + .or_insert_with(|| { + crate::domains::kv::watch_registry::KvWatchRegistry::new(family_id) + }); + let subscription_id = registry.subscribe(session_id, compiled, subscriber); + if subscription_id.is_err() && registry.is_empty() { + watch_registries.remove(&family_id.as_u64()); + } + subscription_id + }; + subscription_id.map_or_else( + |error| KvResponse::Error { error }, + |subscription_id| KvResponse::SubscribeOk { subscription_id }, + ) + } else { + Self::error_response("route family mismatch") + } + } + + fn handle_kv_unsubscribe( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + family_id: crate::runtime::routing::RouteFamily, + pattern: &crate::runtime::routing::Route, + session_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + ) -> KvResponse { + if Self::valid_subscription_request(envelope, meta, family_id, session_id, subscriber) { + if let Err(response) = Self::compile_kv_subscription_pattern(pattern) { + return response; + } + let mut watch_registries = self.core.watch_registries.lock(); + let remove_family = + if let Some(registry) = watch_registries.get_mut(&family_id.as_u64()) { + registry.unsubscribe(session_id, pattern.as_str()); + registry.is_empty() + } else { + false + }; + if remove_family { + watch_registries.remove(&family_id.as_u64()); + } + KvResponse::UnsubscribeOk + } else { + Self::error_response("route family mismatch") + } + } + + fn compile_kv_subscription_pattern( + pattern: &crate::runtime::routing::Route, + ) -> Result { + crate::runtime::DomainKind::Kv + .descriptor() + .compile_registration_pattern(pattern.as_str()) + .map_err(|error| KvResponse::Error { + error: KvError::InvalidSubscriptionPattern(error), + }) + } + + fn kv_route_for_lock(resource_key: &KvResourceLockKey) -> crate::runtime::routing::Route { + crate::runtime::routing::Route::new(format!( + "kv://{}/{}/{}", + resource_key.realm, resource_key.area, resource_key.resource + )) + } + + fn route_kv_notify_to_subscription( + &self, + session_id: u64, + subscription_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + route: &crate::runtime::routing::Route, + mutation_count: u64, + ) { + #[cfg(test)] + let notify_envelope = { + let payload = crate::dispatch::protocol::kv::encode_notify( + subscription_id, + route, + crate::domains::kv::KvNotification { mutation_count }, + ); + let notify_ctx = FrameContext::new( + session_id, + crate::dispatch::protocol::frame::ChannelId::Sub, + crate::dispatch::protocol::tlv::MessageType::new( + crate::dispatch::protocol::kv::msg_type::NOTIFY, + ), + bytes::Bytes::from(payload), + *subscriber.family(), + ); + Envelope::new(subscriber.clone(), notify_ctx) + }; + + #[cfg(not(test))] + let notify_envelope = { + let notification = crate::domains::kv::KvClientNotification::new( + session_id, + *subscriber.family(), + subscription_id, + route.clone(), + crate::domains::kv::KvNotification { mutation_count }, + ); + Envelope::new(subscriber.clone(), notification) + }; + + if self.core.router.route(notify_envelope).is_err() { + self.counter_inc(crate::domains::kv::metrics::METRIC_NOTIFY_DROPS_TOTAL); + } + } + + pub(super) fn route_kv_notification( + &self, + resource_key: &KvResourceLockKey, + mutation_count: u64, + ) { + let (route, watch_targets) = { + let watch_registries = self.core.watch_registries.lock(); + let Some(registry) = watch_registries.get(&resource_key.family_id) else { + return; + }; + let route = Self::kv_route_for_lock(resource_key); + let watch_targets = registry.matching_targets(&route); + (route, watch_targets) + }; + for target in watch_targets { + self.route_kv_notify_to_subscription( + target.session_id, + target.subscription_id, + &target.subscriber, + &route, + mutation_count, + ); + } + } +} diff --git a/src/domains/kv/sink/test_actor_commands.rs b/src/domains/kv/sink/test_actor_commands.rs deleted file mode 100644 index b2e1e40d..00000000 --- a/src/domains/kv/sink/test_actor_commands.rs +++ /dev/null @@ -1,86 +0,0 @@ -use super::model::{KvDomainCommand, KvDomainSink, KvResourceLockKey}; -use std::sync::atomic::Ordering; -use std::time::Duration; - -impl KvDomainSink { - pub(super) fn is_active_for_tests(&self) -> bool { - self.state.active.load(Ordering::Relaxed) - } - - pub(super) fn insert_actor_for_tests( - &self, - session_id: u64, - actor: crate::domains::kv::KvActor, - ) { - self.state.core.actors.lock().insert( - session_id, - std::sync::Arc::new(parking_lot::Mutex::new(actor)), - ); - } - - pub(super) fn watch_actors_are_empty_for_tests(&self) -> bool { - self.state.core.watch_actors.lock().is_empty() - } - - pub(super) fn actors_are_empty_for_tests(&self) -> bool { - self.state.core.actors.lock().is_empty() - } - - pub(super) fn sync_admin_snapshot(&self) { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = self - .actor - .try_send_high_priority(KvDomainCommand::SyncAdminSnapshot(reply_tx)) - { - tracing::warn!(domain = "kv", error = %error, "KV admin snapshot enqueue failed"); - return; - } - - if let Err(error) = reply_rx.recv_timeout(Duration::from_secs(1)) { - tracing::warn!(domain = "kv", error = %error, "KV admin snapshot reply failed"); - } - } - - pub(super) fn latency_snapshots( - &self, - resource_key: &KvResourceLockKey, - ) -> ( - crate::control::admin::KvLatencySnapshot, - crate::control::admin::KvLatencySnapshot, - ) { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = - self.actor - .try_send_high_priority(KvDomainCommand::ReadLatencySnapshots( - resource_key.clone(), - reply_tx, - )) - { - tracing::warn!(domain = "kv", error = %error, "KV latency snapshot enqueue failed"); - return Default::default(); - } - - reply_rx - .recv_timeout(Duration::from_secs(1)) - .unwrap_or_default() - } - - pub(super) fn apply_write_options( - &self, - message: crate::domains::kv::KvMessage, - ) -> crate::domains::kv::KvMessage { - let fallback = message.clone(); - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = self - .actor - .try_send_high_priority(KvDomainCommand::ApplyWriteOptions(message, reply_tx)) - { - tracing::warn!(domain = "kv", error = %error, "KV write-option mapping enqueue failed"); - return fallback; - } - - reply_rx - .recv_timeout(Duration::from_secs(1)) - .unwrap_or(fallback) - } -} diff --git a/src/domains/kv/sink/test_support.rs b/src/domains/kv/sink/test_support.rs new file mode 100644 index 00000000..d18205f3 --- /dev/null +++ b/src/domains/kv/sink/test_support.rs @@ -0,0 +1,77 @@ +//! Test-only controls for observing and driving the managed KV mailbox actor. + +use super::commands::KvDomainCommand; +use super::locks::KvResourceLockKey; +use super::state::KvDomainSink; + +impl KvDomainSink { + /// Stop the mailbox actor without changing the sink's active flag. + pub(super) fn stop_actor_for_tests(&self) { + self.actor.stop(); + } + + /// Report whether the sink has not been stopped. + pub(super) fn is_active_for_tests(&self) -> bool { + use std::sync::atomic::Ordering; + + self.state.active.load(Ordering::Relaxed) + } + + /// Seed one session actor for state-cleanup regressions. + pub(super) fn insert_actor_for_tests( + &self, + session_id: u64, + actor: crate::domains::kv::KvActor, + ) { + self.state.core.actors.lock().insert( + session_id, + std::sync::Arc::new(parking_lot::Mutex::new(actor)), + ); + } + + /// Report whether all watch registries are empty. + pub(super) fn watch_registries_are_empty_for_tests(&self) -> bool { + self.state.core.watch_registries.lock().is_empty() + } + + /// Report whether all session actors are absent. + pub(super) fn actors_are_empty_for_tests(&self) -> bool { + self.state.core.actors.lock().is_empty() + } + + /// Report whether all write locks are absent. + pub(super) fn resource_locks_are_empty_for_tests(&self) -> bool { + self.state.core.resource_locks.lock().is_empty() + } + + /// Rebuild the admin projection through the mailbox actor. + pub(super) fn sync_admin_snapshot(&self) { + let _ = self.request_actor("sync_admin_snapshot", KvDomainCommand::SyncAdminSnapshot); + } + + /// Read the latency snapshots for one resource through the mailbox actor. + pub(super) fn latency_snapshots( + &self, + resource_key: &KvResourceLockKey, + ) -> ( + crate::control::admin::KvLatencySnapshot, + crate::control::admin::KvLatencySnapshot, + ) { + self.request_actor("latency_snapshots", |reply| { + KvDomainCommand::ReadLatencySnapshots(resource_key.clone(), reply) + }) + .unwrap_or_default() + } + + /// Apply the configured BEGIN write policy through the mailbox actor. + pub(super) fn apply_write_options( + &self, + message: crate::domains::kv::KvMessage, + ) -> crate::domains::kv::KvMessage { + let fallback = message.clone(); + self.request_actor("apply_write_options", |reply| { + KvDomainCommand::ApplyWriteOptions(message, reply) + }) + .unwrap_or(fallback) + } +} diff --git a/src/domains/kv/sink/tests.rs b/src/domains/kv/sink/tests.rs index a0c97c04..f71dc064 100644 --- a/src/domains/kv/sink/tests.rs +++ b/src/domains/kv/sink/tests.rs @@ -1,15 +1,19 @@ use super::*; use crate::dispatch::protocol::error_codes; use crate::dispatch::protocol::frame::ChannelId; +use crate::dispatch::protocol::frame_context::FrameContext; use crate::dispatch::protocol::tlv::MessageType; use crate::domains::kv::KvResourceScope; use crate::runtime::routing::{Route, RouteAddress, RouteFamily}; -use crate::runtime::Mailbox; +use crate::runtime::{Envelope, Mailbox, MailboxSink, Router}; use bytes::{BufMut, Bytes}; use std::sync::Arc; use std::time::{Duration, Instant}; +mod configuration; mod correctness; +mod lifecycle; +mod subscriptions; #[inline] fn usize_to_u32_saturating(value: usize) -> u32 { @@ -127,856 +131,3 @@ fn wait_for_active_transaction_count(sink: &KvDomainSink, expected: usize) { } assert_eq!(sink.active_transaction_count(), expected); } - -#[test] -fn should_create_kv_domain_sink() { - // Arrange - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - - // Act - let sink = KvDomainSink::new(store, router, admin_read_model); - - // Assert - assert!(sink.is_active_for_tests()); - assert!(sink.is_actor_running()); -} - -#[test] -fn should_record_kv_latency_samples_by_operation_kind() { - // Arrange - let family = RouteFamily::new(1); - let session_id = 7; - let kv_route = "kv://acme/app/users"; - let kv_address = RouteAddress::new(family, Route::new(kv_route)); - let source_address = RouteAddress::new(family, Route::new("inbox://session/7")); - let mailbox = Arc::new(Mailbox::new(16)); - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - router.register(source_address.clone(), mailbox.clone()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model); - - sink.deliver(Envelope::from_route( - source_address.clone(), - kv_address.clone(), - FrameContext::new( - session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::BEGIN), - encode_kv_begin(kv_route, 1, 0), - family, - ), - )) - .expect("begin KV transaction"); - let begin_frame = receive_frame(&mailbox, "begin ack envelope"); - let tx_id = decode_kv_begin_tx_id(&begin_frame.payload); - - sink.deliver(Envelope::from_route( - source_address.clone(), - kv_address.clone(), - FrameContext::new( - session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::PUT), - encode_kv_put(tx_id, kv_route, b"user:1", b"alice"), - family, - ), - )) - .expect("put KV value"); - let _ = receive_envelope(&mailbox, "put ack envelope"); - - // Act - sink.deliver(Envelope::from_route( - source_address, - kv_address, - FrameContext::new( - session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::COMMIT), - encode_kv_commit(tx_id, kv_route), - family, - ), - )) - .expect("commit KV transaction"); - let _ = receive_envelope(&mailbox, "commit ack envelope"); - let resource_key = KvResourceLockKey::new(1, "acme", "app", "users"); - let (reads_before, writes_before) = sink.latency_snapshots(&resource_key); - assert!(reads_before.avg_ms.abs() < f64::EPSILON); - assert!(writes_before.avg_ms > 0.0); - let value = sink - .admin_get_committed_value(family, "acme", "app", "users", b"user:1") - .expect("read committed KV value"); - - // Assert - assert_eq!(value.as_deref(), Some(&b"alice"[..])); - let (reads_after, writes_after) = sink.latency_snapshots(&resource_key); - assert!(reads_after.avg_ms > 0.0); - assert!(writes_after.avg_ms > 0.0); -} - -#[test] -fn should_map_sync_begin_to_cloud_strict_given_strict_cloud_sync_policy() { - // Arrange - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model) - .with_sync_write_options(cntryl_midge::WriteOptions::cloud_strict()); - let message = crate::domains::kv::KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "acme".to_string(), - "app".to_string(), - "users".to_string(), - ), - mode: crate::domains::kv::TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::sync(), - }; - - // Act - let mapped = sink.apply_write_options(message); - - // Assert - match mapped { - crate::domains::kv::KvMessage::Begin { write_options, .. } => { - assert!(write_options.is_cloud_strict()); - } - _ => panic!("expected KV begin message"), - } -} - -#[test] -fn should_map_buffered_begin_to_cloud_async_given_cloud_storage() { - // Arrange - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model).with_write_options( - cntryl_midge::WriteOptions::cloud_strict(), - cntryl_midge::WriteOptions::cloud_async(), - ); - let message = crate::domains::kv::KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "acme".to_string(), - "app".to_string(), - "users".to_string(), - ), - mode: crate::domains::kv::TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - // Act - let mapped = sink.apply_write_options(message); - - // Assert - match mapped { - crate::domains::kv::KvMessage::Begin { write_options, .. } => { - assert!(write_options.is_cloud_async()); - } - _ => panic!("expected KV begin message"), - } -} - -#[test] -fn should_derive_cloud_async_buffered_policy_given_strict_cloud_sync_builder() { - // Arrange - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model) - .with_sync_write_options(cntryl_midge::WriteOptions::cloud_strict()); - let message = crate::domains::kv::KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "acme".to_string(), - "app".to_string(), - "users".to_string(), - ), - mode: crate::domains::kv::TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - // Act - let mapped = sink.apply_write_options(message); - - // Assert - match mapped { - crate::domains::kv::KvMessage::Begin { write_options, .. } => { - assert!(write_options.is_cloud_async()); - } - _ => panic!("expected KV begin message"), - } -} - -#[test] -fn should_derive_cloud_async_buffered_policy_given_background_cloud_sync_builder() { - // Arrange - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model) - .with_sync_write_options(cntryl_midge::WriteOptions::cloud_async()); - let message = crate::domains::kv::KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "acme".to_string(), - "app".to_string(), - "users".to_string(), - ), - mode: crate::domains::kv::TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - // Act - let mapped = sink.apply_write_options(message); - - // Assert - match mapped { - crate::domains::kv::KvMessage::Begin { write_options, .. } => { - assert!(write_options.is_cloud_async()); - } - _ => panic!("expected KV begin message"), - } -} - -#[test] -fn should_release_resource_lock_given_session_cleanup() { - // Arrange - let family = RouteFamily::new(1); - let first_session_id = 7; - let second_session_id = 8; - let kv_route = "kv://acme/app/users"; - let kv_address = RouteAddress::new(family, Route::new(kv_route)); - let first_address = RouteAddress::new(family, Route::new("inbox://session/7")); - let second_address = RouteAddress::new(family, Route::new("inbox://session/8")); - let first_mailbox = Arc::new(Mailbox::new(8)); - let second_mailbox = Arc::new(Mailbox::new(8)); - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - router.register(first_address.clone(), first_mailbox.clone()); - router.register(second_address.clone(), second_mailbox.clone()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model); - - sink.deliver(Envelope::from_route( - first_address, - kv_address.clone(), - FrameContext::new( - first_session_id, - ChannelId::Sub, - MessageType::new(100), - encode_kv_begin(kv_route, 1, 0), - family, - ), - )) - .expect("begin first KV transaction"); - let first_begin_frame = receive_frame(&first_mailbox, "first begin ack envelope"); - let first_tx_id = decode_kv_begin_tx_id(&first_begin_frame.payload); - assert_eq!(first_begin_frame.payload[0], 0); - assert!(first_tx_id > 0); - assert_eq!(sink.active_transaction_count(), 1); - drain_mailbox(&first_mailbox); - - // Act - sink.deliver(Envelope::new( - RouteAddress::new(family, Route::new("kv://cleanup")), - crate::runtime::SessionCleanup { - session_id: first_session_id, - }, - )) - .expect("cleanup first KV session"); - wait_for_active_transaction_count(&sink, 0); - - sink.deliver(Envelope::from_route( - second_address, - kv_address, - FrameContext::new( - second_session_id, - ChannelId::Sub, - MessageType::new(100), - encode_kv_begin(kv_route, 1, 0), - family, - ), - )) - .expect("begin second KV transaction"); - - // Assert - let second_begin_frame = receive_frame(&second_mailbox, "second begin ack envelope"); - let second_tx_id = decode_kv_begin_tx_id(&second_begin_frame.payload); - assert_eq!(second_begin_frame.payload[0], 0); - assert!(second_tx_id > 0); - assert_eq!(sink.active_transaction_count(), 1); - assert_no_envelope(&first_mailbox); -} - -#[test] -fn should_reject_conflicting_read_write_begin_given_active_transaction_in_other_session() { - // Arrange - let family = RouteFamily::new(1); - let first_session_id = 7; - let second_session_id = 8; - let kv_route = "kv://acme/app/users"; - let kv_address = RouteAddress::new(family, Route::new(kv_route)); - let first_address = RouteAddress::new(family, Route::new("inbox://session/7")); - let second_address = RouteAddress::new(family, Route::new("inbox://session/8")); - let first_mailbox = Arc::new(Mailbox::new(8)); - let second_mailbox = Arc::new(Mailbox::new(8)); - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - router.register(first_address.clone(), first_mailbox.clone()); - router.register(second_address.clone(), second_mailbox.clone()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model); - - sink.deliver(Envelope::from_route( - first_address, - kv_address.clone(), - FrameContext::new( - first_session_id, - ChannelId::Sub, - MessageType::new(100), - encode_kv_begin(kv_route, 1, 0), - family, - ), - )) - .expect("begin first KV transaction"); - let _ = receive_envelope(&first_mailbox, "first begin ack envelope"); - - // Act - sink.deliver(Envelope::from_route( - second_address, - kv_address, - FrameContext::new( - second_session_id, - ChannelId::Sub, - MessageType::new(100), - encode_kv_begin(kv_route, 1, 0), - family, - ), - )) - .expect("begin second KV transaction"); - - // Assert - let second_begin_frame = receive_frame(&second_mailbox, "second begin response envelope"); - assert_eq!( - decode_error_code(&second_begin_frame.payload), - error_codes::kv::ERR_ISOLATION_CONFLICT - ); - assert_eq!(sink.active_transaction_count(), 1); -} - -#[test] -fn should_rebuild_kv_admin_transactions_from_actor_state() { - // Arrange - let family = RouteFamily::new(1); - let session_id = 7; - let kv_route = "kv://acme/app/users"; - let kv_address = RouteAddress::new(family, Route::new(kv_route)); - let source_address = RouteAddress::new(family, Route::new("inbox://session/7")); - let mailbox = Arc::new(Mailbox::new(8)); - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - router.register(source_address.clone(), mailbox.clone()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model.clone()); - - sink.deliver(Envelope::from_route( - source_address, - kv_address, - FrameContext::new( - session_id, - ChannelId::Sub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::BEGIN), - encode_kv_begin(kv_route, 1, 0), - family, - ), - )) - .expect("begin KV transaction"); - let _ = receive_envelope(&mailbox, "begin ack envelope"); - - // Act - sink.sync_admin_snapshot(); - let before_cleanup = admin_read_model.kv_transactions(None); - sink.cleanup_session(session_id); - let after_cleanup = admin_read_model.kv_transactions(None); - - // Assert - assert_eq!(before_cleanup.len(), 1); - assert_eq!(before_cleanup[0].route_family, 1); - assert_eq!(before_cleanup[0].realm, "acme"); - assert_eq!(before_cleanup[0].area, "app"); - assert_eq!(before_cleanup[0].resource, "users"); - assert!(after_cleanup.is_empty()); -} - -#[test] -fn should_route_kv_cleanup_through_managed_actor() { - // Arrange - let family = RouteFamily::new(1); - let session_id = 7; - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store.clone(), router, admin_read_model.clone()); - let mut actor = crate::domains::kv::KvActor::new(store); - let begin_response = actor.handle(crate::domains::kv::KvMessage::Begin { - scope: KvResourceScope::new( - family, - "acme".to_string(), - "app".to_string(), - "users".to_string(), - ), - mode: crate::domains::kv::TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }); - assert!(matches!( - begin_response, - crate::domains::kv::KvResponse::BeginOk { .. } - )); - sink.insert_actor_for_tests(session_id, actor); - sink.sync_admin_snapshot(); - assert_eq!(sink.active_transaction_count(), 1); - assert_eq!(admin_read_model.kv_transactions(None).len(), 1); - - // Act - sink.stop_actor_for_tests(); - sink.cleanup_session(session_id); - sink.sync_admin_snapshot(); - let after_cleanup = admin_read_model.kv_transactions(None); - - // Assert - assert!(!sink.is_actor_running()); - assert_eq!(after_cleanup.len(), 1); -} - -#[test] -fn should_route_kv_live_transaction_count_through_managed_actor() { - // Arrange - let family = RouteFamily::new(1); - let session_id = 7; - let kv_route = "kv://acme/app/users"; - let kv_address = RouteAddress::new(family, Route::new(kv_route)); - let source_address = RouteAddress::new(family, Route::new("inbox://session/7")); - let mailbox = Arc::new(Mailbox::new(8)); - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - router.register(source_address.clone(), mailbox.clone()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model); - sink.deliver(Envelope::from_route( - source_address, - kv_address, - FrameContext::new( - session_id, - ChannelId::Sub, - MessageType::new(100), - encode_kv_begin(kv_route, 1, 0), - family, - ), - )) - .expect("begin KV transaction"); - let _ = receive_envelope(&mailbox, "begin ack envelope"); - assert_eq!(sink.active_transaction_count(), 1); - - // Act - sink.stop_actor_for_tests(); - let active_transaction_count = sink.active_transaction_count(); - - // Assert - assert!(!sink.is_actor_running()); - assert_eq!(active_transaction_count, 0); -} - -#[test] -fn should_route_kv_admin_snapshot_sync_through_managed_actor() { - // Arrange - let family = RouteFamily::new(1); - let session_id = 7; - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store.clone(), router, admin_read_model.clone()); - let mut actor = crate::domains::kv::KvActor::new(store); - let begin_response = actor.handle(crate::domains::kv::KvMessage::Begin { - scope: KvResourceScope::new( - family, - "acme".to_string(), - "app".to_string(), - "users".to_string(), - ), - mode: crate::domains::kv::TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }); - assert!(matches!( - begin_response, - crate::domains::kv::KvResponse::BeginOk { .. } - )); - sink.insert_actor_for_tests(session_id, actor); - - // Act - sink.stop_actor_for_tests(); - sink.sync_admin_snapshot(); - let transactions = admin_read_model.kv_transactions(None); - - // Assert - assert!(!sink.is_actor_running()); - assert!(transactions.is_empty()); -} - -#[test] -fn should_route_kv_latency_snapshot_query_through_managed_actor() { - // Arrange - let family = RouteFamily::new(1); - let session_id = 7; - let kv_route = "kv://acme/app/users"; - let kv_address = RouteAddress::new(family, Route::new(kv_route)); - let source_address = RouteAddress::new(family, Route::new("inbox://session/7")); - let mailbox = Arc::new(Mailbox::new(8)); - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - router.register(source_address.clone(), mailbox.clone()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model); - sink.deliver(Envelope::from_route( - source_address.clone(), - kv_address.clone(), - FrameContext::new( - session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::BEGIN), - encode_kv_begin(kv_route, 1, 0), - family, - ), - )) - .expect("begin KV transaction"); - let begin_frame = receive_frame(&mailbox, "begin ack envelope"); - let tx_id = decode_kv_begin_tx_id(&begin_frame.payload); - sink.deliver(Envelope::from_route( - source_address.clone(), - kv_address.clone(), - FrameContext::new( - session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::PUT), - encode_kv_put(tx_id, kv_route, b"user:1", b"alice"), - family, - ), - )) - .expect("put KV value"); - let _ = receive_envelope(&mailbox, "put ack envelope"); - sink.deliver(Envelope::from_route( - source_address, - kv_address, - FrameContext::new( - session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::COMMIT), - encode_kv_commit(tx_id, kv_route), - family, - ), - )) - .expect("commit KV transaction"); - let _ = receive_envelope(&mailbox, "commit ack envelope"); - let resource_key = KvResourceLockKey::new(1, "acme", "app", "users"); - - // Act - sink.stop_actor_for_tests(); - let (reads, writes) = sink.latency_snapshots(&resource_key); - - // Assert - assert!(!sink.is_actor_running()); - assert!(reads.avg_ms.abs() < f64::EPSILON); - assert!(reads.p95_ms.abs() < f64::EPSILON); - assert!(writes.avg_ms.abs() < f64::EPSILON); - assert!(writes.p95_ms.abs() < f64::EPSILON); -} - -#[test] -fn should_route_kv_sync_write_options_mapping_through_managed_actor() { - // Arrange - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model) - .with_sync_write_options(cntryl_midge::WriteOptions::cloud_strict()); - let message = crate::domains::kv::KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "acme".to_string(), - "app".to_string(), - "users".to_string(), - ), - mode: crate::domains::kv::TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::sync(), - }; - - // Act - sink.stop_actor_for_tests(); - let mapped = sink.apply_write_options(message); - - // Assert - assert!(!sink.is_actor_running()); - match mapped { - crate::domains::kv::KvMessage::Begin { write_options, .. } => { - assert!(!write_options.is_cloud_strict()); - } - _ => panic!("expected KV begin message"), - } -} - -#[test] -fn should_notify_kv_subscriber_given_committed_put() { - // Arrange - let family = RouteFamily::new(1); - let watch_session_id = 7; - let writer_session_id = 8; - let kv_route = "kv://acme/app/users"; - let kv_address = RouteAddress::new(family, Route::new(kv_route)); - let watcher_address = RouteAddress::new(family, Route::new("inbox://session/7")); - let writer_address = RouteAddress::new(family, Route::new("inbox://session/8")); - let watcher_mailbox = Arc::new(Mailbox::new(16)); - let writer_mailbox = Arc::new(Mailbox::new(16)); - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - router.register(watcher_address.clone(), watcher_mailbox.clone()); - router.register(writer_address.clone(), writer_mailbox.clone()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model); - - // Act - sink.deliver(Envelope::from_route( - watcher_address, - kv_address.clone(), - FrameContext::new( - watch_session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::SUBSCRIBE), - encode_kv_subscribe(kv_route), - family, - ), - )) - .expect("subscribe to KV route"); - let subscribe_frame = receive_frame(&watcher_mailbox, "subscribe ack envelope"); - let subscription_id = decode_kv_subscription_id(&subscribe_frame.payload); - - sink.deliver(Envelope::from_route( - writer_address.clone(), - kv_address.clone(), - FrameContext::new( - writer_session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::BEGIN), - encode_kv_begin(kv_route, 1, 0), - family, - ), - )) - .expect("begin KV transaction"); - let begin_frame = receive_frame(&writer_mailbox, "begin ack envelope"); - let tx_id = decode_kv_begin_tx_id(&begin_frame.payload); - - sink.deliver(Envelope::from_route( - writer_address.clone(), - kv_address.clone(), - FrameContext::new( - writer_session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::PUT), - encode_kv_put(tx_id, kv_route, b"user:1", b"alice"), - family, - ), - )) - .expect("put KV value"); - let _ = receive_envelope(&writer_mailbox, "put ack envelope"); - - sink.deliver(Envelope::from_route( - writer_address, - kv_address, - FrameContext::new( - writer_session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::COMMIT), - encode_kv_commit(tx_id, kv_route), - family, - ), - )) - .expect("commit KV transaction"); - let _ = receive_envelope(&writer_mailbox, "commit ack envelope"); - - // Assert - let notify_frame = receive_frame(&watcher_mailbox, "KV notify envelope"); - assert_eq!( - notify_frame.msg_type.as_u16(), - crate::dispatch::protocol::kv::msg_type::NOTIFY - ); - let (delivered_subscription_id, delivered_route, mutation_count) = - decode_kv_watch_delivery(¬ify_frame); - assert_eq!(delivered_subscription_id, subscription_id); - assert_eq!(delivered_route, kv_route); - assert_eq!(mutation_count, 1); -} - -#[test] -fn should_not_notify_kv_subscriber_given_empty_commit() { - // Arrange - let family = RouteFamily::new(1); - let watch_session_id = 7; - let writer_session_id = 8; - let kv_route = "kv://acme/app/users"; - let kv_address = RouteAddress::new(family, Route::new(kv_route)); - let watcher_address = RouteAddress::new(family, Route::new("inbox://session/7")); - let writer_address = RouteAddress::new(family, Route::new("inbox://session/8")); - let watcher_mailbox = Arc::new(Mailbox::new(16)); - let writer_mailbox = Arc::new(Mailbox::new(16)); - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - router.register(watcher_address.clone(), watcher_mailbox.clone()); - router.register(writer_address.clone(), writer_mailbox.clone()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model); - - sink.deliver(Envelope::from_route( - watcher_address, - kv_address.clone(), - FrameContext::new( - watch_session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::SUBSCRIBE), - encode_kv_subscribe(kv_route), - family, - ), - )) - .expect("subscribe to KV route"); - let _ = receive_envelope(&watcher_mailbox, "subscribe ack envelope"); - - // Act - sink.deliver(Envelope::from_route( - writer_address.clone(), - kv_address.clone(), - FrameContext::new( - writer_session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::BEGIN), - encode_kv_begin(kv_route, 1, 0), - family, - ), - )) - .expect("begin KV transaction"); - let begin_frame = receive_frame(&writer_mailbox, "begin ack envelope"); - let tx_id = decode_kv_begin_tx_id(&begin_frame.payload); - - sink.deliver(Envelope::from_route( - writer_address, - kv_address, - FrameContext::new( - writer_session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::COMMIT), - encode_kv_commit(tx_id, kv_route), - family, - ), - )) - .expect("commit empty KV transaction"); - let _ = receive_envelope(&writer_mailbox, "commit ack envelope"); - - // Assert - assert_no_envelope(&watcher_mailbox); -} - -#[test] -fn should_remove_kv_subscription_given_unsubscribe() { - // Arrange - let family = RouteFamily::new(1); - let watch_session_id = 7; - let writer_session_id = 8; - let kv_route = "kv://acme/app/users"; - let kv_address = RouteAddress::new(family, Route::new(kv_route)); - let watcher_address = RouteAddress::new(family, Route::new("inbox://session/7")); - let writer_address = RouteAddress::new(family, Route::new("inbox://session/8")); - let watcher_mailbox = Arc::new(Mailbox::new(16)); - let writer_mailbox = Arc::new(Mailbox::new(16)); - let store = crate::testkit::create_test_engine_with_cfs(vec![1]); - let router = Arc::new(Router::new()); - router.register(watcher_address.clone(), watcher_mailbox.clone()); - router.register(writer_address.clone(), writer_mailbox.clone()); - let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); - let sink = KvDomainSink::new(store, router, admin_read_model); - - sink.deliver(Envelope::from_route( - watcher_address.clone(), - kv_address.clone(), - FrameContext::new( - watch_session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::SUBSCRIBE), - encode_kv_subscribe(kv_route), - family, - ), - )) - .expect("subscribe to KV route"); - let _ = receive_envelope(&watcher_mailbox, "subscribe ack envelope"); - - // Act - sink.deliver(Envelope::from_route( - watcher_address, - kv_address.clone(), - FrameContext::new( - watch_session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::UNSUBSCRIBE), - encode_kv_unsubscribe(kv_route), - family, - ), - )) - .expect("unsubscribe from KV route"); - let _ = receive_envelope(&watcher_mailbox, "unsubscribe ack envelope"); - - sink.deliver(Envelope::from_route( - writer_address.clone(), - kv_address.clone(), - FrameContext::new( - writer_session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::BEGIN), - encode_kv_begin(kv_route, 1, 0), - family, - ), - )) - .expect("begin KV transaction"); - let begin_frame = receive_frame(&writer_mailbox, "begin ack envelope"); - let tx_id = decode_kv_begin_tx_id(&begin_frame.payload); - - sink.deliver(Envelope::from_route( - writer_address.clone(), - kv_address.clone(), - FrameContext::new( - writer_session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::PUT), - encode_kv_put(tx_id, kv_route, b"user:1", b"alice"), - family, - ), - )) - .expect("put KV value"); - let _ = receive_envelope(&writer_mailbox, "put ack envelope"); - - sink.deliver(Envelope::from_route( - writer_address, - kv_address, - FrameContext::new( - writer_session_id, - ChannelId::Pub, - MessageType::new(crate::dispatch::protocol::kv::msg_type::COMMIT), - encode_kv_commit(tx_id, kv_route), - family, - ), - )) - .expect("commit KV transaction"); - let _ = receive_envelope(&writer_mailbox, "commit ack envelope"); - - // Assert - assert_no_envelope(&watcher_mailbox); - assert!(sink.watch_actors_are_empty_for_tests()); -} diff --git a/src/domains/kv/sink/tests/configuration.rs b/src/domains/kv/sink/tests/configuration.rs new file mode 100644 index 00000000..8ad2f69a --- /dev/null +++ b/src/domains/kv/sink/tests/configuration.rs @@ -0,0 +1,235 @@ +use super::*; + +#[test] +fn should_create_kv_domain_sink() { + // Arrange + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + + // Act + let sink = KvDomainSink::new(store, router, admin_read_model); + + // Assert + assert!(sink.is_active_for_tests()); + assert!(sink.is_actor_running()); +} + +#[test] +fn should_route_kv_counters_to_configured_collector() { + // Arrange + const TEST_COUNTER: &str = "fitz_kv_test_scoped_counter_total"; + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let configured = crate::observability::metrics::MetricsCollector::new(); + let global = crate::observability::metrics(); + let global_before = global.counter_get(TEST_COUNTER); + let sink = KvDomainSink::new(store, router, admin_read_model).with_metrics(configured.clone()); + + // Act + sink.state.runtime().counter_inc(TEST_COUNTER); + + // Assert + assert_eq!(configured.counter_get(TEST_COUNTER), 1); + assert_eq!(global.counter_get(TEST_COUNTER), global_before); +} + +#[test] +fn should_record_kv_latency_samples_by_operation_kind() { + // Arrange + let family = RouteFamily::new(1); + let session_id = 7; + let kv_route = "kv://acme/app/users"; + let kv_address = RouteAddress::new(family, Route::new(kv_route)); + let source_address = RouteAddress::new(family, Route::new("inbox://session/7")); + let mailbox = Arc::new(Mailbox::new(16)); + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + router.register(source_address.clone(), mailbox.clone()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model); + + sink.deliver(Envelope::from_route( + source_address.clone(), + kv_address.clone(), + FrameContext::new( + session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::BEGIN), + encode_kv_begin(kv_route, 1, 0), + family, + ), + )) + .expect("begin KV transaction"); + let begin_frame = receive_frame(&mailbox, "begin ack envelope"); + let tx_id = decode_kv_begin_tx_id(&begin_frame.payload); + + sink.deliver(Envelope::from_route( + source_address.clone(), + kv_address.clone(), + FrameContext::new( + session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::PUT), + encode_kv_put(tx_id, kv_route, b"user:1", b"alice"), + family, + ), + )) + .expect("put KV value"); + let _ = receive_envelope(&mailbox, "put ack envelope"); + + // Act + sink.deliver(Envelope::from_route( + source_address, + kv_address, + FrameContext::new( + session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::COMMIT), + encode_kv_commit(tx_id, kv_route), + family, + ), + )) + .expect("commit KV transaction"); + let _ = receive_envelope(&mailbox, "commit ack envelope"); + let resource_key = KvResourceLockKey::new(1, "acme", "app", "users"); + let (reads_before, writes_before) = sink.latency_snapshots(&resource_key); + assert!(reads_before.avg_ms.abs() < f64::EPSILON); + assert!(writes_before.avg_ms > 0.0); + let value = sink + .admin_get_committed_value(family, "acme", "app", "users", b"user:1") + .expect("read committed KV value"); + + // Assert + assert_eq!(value.as_deref(), Some(&b"alice"[..])); + let (reads_after, writes_after) = sink.latency_snapshots(&resource_key); + assert!(reads_after.avg_ms > 0.0); + assert!(writes_after.avg_ms > 0.0); +} + +#[test] +fn should_map_sync_begin_to_cloud_strict_given_strict_cloud_sync_policy() { + // Arrange + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model) + .with_sync_write_options(cntryl_midge::WriteOptions::cloud_strict()); + let message = crate::domains::kv::KvMessage::Begin { + scope: KvResourceScope::new( + RouteFamily::new(1), + "acme".to_string(), + "app".to_string(), + "users".to_string(), + ), + mode: crate::domains::kv::TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::sync(), + }; + + // Act + let mapped = sink.apply_write_options(message); + + // Assert + match mapped { + crate::domains::kv::KvMessage::Begin { write_options, .. } => { + assert!(write_options.is_cloud_strict()); + } + _ => panic!("expected KV begin message"), + } +} + +#[test] +fn should_map_buffered_begin_to_cloud_async_given_cloud_storage() { + // Arrange + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model).with_write_options( + cntryl_midge::WriteOptions::cloud_strict(), + cntryl_midge::WriteOptions::cloud_async(), + ); + let message = crate::domains::kv::KvMessage::Begin { + scope: KvResourceScope::new( + RouteFamily::new(1), + "acme".to_string(), + "app".to_string(), + "users".to_string(), + ), + mode: crate::domains::kv::TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }; + + // Act + let mapped = sink.apply_write_options(message); + + // Assert + match mapped { + crate::domains::kv::KvMessage::Begin { write_options, .. } => { + assert!(write_options.is_cloud_async()); + } + _ => panic!("expected KV begin message"), + } +} + +#[test] +fn should_derive_cloud_async_buffered_policy_given_strict_cloud_sync_builder() { + // Arrange + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model) + .with_sync_write_options(cntryl_midge::WriteOptions::cloud_strict()); + let message = crate::domains::kv::KvMessage::Begin { + scope: KvResourceScope::new( + RouteFamily::new(1), + "acme".to_string(), + "app".to_string(), + "users".to_string(), + ), + mode: crate::domains::kv::TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }; + + // Act + let mapped = sink.apply_write_options(message); + + // Assert + match mapped { + crate::domains::kv::KvMessage::Begin { write_options, .. } => { + assert!(write_options.is_cloud_async()); + } + _ => panic!("expected KV begin message"), + } +} + +#[test] +fn should_derive_cloud_async_buffered_policy_given_background_cloud_sync_builder() { + // Arrange + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model) + .with_sync_write_options(cntryl_midge::WriteOptions::cloud_async()); + let message = crate::domains::kv::KvMessage::Begin { + scope: KvResourceScope::new( + RouteFamily::new(1), + "acme".to_string(), + "app".to_string(), + "users".to_string(), + ), + mode: crate::domains::kv::TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }; + + // Act + let mapped = sink.apply_write_options(message); + + // Assert + match mapped { + crate::domains::kv::KvMessage::Begin { write_options, .. } => { + assert!(write_options.is_cloud_async()); + } + _ => panic!("expected KV begin message"), + } +} diff --git a/src/domains/kv/sink/tests/correctness.rs b/src/domains/kv/sink/tests/correctness.rs index 32dce062..83e22457 100644 --- a/src/domains/kv/sink/tests/correctness.rs +++ b/src/domains/kv/sink/tests/correctness.rs @@ -251,7 +251,7 @@ fn should_reject_invalid_admin_inventory_route_family_without_panicking() { let sink = new_correctness_sink(Arc::new(Router::new())); // Act - let result = sink.state.runtime().admin_inventory_for_family(u64::MAX); + let result = sink.admin_inventory_for_family_for_tests(u64::MAX); // Assert assert_eq!( @@ -388,5 +388,5 @@ fn should_reject_kv_subscription_before_allocating_family_state() { decode_error_code(&response.payload), error_codes::kv::ERR_INVALID_SUBSCRIPTION_PATTERN ); - assert!(sink.watch_actors_are_empty_for_tests()); + assert!(sink.watch_registries_are_empty_for_tests()); } diff --git a/src/domains/kv/sink/tests/lifecycle.rs b/src/domains/kv/sink/tests/lifecycle.rs new file mode 100644 index 00000000..fabc586b --- /dev/null +++ b/src/domains/kv/sink/tests/lifecycle.rs @@ -0,0 +1,430 @@ +use super::*; + +#[test] +fn should_reject_queued_begin_after_cleanup_without_recreating_session_state() { + // Arrange + let family = RouteFamily::new(1); + let session_id = 7; + let kv_route = "kv://acme/app/users"; + let kv_address = RouteAddress::new(family, Route::new(kv_route)); + let source_address = RouteAddress::new(family, Route::new("inbox://session/7")); + let mailbox = Arc::new(Mailbox::new(8)); + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + router.register(source_address.clone(), mailbox.clone()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model.clone()); + let previously_queued_begin = Envelope::from_route( + source_address, + kv_address, + FrameContext::new( + session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::BEGIN), + encode_kv_begin(kv_route, 1, 0), + family, + ), + ); + + // Act + sink.cleanup_session(session_id); + sink.deliver(previously_queued_begin) + .expect("deliver queued BEGIN after cleanup"); + let response = receive_frame(&mailbox, "queued BEGIN rejection"); + + // Assert + assert_eq!( + decode_error_code(&response.payload), + error_codes::kv::ERR_INVALID_ROUTE + ); + assert!(sink.actors_are_empty_for_tests()); + assert_eq!(sink.active_transaction_count(), 0); + assert!(sink.resource_locks_are_empty_for_tests()); + assert!(sink.watch_registries_are_empty_for_tests()); + assert!(admin_read_model.kv_transactions(None).is_empty()); +} + +#[test] +fn should_release_resource_lock_given_session_cleanup() { + // Arrange + let family = RouteFamily::new(1); + let first_session_id = 7; + let second_session_id = 8; + let kv_route = "kv://acme/app/users"; + let kv_address = RouteAddress::new(family, Route::new(kv_route)); + let first_address = RouteAddress::new(family, Route::new("inbox://session/7")); + let second_address = RouteAddress::new(family, Route::new("inbox://session/8")); + let first_mailbox = Arc::new(Mailbox::new(8)); + let second_mailbox = Arc::new(Mailbox::new(8)); + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + router.register(first_address.clone(), first_mailbox.clone()); + router.register(second_address.clone(), second_mailbox.clone()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model); + + sink.deliver(Envelope::from_route( + first_address, + kv_address.clone(), + FrameContext::new( + first_session_id, + ChannelId::Sub, + MessageType::new(100), + encode_kv_begin(kv_route, 1, 0), + family, + ), + )) + .expect("begin first KV transaction"); + let first_begin_frame = receive_frame(&first_mailbox, "first begin ack envelope"); + let first_tx_id = decode_kv_begin_tx_id(&first_begin_frame.payload); + assert_eq!(first_begin_frame.payload[0], 0); + assert!(first_tx_id > 0); + assert_eq!(sink.active_transaction_count(), 1); + drain_mailbox(&first_mailbox); + + // Act + sink.deliver(Envelope::new( + RouteAddress::new(family, Route::new("kv://cleanup")), + crate::runtime::SessionCleanup { + session_id: first_session_id, + }, + )) + .expect("cleanup first KV session"); + wait_for_active_transaction_count(&sink, 0); + + sink.deliver(Envelope::from_route( + second_address, + kv_address, + FrameContext::new( + second_session_id, + ChannelId::Sub, + MessageType::new(100), + encode_kv_begin(kv_route, 1, 0), + family, + ), + )) + .expect("begin second KV transaction"); + + // Assert + let second_begin_frame = receive_frame(&second_mailbox, "second begin ack envelope"); + let second_tx_id = decode_kv_begin_tx_id(&second_begin_frame.payload); + assert_eq!(second_begin_frame.payload[0], 0); + assert!(second_tx_id > 0); + assert_eq!(sink.active_transaction_count(), 1); + assert_no_envelope(&first_mailbox); +} + +#[test] +fn should_reject_conflicting_read_write_begin_given_active_transaction_in_other_session() { + // Arrange + let family = RouteFamily::new(1); + let first_session_id = 7; + let second_session_id = 8; + let kv_route = "kv://acme/app/users"; + let kv_address = RouteAddress::new(family, Route::new(kv_route)); + let first_address = RouteAddress::new(family, Route::new("inbox://session/7")); + let second_address = RouteAddress::new(family, Route::new("inbox://session/8")); + let first_mailbox = Arc::new(Mailbox::new(8)); + let second_mailbox = Arc::new(Mailbox::new(8)); + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + router.register(first_address.clone(), first_mailbox.clone()); + router.register(second_address.clone(), second_mailbox.clone()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model); + + sink.deliver(Envelope::from_route( + first_address, + kv_address.clone(), + FrameContext::new( + first_session_id, + ChannelId::Sub, + MessageType::new(100), + encode_kv_begin(kv_route, 1, 0), + family, + ), + )) + .expect("begin first KV transaction"); + let _ = receive_envelope(&first_mailbox, "first begin ack envelope"); + + // Act + sink.deliver(Envelope::from_route( + second_address, + kv_address, + FrameContext::new( + second_session_id, + ChannelId::Sub, + MessageType::new(100), + encode_kv_begin(kv_route, 1, 0), + family, + ), + )) + .expect("begin second KV transaction"); + + // Assert + let second_begin_frame = receive_frame(&second_mailbox, "second begin response envelope"); + assert_eq!( + decode_error_code(&second_begin_frame.payload), + error_codes::kv::ERR_ISOLATION_CONFLICT + ); + assert_eq!(sink.active_transaction_count(), 1); +} + +#[test] +fn should_rebuild_kv_admin_transactions_from_actor_state() { + // Arrange + let family = RouteFamily::new(1); + let session_id = 7; + let kv_route = "kv://acme/app/users"; + let kv_address = RouteAddress::new(family, Route::new(kv_route)); + let source_address = RouteAddress::new(family, Route::new("inbox://session/7")); + let mailbox = Arc::new(Mailbox::new(8)); + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + router.register(source_address.clone(), mailbox.clone()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model.clone()); + + sink.deliver(Envelope::from_route( + source_address, + kv_address, + FrameContext::new( + session_id, + ChannelId::Sub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::BEGIN), + encode_kv_begin(kv_route, 1, 0), + family, + ), + )) + .expect("begin KV transaction"); + let _ = receive_envelope(&mailbox, "begin ack envelope"); + + // Act + sink.sync_admin_snapshot(); + let before_cleanup = admin_read_model.kv_transactions(None); + sink.cleanup_session(session_id); + let after_cleanup = admin_read_model.kv_transactions(None); + + // Assert + assert_eq!(before_cleanup.len(), 1); + assert_eq!(before_cleanup[0].route_family, 1); + assert_eq!(before_cleanup[0].realm, "acme"); + assert_eq!(before_cleanup[0].area, "app"); + assert_eq!(before_cleanup[0].resource, "users"); + assert!(after_cleanup.is_empty()); +} + +#[test] +fn should_route_kv_cleanup_through_managed_actor() { + // Arrange + let family = RouteFamily::new(1); + let session_id = 7; + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store.clone(), router, admin_read_model.clone()); + let mut actor = crate::domains::kv::KvActor::new(store); + let begin_response = actor.handle(crate::domains::kv::KvMessage::Begin { + scope: KvResourceScope::new( + family, + "acme".to_string(), + "app".to_string(), + "users".to_string(), + ), + mode: crate::domains::kv::TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }); + assert!(matches!( + begin_response, + crate::domains::kv::KvResponse::BeginOk { .. } + )); + sink.insert_actor_for_tests(session_id, actor); + sink.sync_admin_snapshot(); + assert_eq!(sink.active_transaction_count(), 1); + assert_eq!(admin_read_model.kv_transactions(None).len(), 1); + + // Act + sink.stop_actor_for_tests(); + sink.cleanup_session(session_id); + sink.sync_admin_snapshot(); + let after_cleanup = admin_read_model.kv_transactions(None); + + // Assert + assert!(!sink.is_actor_running()); + assert_eq!(after_cleanup.len(), 1); +} + +#[test] +fn should_route_kv_live_transaction_count_through_managed_actor() { + // Arrange + let family = RouteFamily::new(1); + let session_id = 7; + let kv_route = "kv://acme/app/users"; + let kv_address = RouteAddress::new(family, Route::new(kv_route)); + let source_address = RouteAddress::new(family, Route::new("inbox://session/7")); + let mailbox = Arc::new(Mailbox::new(8)); + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + router.register(source_address.clone(), mailbox.clone()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model); + sink.deliver(Envelope::from_route( + source_address, + kv_address, + FrameContext::new( + session_id, + ChannelId::Sub, + MessageType::new(100), + encode_kv_begin(kv_route, 1, 0), + family, + ), + )) + .expect("begin KV transaction"); + let _ = receive_envelope(&mailbox, "begin ack envelope"); + assert_eq!(sink.active_transaction_count(), 1); + + // Act + sink.stop_actor_for_tests(); + let active_transaction_count = sink.active_transaction_count(); + + // Assert + assert!(!sink.is_actor_running()); + assert_eq!(active_transaction_count, 0); +} + +#[test] +fn should_route_kv_admin_snapshot_sync_through_managed_actor() { + // Arrange + let family = RouteFamily::new(1); + let session_id = 7; + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store.clone(), router, admin_read_model.clone()); + let mut actor = crate::domains::kv::KvActor::new(store); + let begin_response = actor.handle(crate::domains::kv::KvMessage::Begin { + scope: KvResourceScope::new( + family, + "acme".to_string(), + "app".to_string(), + "users".to_string(), + ), + mode: crate::domains::kv::TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::buffered(), + }); + assert!(matches!( + begin_response, + crate::domains::kv::KvResponse::BeginOk { .. } + )); + sink.insert_actor_for_tests(session_id, actor); + + // Act + sink.stop_actor_for_tests(); + sink.sync_admin_snapshot(); + let transactions = admin_read_model.kv_transactions(None); + + // Assert + assert!(!sink.is_actor_running()); + assert!(transactions.is_empty()); +} + +#[test] +fn should_route_kv_latency_snapshot_query_through_managed_actor() { + // Arrange + let family = RouteFamily::new(1); + let session_id = 7; + let kv_route = "kv://acme/app/users"; + let kv_address = RouteAddress::new(family, Route::new(kv_route)); + let source_address = RouteAddress::new(family, Route::new("inbox://session/7")); + let mailbox = Arc::new(Mailbox::new(8)); + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + router.register(source_address.clone(), mailbox.clone()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model); + sink.deliver(Envelope::from_route( + source_address.clone(), + kv_address.clone(), + FrameContext::new( + session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::BEGIN), + encode_kv_begin(kv_route, 1, 0), + family, + ), + )) + .expect("begin KV transaction"); + let begin_frame = receive_frame(&mailbox, "begin ack envelope"); + let tx_id = decode_kv_begin_tx_id(&begin_frame.payload); + sink.deliver(Envelope::from_route( + source_address.clone(), + kv_address.clone(), + FrameContext::new( + session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::PUT), + encode_kv_put(tx_id, kv_route, b"user:1", b"alice"), + family, + ), + )) + .expect("put KV value"); + let _ = receive_envelope(&mailbox, "put ack envelope"); + sink.deliver(Envelope::from_route( + source_address, + kv_address, + FrameContext::new( + session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::COMMIT), + encode_kv_commit(tx_id, kv_route), + family, + ), + )) + .expect("commit KV transaction"); + let _ = receive_envelope(&mailbox, "commit ack envelope"); + let resource_key = KvResourceLockKey::new(1, "acme", "app", "users"); + + // Act + sink.stop_actor_for_tests(); + let (reads, writes) = sink.latency_snapshots(&resource_key); + + // Assert + assert!(!sink.is_actor_running()); + assert!(reads.avg_ms.abs() < f64::EPSILON); + assert!(reads.p95_ms.abs() < f64::EPSILON); + assert!(writes.avg_ms.abs() < f64::EPSILON); + assert!(writes.p95_ms.abs() < f64::EPSILON); +} + +#[test] +fn should_route_kv_sync_write_options_mapping_through_managed_actor() { + // Arrange + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model) + .with_sync_write_options(cntryl_midge::WriteOptions::cloud_strict()); + let message = crate::domains::kv::KvMessage::Begin { + scope: KvResourceScope::new( + RouteFamily::new(1), + "acme".to_string(), + "app".to_string(), + "users".to_string(), + ), + mode: crate::domains::kv::TxMode::ReadWrite, + write_options: cntryl_midge::WriteOptions::sync(), + }; + + // Act + sink.stop_actor_for_tests(); + let mapped = sink.apply_write_options(message); + + // Assert + assert!(!sink.is_actor_running()); + match mapped { + crate::domains::kv::KvMessage::Begin { write_options, .. } => { + assert!(!write_options.is_cloud_strict()); + } + _ => panic!("expected KV begin message"), + } +} diff --git a/src/domains/kv/sink/tests/subscriptions.rs b/src/domains/kv/sink/tests/subscriptions.rs new file mode 100644 index 00000000..f2d7f878 --- /dev/null +++ b/src/domains/kv/sink/tests/subscriptions.rs @@ -0,0 +1,255 @@ +use super::*; + +#[test] +fn should_notify_kv_subscriber_given_committed_put() { + // Arrange + let family = RouteFamily::new(1); + let watch_session_id = 7; + let writer_session_id = 8; + let kv_route = "kv://acme/app/users"; + let kv_address = RouteAddress::new(family, Route::new(kv_route)); + let watcher_address = RouteAddress::new(family, Route::new("inbox://session/7")); + let writer_address = RouteAddress::new(family, Route::new("inbox://session/8")); + let watcher_mailbox = Arc::new(Mailbox::new(16)); + let writer_mailbox = Arc::new(Mailbox::new(16)); + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + router.register(watcher_address.clone(), watcher_mailbox.clone()); + router.register(writer_address.clone(), writer_mailbox.clone()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model); + + // Act + sink.deliver(Envelope::from_route( + watcher_address, + kv_address.clone(), + FrameContext::new( + watch_session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::SUBSCRIBE), + encode_kv_subscribe(kv_route), + family, + ), + )) + .expect("subscribe to KV route"); + let subscribe_frame = receive_frame(&watcher_mailbox, "subscribe ack envelope"); + let subscription_id = decode_kv_subscription_id(&subscribe_frame.payload); + + sink.deliver(Envelope::from_route( + writer_address.clone(), + kv_address.clone(), + FrameContext::new( + writer_session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::BEGIN), + encode_kv_begin(kv_route, 1, 0), + family, + ), + )) + .expect("begin KV transaction"); + let begin_frame = receive_frame(&writer_mailbox, "begin ack envelope"); + let tx_id = decode_kv_begin_tx_id(&begin_frame.payload); + + sink.deliver(Envelope::from_route( + writer_address.clone(), + kv_address.clone(), + FrameContext::new( + writer_session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::PUT), + encode_kv_put(tx_id, kv_route, b"user:1", b"alice"), + family, + ), + )) + .expect("put KV value"); + let _ = receive_envelope(&writer_mailbox, "put ack envelope"); + + sink.deliver(Envelope::from_route( + writer_address, + kv_address, + FrameContext::new( + writer_session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::COMMIT), + encode_kv_commit(tx_id, kv_route), + family, + ), + )) + .expect("commit KV transaction"); + let _ = receive_envelope(&writer_mailbox, "commit ack envelope"); + + // Assert + let notify_frame = receive_frame(&watcher_mailbox, "KV notify envelope"); + assert_eq!( + notify_frame.msg_type.as_u16(), + crate::dispatch::protocol::kv::msg_type::NOTIFY + ); + let (delivered_subscription_id, delivered_route, mutation_count) = + decode_kv_watch_delivery(¬ify_frame); + assert_eq!(delivered_subscription_id, subscription_id); + assert_eq!(delivered_route, kv_route); + assert_eq!(mutation_count, 1); +} + +#[test] +fn should_not_notify_kv_subscriber_given_empty_commit() { + // Arrange + let family = RouteFamily::new(1); + let watch_session_id = 7; + let writer_session_id = 8; + let kv_route = "kv://acme/app/users"; + let kv_address = RouteAddress::new(family, Route::new(kv_route)); + let watcher_address = RouteAddress::new(family, Route::new("inbox://session/7")); + let writer_address = RouteAddress::new(family, Route::new("inbox://session/8")); + let watcher_mailbox = Arc::new(Mailbox::new(16)); + let writer_mailbox = Arc::new(Mailbox::new(16)); + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + router.register(watcher_address.clone(), watcher_mailbox.clone()); + router.register(writer_address.clone(), writer_mailbox.clone()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model); + + sink.deliver(Envelope::from_route( + watcher_address, + kv_address.clone(), + FrameContext::new( + watch_session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::SUBSCRIBE), + encode_kv_subscribe(kv_route), + family, + ), + )) + .expect("subscribe to KV route"); + let _ = receive_envelope(&watcher_mailbox, "subscribe ack envelope"); + + // Act + sink.deliver(Envelope::from_route( + writer_address.clone(), + kv_address.clone(), + FrameContext::new( + writer_session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::BEGIN), + encode_kv_begin(kv_route, 1, 0), + family, + ), + )) + .expect("begin KV transaction"); + let begin_frame = receive_frame(&writer_mailbox, "begin ack envelope"); + let tx_id = decode_kv_begin_tx_id(&begin_frame.payload); + + sink.deliver(Envelope::from_route( + writer_address, + kv_address, + FrameContext::new( + writer_session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::COMMIT), + encode_kv_commit(tx_id, kv_route), + family, + ), + )) + .expect("commit empty KV transaction"); + let _ = receive_envelope(&writer_mailbox, "commit ack envelope"); + + // Assert + assert_no_envelope(&watcher_mailbox); +} + +#[test] +fn should_remove_kv_subscription_given_unsubscribe() { + // Arrange + let family = RouteFamily::new(1); + let watch_session_id = 7; + let writer_session_id = 8; + let kv_route = "kv://acme/app/users"; + let kv_address = RouteAddress::new(family, Route::new(kv_route)); + let watcher_address = RouteAddress::new(family, Route::new("inbox://session/7")); + let writer_address = RouteAddress::new(family, Route::new("inbox://session/8")); + let watcher_mailbox = Arc::new(Mailbox::new(16)); + let writer_mailbox = Arc::new(Mailbox::new(16)); + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + router.register(watcher_address.clone(), watcher_mailbox.clone()); + router.register(writer_address.clone(), writer_mailbox.clone()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = KvDomainSink::new(store, router, admin_read_model); + + sink.deliver(Envelope::from_route( + watcher_address.clone(), + kv_address.clone(), + FrameContext::new( + watch_session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::SUBSCRIBE), + encode_kv_subscribe(kv_route), + family, + ), + )) + .expect("subscribe to KV route"); + let _ = receive_envelope(&watcher_mailbox, "subscribe ack envelope"); + + // Act + sink.deliver(Envelope::from_route( + watcher_address, + kv_address.clone(), + FrameContext::new( + watch_session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::UNSUBSCRIBE), + encode_kv_unsubscribe(kv_route), + family, + ), + )) + .expect("unsubscribe from KV route"); + let _ = receive_envelope(&watcher_mailbox, "unsubscribe ack envelope"); + + sink.deliver(Envelope::from_route( + writer_address.clone(), + kv_address.clone(), + FrameContext::new( + writer_session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::BEGIN), + encode_kv_begin(kv_route, 1, 0), + family, + ), + )) + .expect("begin KV transaction"); + let begin_frame = receive_frame(&writer_mailbox, "begin ack envelope"); + let tx_id = decode_kv_begin_tx_id(&begin_frame.payload); + + sink.deliver(Envelope::from_route( + writer_address.clone(), + kv_address.clone(), + FrameContext::new( + writer_session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::PUT), + encode_kv_put(tx_id, kv_route, b"user:1", b"alice"), + family, + ), + )) + .expect("put KV value"); + let _ = receive_envelope(&writer_mailbox, "put ack envelope"); + + sink.deliver(Envelope::from_route( + writer_address, + kv_address, + FrameContext::new( + writer_session_id, + ChannelId::Pub, + MessageType::new(crate::dispatch::protocol::kv::msg_type::COMMIT), + encode_kv_commit(tx_id, kv_route), + family, + ), + )) + .expect("commit KV transaction"); + let _ = receive_envelope(&writer_mailbox, "commit ack envelope"); + + // Assert + assert_no_envelope(&watcher_mailbox); + assert!(sink.watch_registries_are_empty_for_tests()); +} diff --git a/src/domains/kv/sink/transactions.rs b/src/domains/kv/sink/transactions.rs new file mode 100644 index 00000000..2f8684ce --- /dev/null +++ b/src/domains/kv/sink/transactions.rs @@ -0,0 +1,288 @@ +//! BEGIN, COMMIT, ROLLBACK, and transaction outcome coordination. + +use super::locks::{KvResourceLockKey, KvResourceLockOwner}; +use super::state::{ + KvAdminTransactionUpdate, KvCommitNotification, KvDomainRuntime, KvOperationOutcome, +}; +use crate::domains::kv::{KvError, KvResponse}; +use crate::runtime::{DeliveryError, Envelope}; + +impl KvDomainRuntime<'_> { + pub(super) fn handle_actor_operation_frame( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + request_started: std::time::Instant, + operation_started: std::time::Instant, + kv_message: crate::domains::kv::KvMessage, + ) -> Result<(), DeliveryError> { + use crate::domains::kv::{KvMessage, TxMode}; + if Self::kv_message_family(&kv_message) != meta.route_family { + let response = Self::error_response("route family mismatch"); + self.route_kv_response(envelope, meta, &response, request_started)?; + return Ok(()); + } + if self.is_cleaned_up_session(meta.session_id) { + let response = Self::error_response("session already closed"); + self.route_kv_response(envelope, meta, &response, request_started)?; + return Ok(()); + } + + let kv_message = self.apply_write_options(kv_message); + let session_id = meta.session_id; + let read_tx_id = match &kv_message { + KvMessage::Get { tx_id, .. } | KvMessage::Scan { tx_id, .. } => Some(*tx_id), + _ => None, + }; + let is_commit = matches!(&kv_message, KvMessage::Commit { .. }); + + if matches!( + &kv_message, + KvMessage::Begin { + mode: TxMode::ReadWrite, + .. + } + ) { + if let KvMessage::Begin { scope, .. } = &kv_message { + self.expire_resource_lock_if_idle(&KvResourceLockKey::new( + scope.route_family.as_u64(), + &scope.realm, + &scope.area, + &scope.resource, + )); + } + } else { + self.expire_idle_transactions_for_session(session_id); + } + + tracing::trace!( + domain = "kv", + session_id = session_id, + msg_type = meta.message_type, + "KV deliver: getting or creating actor for session" + ); + + self.touch_resource_lock(session_id, &kv_message); + let KvOperationOutcome { + response, + admin_update, + commit_notification, + } = self.dispatch_actor_operation(session_id, meta, kv_message); + if matches!( + &response, + KvResponse::Error { + error: KvError::InvalidTxId, + .. + } + ) { + self.counter_inc("fitz_kv_invalid_transaction_rejects_total"); + } + match (&response, read_tx_id, is_commit) { + (KvResponse::GetResult { .. } | KvResponse::ScanResult { .. }, Some(tx_id), _) => { + if let Some(resource_key) = self.resource_key_for_tx(session_id, tx_id) { + self.record_read_latency(&resource_key, operation_started); + } + } + (KvResponse::CommitOk, _, true) => { + if let Some(notification) = commit_notification.as_ref() { + self.record_write_latency(¬ification.resource_key, operation_started); + } + } + _ => {} + } + self.apply_admin_transaction_update(admin_update); + if let Some(notification) = commit_notification { + self.route_kv_notification(¬ification.resource_key, notification.mutation_count); + } + + tracing::debug!( + domain = "kv", + session = meta.session_id, + response = ?std::mem::discriminant(&response), + "KV actor returned response" + ); + + self.route_kv_response(envelope, meta, &response, request_started) + } + + pub(super) fn handle_begin_read_write( + &self, + session_id: u64, + lock_key: &KvResourceLockKey, + kv_message: crate::domains::kv::KvMessage, + ) -> KvOperationOutcome { + let held_by_same_session = self.session_holds_resource_lock(session_id, lock_key); + if self + .conflicting_session_for_resource(session_id, lock_key) + .is_some() + { + return KvOperationOutcome::new( + KvResponse::Error { + error: KvError::Conflict("resource locked by another session".to_string()), + }, + KvAdminTransactionUpdate::None, + None, + ); + } + if held_by_same_session { + return KvOperationOutcome::new( + KvResponse::Error { + error: KvError::Conflict( + "resource already has a read-write transaction for this session" + .to_string(), + ), + }, + KvAdminTransactionUpdate::None, + None, + ); + } + + let log_context = "BEGIN (ReadWrite, acquiring lock)"; + let actor = self.actor_for_session(session_id, "begin"); + let mut actor = actor.lock(); + tracing::trace!( + domain = "kv", + session_id = session_id, + "Calling actor.handle() for {log_context}" + ); + let response = actor.handle(kv_message); + if let KvResponse::BeginOk { tx_id } = response { + self.core.resource_locks.lock().insert( + lock_key.clone(), + KvResourceLockOwner { + session_id, + tx_id, + last_activity: std::time::Instant::now(), + }, + ); + tracing::trace!( + domain = "kv", + session_id = session_id, + tx_id = tx_id, + "BEGIN succeeded with actor-owned transaction scope" + ); + let transaction = crate::control::admin::KvTransaction::snapshot( + lock_key.family_id, + tx_id, + session_id, + &lock_key.realm, + &lock_key.area, + &lock_key.resource, + &chrono::Utc::now().to_rfc3339(), + ); + KvOperationOutcome::new( + response, + KvAdminTransactionUpdate::Upsert(transaction), + None, + ) + } else { + KvOperationOutcome::new(response, KvAdminTransactionUpdate::None, None) + } + } + + pub(super) fn handle_commit_frame( + &self, + session_id: u64, + route_family: crate::runtime::routing::RouteFamily, + tx_id: u64, + kv_message: crate::domains::kv::KvMessage, + ) -> KvOperationOutcome { + let actor = self.actor_for_session(session_id, "commit"); + let mut actor = actor.lock(); + tracing::trace!( + domain = "kv", + session_id = session_id, + tx_id = tx_id, + "Calling actor.handle() for COMMIT" + ); + let mutation_count = actor.mutation_count_for_tx(tx_id).unwrap_or(0); + let lock_key = actor + .resource_scope_for_tx(tx_id) + .map(|scope| KvResourceLockKey::from_scope(&scope)); + if lock_key + .as_ref() + .is_some_and(|key| key.family_id != route_family.as_u64()) + { + return KvOperationOutcome::new( + KvResponse::Error { + error: KvError::InvalidRequest("route family mismatch".to_string()), + }, + KvAdminTransactionUpdate::None, + None, + ); + } + let had_transaction = lock_key.is_some(); + let response = actor.handle(kv_message); + let admin_update = if had_transaction && actor.resource_scope_for_tx(tx_id).is_none() { + if let Some(lock_key) = &lock_key { + self.core.resource_locks.lock().remove(lock_key); + } + KvAdminTransactionUpdate::Remove { session_id, tx_id } + } else { + KvAdminTransactionUpdate::None + }; + if let KvResponse::CommitOk = response { + if let Some(lock_key) = lock_key { + let notify = (mutation_count > 0).then_some(KvCommitNotification { + resource_key: lock_key, + mutation_count, + }); + KvOperationOutcome::new(response, admin_update, notify) + } else { + KvOperationOutcome::new(response, admin_update, None) + } + } else { + self.counter_inc("fitz_kv_commits_failed_total"); + KvOperationOutcome::new(response, admin_update, None) + } + } + + pub(super) fn handle_rollback_frame( + &self, + session_id: u64, + route_family: crate::runtime::routing::RouteFamily, + tx_id: u64, + kv_message: crate::domains::kv::KvMessage, + ) -> KvOperationOutcome { + let actor = self.actor_for_session(session_id, "rollback"); + let mut actor = actor.lock(); + tracing::trace!( + domain = "kv", + session_id = session_id, + tx_id = tx_id, + "Calling actor.handle() for ROLLBACK" + ); + let resource_scope = actor.resource_scope_for_tx(tx_id); + if resource_scope + .as_ref() + .is_some_and(|scope| scope.route_family != route_family) + { + return KvOperationOutcome::new( + KvResponse::Error { + error: KvError::InvalidRequest("route family mismatch".to_string()), + }, + KvAdminTransactionUpdate::None, + None, + ); + } + let response = actor.handle(kv_message); + let admin_update = + if resource_scope.is_some() && actor.resource_scope_for_tx(tx_id).is_none() { + if let Some(scope) = &resource_scope { + self.core + .resource_locks + .lock() + .remove(&KvResourceLockKey::from_scope(scope)); + } + KvAdminTransactionUpdate::Remove { session_id, tx_id } + } else { + KvAdminTransactionUpdate::None + }; + if let KvResponse::RollbackOk = response { + self.counter_inc("fitz_kv_rollbacks_total"); + KvOperationOutcome::new(response, admin_update, None) + } else { + KvOperationOutcome::new(response, admin_update, None) + } + } +} diff --git a/src/domains/kv/sink/write_policy.rs b/src/domains/kv/sink/write_policy.rs new file mode 100644 index 00000000..cbc1c879 --- /dev/null +++ b/src/domains/kv/sink/write_policy.rs @@ -0,0 +1,35 @@ +//! BEGIN write-option rewriting from broker configuration. +//! +//! Production delivery and the test-only `ApplyWriteOptions` mailbox probe +//! both call this single policy function. + +use super::state::KvDomainRuntime; + +impl KvDomainRuntime<'_> { + pub(super) fn apply_write_options( + &self, + message: crate::domains::kv::KvMessage, + ) -> crate::domains::kv::KvMessage { + match message { + crate::domains::kv::KvMessage::Begin { + scope, + mode, + write_options, + } if write_options.is_sync() + || write_options == cntryl_midge::WriteOptions::buffered() => + { + let write_options = if write_options.is_sync() { + self.core.sync_write_options + } else { + self.core.buffered_write_options + }; + crate::domains::kv::KvMessage::Begin { + scope, + mode, + write_options, + } + } + message => message, + } + } +} diff --git a/src/domains/kv/tests/admin_projection.rs b/src/domains/kv/tests/admin_projection.rs new file mode 100644 index 00000000..0d1a6d3a --- /dev/null +++ b/src/domains/kv/tests/admin_projection.rs @@ -0,0 +1,42 @@ +use super::*; + +#[test] +fn should_refresh_projection_when_marked_dirty() { + // Arrange + let read_model = AdminReadModel::new(); + let projection = KvAdminProjection::new(read_model.clone()); + projection.mark_dirty(); + + // Act + projection.refresh_if_dirty(|| { + vec![KvTransaction::snapshot( + 1, + 41, + 7, + "acme", + "app", + "users", + "2026-07-01T00:00:00Z", + )] + }); + + // Assert + assert_eq!(read_model.kv_transactions(None).len(), 1); +} + +#[test] +fn should_record_projection_latency_by_operation_kind() { + // Arrange + let read_model = AdminReadModel::new(); + let projection = KvAdminProjection::new(read_model); + let key = KvResourceLockKey::new(1, "acme", "app", "users"); + + // Act + projection.record_write_latency(&key, 5.0); + projection.record_read_latency(&key, 3.0); + let (reads, writes) = projection.latency_snapshots(&key); + + // Assert + assert!((reads.avg_ms - 3.0).abs() < f64::EPSILON); + assert!((writes.avg_ms - 5.0).abs() < f64::EPSILON); +} diff --git a/src/domains/kv/tests/protocol.rs b/src/domains/kv/tests/protocol.rs new file mode 100644 index 00000000..b13f467c --- /dev/null +++ b/src/domains/kv/tests/protocol.rs @@ -0,0 +1,17 @@ +use super::*; + +#[test] +fn should_expose_scope_for_every_kv_message_variant() { + // Arrange + let scope = KvResourceScope::new(RouteFamily::new(7), "realm", "area", "resource"); + let message = KvMessage::Rollback { + tx_id: 1, + scope: scope.clone(), + }; + + // Act + let actual = message.scope(); + + // Assert + assert_eq!(actual, &scope); +} diff --git a/src/domains/kv/tests/scan_wire_budget.rs b/src/domains/kv/tests/scan_wire_budget.rs new file mode 100644 index 00000000..dc494704 --- /dev/null +++ b/src/domains/kv/tests/scan_wire_budget.rs @@ -0,0 +1,78 @@ +use super::{kv_scan_item_wire_bytes, kv_scan_response_byte_ceiling}; +use crate::domains::kv::{KvPair, KvResponse}; +use bytes::Bytes; + +fn encoded_len(items: Vec) -> usize { + crate::dispatch::protocol::kv::encode_response(&KvResponse::ScanResult { + items, + has_more: false, + }) + .len() +} + +#[test] +fn should_match_the_codec_exactly_for_a_single_pair() { + // Arrange + // Budgeting more than the codec writes rejects wire-valid responses at + // the boundary; budgeting less emits unframable ones. Both are bugs, so + // the arithmetic is pinned to the encoder. + let key = Bytes::from(vec![b'k'; 300]); + let value = Bytes::from(vec![b'v'; 1_024]); + + // Act + let budgeted = + kv_scan_item_wire_bytes(key.len(), value.len()) + super::KV_SCAN_ENVELOPE_OVERHEAD_BYTES; + let actual = encoded_len(vec![KvPair { key, value }]); + + // Assert + assert_eq!(budgeted, actual, "budget must equal the encoded length"); +} + +#[test] +fn should_admit_the_largest_wire_valid_single_pair() { + // Arrange + // The exact case an over-generous budget rejected. + let key = Bytes::from(vec![b'k'; 300]); + let value = Bytes::from(vec![b'v'; 65_200]); + let pair = KvPair { + key: key.clone(), + value: value.clone(), + }; + + // Act + let cost = kv_scan_item_wire_bytes(key.len(), value.len()); + let actual = encoded_len(vec![pair]); + + // Assert + assert!( + u16::try_from(actual).is_ok(), + "this response is wire-valid at {actual} bytes" + ); + assert!( + cost <= kv_scan_response_byte_ceiling(), + "a wire-valid pair must not be rejected: {cost} charged against {}", + kv_scan_response_byte_ceiling() + ); +} + +#[test] +fn should_match_the_codec_exactly_across_many_pairs() { + // Arrange + let items = (0..50) + .map(|index| KvPair { + key: Bytes::from(format!("key-{index:03}")), + value: Bytes::from(vec![b'v'; 100 + index]), + }) + .collect::>(); + + // Act + let budgeted = items + .iter() + .map(|item| kv_scan_item_wire_bytes(item.key.len(), item.value.len())) + .sum::() + + super::KV_SCAN_ENVELOPE_OVERHEAD_BYTES; + let actual = encoded_len(items); + + // Assert + assert_eq!(budgeted, actual); +} diff --git a/src/domains/kv/tests/watch_registry.rs b/src/domains/kv/tests/watch_registry.rs new file mode 100644 index 00000000..6a439181 --- /dev/null +++ b/src/domains/kv/tests/watch_registry.rs @@ -0,0 +1,22 @@ +use super::*; + +#[test] +fn should_remove_watch_session_subscriptions_on_cleanup() { + // Arrange + let family = RouteFamily::new(1); + let mut registry = KvWatchRegistry::new(family); + let route = RouteAddress::new(family, Route::new("inbox://session/7")); + registry + .subscribe(7, Pattern::new("kv://acme/app/users"), route.clone()) + .expect("subscribe users"); + registry + .subscribe(7, Pattern::new("kv://acme/app/orders"), route) + .expect("subscribe orders"); + + // Act + let removed = registry.remove_session(7); + + // Assert + assert_eq!(removed, 2); + assert!(registry.is_empty()); +} diff --git a/src/domains/kv/watch.rs b/src/domains/kv/watch_registry.rs similarity index 73% rename from src/domains/kv/watch.rs rename to src/domains/kv/watch_registry.rs index b2230f02..2ac9729e 100644 --- a/src/domains/kv/watch.rs +++ b/src/domains/kv/watch_registry.rs @@ -1,16 +1,18 @@ +//! Per-family ephemeral KV watch subscription state. + use crate::domains::subscription_state::{RoutedSubscription, RoutedSubscriptionSet}; use crate::runtime::matcher::Pattern; use crate::runtime::routing::{Route, RouteAddress, RouteFamily}; use std::sync::atomic::{AtomicU64, Ordering}; -pub struct KvWatchActor { +pub(crate) struct KvWatchRegistry { family_id: RouteFamily, subscriptions: RoutedSubscriptionSet, next_sub_id: AtomicU64, } #[derive(Clone)] -pub struct KvWatchTarget { +pub(crate) struct KvWatchTarget { pub session_id: u64, pub subscription_id: u64, pub subscriber: RouteAddress, @@ -37,9 +39,9 @@ impl RoutedSubscription for KvWatchSubscription { } } -impl KvWatchActor { +impl KvWatchRegistry { #[must_use] - pub fn new(family_id: RouteFamily) -> Self { + pub(crate) fn new(family_id: RouteFamily) -> Self { Self { family_id, subscriptions: RoutedSubscriptionSet::new(), @@ -51,7 +53,7 @@ impl KvWatchActor { /// /// Returns `KvError::SubscriptionLimit` when a new wildcard registration /// would exceed the per-session wildcard quota. - pub fn subscribe( + pub(crate) fn subscribe( &mut self, session_id: u64, pattern: Pattern, @@ -88,28 +90,28 @@ impl KvWatchActor { Ok(subscription_id) } - pub fn unsubscribe(&mut self, session_id: u64, pattern: &str) -> usize { + pub(crate) fn unsubscribe(&mut self, session_id: u64, pattern: &str) -> usize { self.subscriptions .remove_session_pattern(self.family_id, session_id, pattern) } - pub fn remove_session(&mut self, session_id: u64) -> usize { + pub(crate) fn remove_session(&mut self, session_id: u64) -> usize { self.subscriptions .remove_session(self.family_id, session_id) } #[must_use] - pub fn is_empty(&self) -> bool { + pub(crate) fn is_empty(&self) -> bool { self.subscriptions.is_empty() } #[must_use] - pub fn subscription_count(&self) -> usize { + pub(crate) fn subscription_count(&self) -> usize { self.subscriptions.subscription_count() } #[must_use] - pub fn matching_targets(&self, route: &Route) -> Vec { + pub(crate) fn matching_targets(&self, route: &Route) -> Vec { let mut targets = Vec::with_capacity(self.subscriptions.matching_capacity_hint(route.as_str())); self.subscriptions.for_each_matching_route( @@ -128,27 +130,5 @@ impl KvWatchActor { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn should_remove_watch_session_subscriptions_on_cleanup() { - // Arrange - let family = RouteFamily::new(1); - let mut actor = KvWatchActor::new(family); - let route = RouteAddress::new(family, Route::new("inbox://session/7")); - actor - .subscribe(7, Pattern::new("kv://acme/app/users"), route.clone()) - .expect("subscribe users"); - actor - .subscribe(7, Pattern::new("kv://acme/app/orders"), route) - .expect("subscribe orders"); - - // Act - let removed = actor.remove_session(7); - - // Assert - assert_eq!(removed, 2); - assert!(actor.is_empty()); - } -} +#[path = "tests/watch_registry.rs"] +mod tests; diff --git a/src/domains/lease/sink/domain_sink_impl/mod.rs b/src/domains/lease/sink/acquire.rs similarity index 77% rename from src/domains/lease/sink/domain_sink_impl/mod.rs rename to src/domains/lease/sink/acquire.rs index ce88d7a8..208ed0fa 100644 --- a/src/domains/lease/sink/domain_sink_impl/mod.rs +++ b/src/domains/lease/sink/acquire.rs @@ -1,46 +1,16 @@ +//! Lease acquire/extend/release/query business logic: authorization, +//! ownership mutation, and FIFO waiter queuing. + use super::model::{ Duration, Instant, LeaseAcquireRequest, LeaseDomainRuntime, Ordering, PendingAcquire, QueuedAcquireRequest, SinkLeaseState, Utc, LEASE_MAX_QUEUE_DEPTH, LEASE_MAX_WAIT_SECONDS, }; -use crate::domains::subscription_state::RoutedSubscriptionSet; - -mod expiry; -mod routing; -mod waiter_tracking; enum AcquireDecision { Respond(crate::domains::lease::protocol::LeaseResponse), Queue(QueuedAcquireRequest), } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum WaiterProgress { - Unchanged, - Expired, - Consumed, -} - -#[derive(Clone, Copy)] -enum DeliveryDropKind { - Response, - Notification, -} - -impl WaiterProgress { - const fn changed(self) -> bool { - !matches!(self, Self::Unchanged) - } -} - -impl DeliveryDropKind { - const fn label(self) -> &'static str { - match self { - Self::Response => "response", - Self::Notification => "notification", - } - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum LeaseAuthorization { Missing, @@ -77,9 +47,6 @@ fn authorize_owned_lease( } } -#[cfg(test)] -mod tests; - impl LeaseDomainRuntime<'_> { fn apply_lease_effects( &self, @@ -102,74 +69,6 @@ impl LeaseDomainRuntime<'_> { } } - /// Drops session waiters before ownership and grants released keys in FIFO order. - pub fn cleanup_session(&self, session_id: u64) { - let now = Instant::now(); - let tracked_keys = self - .core - .session_leases - .lock() - .remove(&session_id) - .map(|keys| keys.into_iter().collect::>()) - .unwrap_or_default(); - let removed_waiters = self.remove_session_waiters(session_id); - - let mut removed_keys = Vec::with_capacity(tracked_keys.len()); - if !tracked_keys.is_empty() { - let mut leases = self.core.leases.lock(); - for key in tracked_keys { - if leases.remove(&key).is_some() { - removed_keys.push(key); - } - } - } - - let removed_subscriptions = self.unsubscribe_all(session_id); - for key in &removed_keys { - self.remove_admin_lease(key); - self.notify_lease_change(key); - } - for key in &removed_keys { - let _ = self.advance_waiter_queue(key, now); - } - - tracing::debug!( - domain = "lease", - session = session_id, - count_removed = removed_keys.len(), - waiters_removed = removed_waiters, - subscriptions_removed = removed_subscriptions, - "Lease: released all leases for disconnected session" - ); - self.refresh_metrics_gauges(); - } - - pub fn lease_count(&self) -> usize { - self.core.leases.lock().len() - } - - pub fn subscription_count(&self) -> usize { - let families = self.core.families.lock(); - families - .values() - .map(RoutedSubscriptionSet::subscription_count) - .sum() - } - - pub(super) fn unsubscribe_all(&self, session_id: u64) -> usize { - let mut families = self.core.families.lock(); - let mut removed = 0; - for (family_id, state) in families.iter_mut() { - removed += state.remove_session( - crate::runtime::routing::RouteFamily::try_from(*family_id) - .expect("lease family IDs originate from RouteFamily"), - session_id, - ); - } - families.retain(|_, state| !state.is_empty()); - removed - } - pub(super) fn next_fencing_token(&self) -> Option { self.core .next_token @@ -490,19 +389,4 @@ impl LeaseDomainRuntime<'_> { } #[cfg(test)] -fn test_protocol_channel_from_client( - channel: crate::runtime::ClientChannel, -) -> crate::dispatch::protocol::frame::ChannelId { - match channel { - crate::runtime::ClientChannel::Control => { - crate::dispatch::protocol::frame::ChannelId::Control - } - crate::runtime::ClientChannel::Pub => crate::dispatch::protocol::frame::ChannelId::Pub, - crate::runtime::ClientChannel::Sub => crate::dispatch::protocol::frame::ChannelId::Sub, - crate::runtime::ClientChannel::Rpc => crate::dispatch::protocol::frame::ChannelId::Rpc, - crate::runtime::ClientChannel::Lease => crate::dispatch::protocol::frame::ChannelId::Lease, - crate::runtime::ClientChannel::Internal => { - crate::dispatch::protocol::frame::ChannelId::Internal - } - } -} +mod tests; diff --git a/src/domains/lease/sink/domain_sink_impl/tests.rs b/src/domains/lease/sink/acquire/tests.rs similarity index 100% rename from src/domains/lease/sink/domain_sink_impl/tests.rs rename to src/domains/lease/sink/acquire/tests.rs diff --git a/src/domains/lease/sink/cleanup.rs b/src/domains/lease/sink/cleanup.rs new file mode 100644 index 00000000..c4dd9c28 --- /dev/null +++ b/src/domains/lease/sink/cleanup.rs @@ -0,0 +1,126 @@ +//! Disconnect cleanup and stale queued-request rejection state. +//! +//! `SessionCleanup` is delivered on the high-priority mailbox lane, so it can +//! pass an older, already-queued normal-lane request from the same session. +//! Remembering the cleaned-up session lets that stale request fail instead of +//! silently recreating a lease/waiter/subscription for a session that is +//! already gone and will never be cleaned up again. + +use super::model::{Instant, LeaseDomainRuntime}; + +impl LeaseDomainRuntime<'_> { + pub(super) fn is_cleaned_up_session(&self, session_id: u64) -> bool { + self.core.cleaned_up_sessions.lock().contains(session_id) + } + + pub(super) fn handle_cleanup_envelope(&self, envelope: &crate::runtime::Envelope) -> bool { + if let Some(cleanup) = envelope.payload::() { + // Mark first so an older normal-lane request that cleanup jumped + // over cannot recreate a lease, waiter, or subscription for this + // session below. + self.core + .cleaned_up_sessions + .lock() + .mark(cleanup.session_id); + self.cleanup_session(cleanup.session_id); + return true; + } + + false + } + + /// Drops session waiters before ownership and grants released keys in FIFO order. + pub fn cleanup_session(&self, session_id: u64) { + let now = Instant::now(); + let tracked_keys = self + .core + .session_leases + .lock() + .remove(&session_id) + .map(|keys| keys.into_iter().collect::>()) + .unwrap_or_default(); + let removed_waiters = self.remove_session_waiters(session_id); + + let mut removed_keys = Vec::with_capacity(tracked_keys.len()); + if !tracked_keys.is_empty() { + let mut leases = self.core.leases.lock(); + for key in tracked_keys { + if leases.remove(&key).is_some() { + removed_keys.push(key); + } + } + } + + let removed_subscriptions = self.unsubscribe_all(session_id); + for key in &removed_keys { + self.remove_admin_lease(key); + self.notify_lease_change(key); + } + for key in &removed_keys { + let _ = self.advance_waiter_queue(key, now); + } + + tracing::debug!( + domain = "lease", + session = session_id, + count_removed = removed_keys.len(), + waiters_removed = removed_waiters, + subscriptions_removed = removed_subscriptions, + "Lease: released all leases for disconnected session" + ); + self.refresh_metrics_gauges(); + } + + /// Removes every queued waiter owned by the session before empty queues are dropped. + pub(in crate::domains::lease::sink) fn remove_session_waiters(&self, session_id: u64) -> usize { + let waiter_refs = self + .core + .session_waiters + .lock() + .remove(&session_id) + .map(|waiters| waiters.into_iter().collect::>()) + .unwrap_or_default(); + + if waiter_refs.is_empty() { + return 0; + } + + let mut removed = 0; + let mut pending_acquires = self.core.pending_acquires.lock(); + let mut empty_keys = Vec::new(); + for waiter_ref in waiter_refs { + if let Some(queue) = pending_acquires.get_mut(&waiter_ref.key) { + if let Some(index) = queue + .iter() + .position(|waiter| waiter.queued_token == waiter_ref.queued_token) + { + queue.remove(index); + removed += 1; + } + if queue.is_empty() { + empty_keys.push(waiter_ref.key.clone()); + } + } + } + + for key in empty_keys { + pending_acquires.remove(&key); + } + + removed + } + + pub(super) fn unsubscribe_all(&self, session_id: u64) -> usize { + let mut families = self.core.families.lock(); + let mut removed = 0; + for (family_id, state) in families.iter_mut() { + removed += state.remove_session( + crate::runtime::routing::RouteFamily::try_from(*family_id) + .expect("lease family IDs originate from RouteFamily"), + session_id, + ); + } + families.retain(|_, state| !state.is_empty()); + removed + } +} diff --git a/src/domains/lease/sink/delivery.rs b/src/domains/lease/sink/delivery.rs new file mode 100644 index 00000000..6357f458 --- /dev/null +++ b/src/domains/lease/sink/delivery.rs @@ -0,0 +1,101 @@ +//! Publish fan-out: notifying subscribers of a lease state change. + +#[cfg(test)] +use crate::dispatch::protocol::frame_context::FrameContext; +use crate::runtime::Envelope; + +use super::model::LeaseDomainRuntime; + +impl LeaseDomainRuntime<'_> { + pub(in crate::domains::lease::sink) fn notify_lease_change( + &self, + key: &crate::domains::lease::protocol::LeaseKey, + ) { + if !self.core.families.lock().contains_key(&key.family.as_u64()) { + return; + } + + let event = crate::runtime::DomainPublishEvent::new( + key.family, + key.to_route(), + bytes::Bytes::new(), + ); + self.handle_domain_publish(&event); + } + + /// Removes both the per-session references and their matching per-key queue entries. + pub(in crate::domains::lease::sink) fn handle_domain_publish( + &self, + event: &crate::runtime::DomainPublishEvent, + ) { + let family_id = event.family_id.as_u64(); + let targets = { + let families = self.core.families.lock(); + let mut targets = Vec::new(); + if let Some(family_state) = families.get(&family_id) { + family_state.for_each_matching(event, |sub| { + targets.push(( + sub.session_id, + sub.subscription_id, + sub.route_address.clone(), + )); + }); + } + targets + }; + + #[cfg(test)] + let mut payload_encoder = + crate::dispatch::protocol::payload_codec::PayloadEncoder::with_capacity(256); + for (session_id, subscription_id, route_address) in targets { + #[cfg(test)] + { + let notify_payload = crate::dispatch::protocol::lease_codec::encode_notify_into( + &mut payload_encoder, + subscription_id, + event.route.as_str(), + &event.payload, + ); + let notify_ctx = FrameContext::new( + session_id, + crate::dispatch::protocol::frame::ChannelId::Sub, + crate::dispatch::protocol::tlv::MessageType::new( + crate::dispatch::protocol::lease_codec::msg_type::NOTIFY, + ), + bytes::Bytes::from(notify_payload), + event.family_id, + ); + + let notify_envelope = Envelope::new(route_address, notify_ctx); + if let Err(error) = self.core.router.route(notify_envelope) { + self.record_dropped_delivery( + super::observability::DeliveryDropKind::Notification, + session_id, + event.family_id, + &error, + ); + } + } + + #[cfg(not(test))] + { + let notification = crate::domains::lease::LeaseClientNotification::new( + session_id, + event.family_id, + subscription_id, + event.route.clone(), + event.payload.clone(), + ); + let notify_envelope = Envelope::new(route_address, notification); + if let Err(error) = self.core.router.route(notify_envelope) { + self.record_dropped_delivery( + super::observability::DeliveryDropKind::Notification, + session_id, + event.family_id, + &error, + ); + } + } + } + } +} diff --git a/src/domains/lease/sink/domain_sink_impl/routing.rs b/src/domains/lease/sink/domain_sink_impl/routing.rs deleted file mode 100644 index 818f090e..00000000 --- a/src/domains/lease/sink/domain_sink_impl/routing.rs +++ /dev/null @@ -1,235 +0,0 @@ -use super::super::model::{LeaseDomainRuntime, PendingAcquire}; -#[cfg(test)] -use super::test_protocol_channel_from_client; -use super::DeliveryDropKind; -#[cfg(test)] -use crate::dispatch::protocol::frame_context::FrameContext; -use crate::runtime::Envelope; - -impl LeaseDomainRuntime<'_> { - fn record_dropped_delivery( - &self, - kind: DeliveryDropKind, - session_id: u64, - route_family: crate::runtime::routing::RouteFamily, - error: &impl std::fmt::Display, - ) { - match (self.core.metrics.as_ref(), kind) { - (Some(metrics), DeliveryDropKind::Response) => metrics.record_response_drop(), - (Some(metrics), DeliveryDropKind::Notification) => metrics.record_notify_drop(), - (None, DeliveryDropKind::Response) => crate::observability::counter_inc( - crate::domains::lease::metrics::METRIC_RESPONSE_DROPS_TOTAL, - ), - (None, DeliveryDropKind::Notification) => crate::observability::counter_inc( - crate::domains::lease::metrics::METRIC_NOTIFY_DROPS_TOTAL, - ), - } - tracing::warn!( - domain = "lease", - delivery_kind = kind.label(), - session_id, - route_family = route_family.as_u64(), - error = %error, - "Dropped best-effort Lease delivery" - ); - } - - pub(in crate::domains::lease::sink) fn send_waiter_response( - &self, - waiter: &PendingAcquire, - response: &crate::domains::lease::protocol::LeaseResponse, - ) { - #[cfg(test)] - let response_ctx = { - let mut payload_encoder = - crate::dispatch::protocol::payload_codec::PayloadEncoder::with_capacity(128); - let response_bytes = - crate::dispatch::protocol::lease_codec::encode_domain_response_into( - &mut payload_encoder, - response, - ); - FrameContext::new( - waiter.owner_session_id, - test_protocol_channel_from_client(waiter.channel), - crate::dispatch::protocol::tlv::MessageType::new( - crate::dispatch::protocol::lease_codec::msg_type::ACQUIRE, - ), - bytes::Bytes::from(response_bytes), - waiter.route_family, - ) - }; - - #[cfg(not(test))] - let response_ctx = crate::domains::lease::LeaseClientResponse::new( - crate::runtime::ClientFrameMeta::new( - waiter.owner_session_id, - waiter.channel, - crate::dispatch::protocol::lease_codec::msg_type::ACQUIRE, - waiter.route_family, - ), - response.clone(), - ); - - let response_envelope = Envelope::from_route( - waiter.reply_source.clone(), - waiter.reply_destination.clone(), - response_ctx, - ); - if let Err(error) = self.core.router.route(response_envelope) { - self.record_dropped_delivery( - DeliveryDropKind::Response, - waiter.owner_session_id, - waiter.route_family, - &error, - ); - } - } - - pub(in crate::domains::lease::sink) fn route_lease_response( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - response: &crate::domains::lease::protocol::LeaseResponse, - request_started: Option, - ) { - #[cfg(test)] - let response_ctx = { - let response_bytes = - crate::dispatch::protocol::lease_codec::encode_domain_response(response); - FrameContext::new( - meta.session_id, - test_protocol_channel_from_client(meta.channel), - crate::dispatch::protocol::tlv::MessageType::new(meta.message_type), - bytes::Bytes::from(response_bytes), - meta.route_family, - ) - }; - - #[cfg(not(test))] - let response_ctx = crate::domains::lease::LeaseClientResponse::new(meta, response.clone()); - - if let Some(response_envelope) = envelope.try_reply_to(response_ctx) { - let response_sink = self - .core - .router - .resolve_sink(response_envelope.destination()); - if let Some(sink) = response_sink { - if let Err(error) = sink.deliver(response_envelope) { - self.record_dropped_delivery( - DeliveryDropKind::Response, - meta.session_id, - meta.route_family, - &error, - ); - } - } else if let Err(error) = self.core.router.route(response_envelope) { - self.record_dropped_delivery( - DeliveryDropKind::Response, - meta.session_id, - meta.route_family, - &error, - ); - } - } - - if let (Some(metrics), Some(started_at)) = (self.core.metrics.as_ref(), request_started) { - if Self::lease_response_is_failure(response) { - metrics.record_failure(started_at); - } else { - metrics.record_success(started_at); - } - } - } - - pub(in crate::domains::lease::sink) fn notify_lease_change( - &self, - key: &crate::domains::lease::protocol::LeaseKey, - ) { - if !self.core.families.lock().contains_key(&key.family.as_u64()) { - return; - } - - let event = crate::runtime::DomainPublishEvent::new( - key.family, - key.to_route(), - bytes::Bytes::new(), - ); - self.handle_domain_publish(&event); - } - - /// Removes both the per-session references and their matching per-key queue entries. - pub(in crate::domains::lease::sink) fn handle_domain_publish( - &self, - event: &crate::runtime::DomainPublishEvent, - ) { - let family_id = event.family_id.as_u64(); - let targets = { - let families = self.core.families.lock(); - let mut targets = Vec::new(); - if let Some(family_state) = families.get(&family_id) { - family_state.for_each_matching(event, |sub| { - targets.push(( - sub.session_id, - sub.subscription_id, - sub.route_address.clone(), - )); - }); - } - targets - }; - - #[cfg(test)] - let mut payload_encoder = - crate::dispatch::protocol::payload_codec::PayloadEncoder::with_capacity(256); - for (session_id, subscription_id, route_address) in targets { - #[cfg(test)] - { - let notify_payload = crate::dispatch::protocol::lease_codec::encode_notify_into( - &mut payload_encoder, - subscription_id, - event.route.as_str(), - &event.payload, - ); - let notify_ctx = FrameContext::new( - session_id, - crate::dispatch::protocol::frame::ChannelId::Sub, - crate::dispatch::protocol::tlv::MessageType::new( - crate::dispatch::protocol::lease_codec::msg_type::NOTIFY, - ), - bytes::Bytes::from(notify_payload), - event.family_id, - ); - - let notify_envelope = Envelope::new(route_address, notify_ctx); - if let Err(error) = self.core.router.route(notify_envelope) { - self.record_dropped_delivery( - DeliveryDropKind::Notification, - session_id, - event.family_id, - &error, - ); - } - } - - #[cfg(not(test))] - { - let notification = crate::domains::lease::LeaseClientNotification::new( - session_id, - event.family_id, - subscription_id, - event.route.clone(), - event.payload.clone(), - ); - let notify_envelope = Envelope::new(route_address, notification); - if let Err(error) = self.core.router.route(notify_envelope) { - self.record_dropped_delivery( - DeliveryDropKind::Notification, - session_id, - event.family_id, - &error, - ); - } - } - } - } -} diff --git a/src/domains/lease/sink/domain_sink_impl/expiry.rs b/src/domains/lease/sink/expiry.rs similarity index 84% rename from src/domains/lease/sink/domain_sink_impl/expiry.rs rename to src/domains/lease/sink/expiry.rs index 1eb9a1f4..e76b68ec 100644 --- a/src/domains/lease/sink/domain_sink_impl/expiry.rs +++ b/src/domains/lease/sink/expiry.rs @@ -1,7 +1,22 @@ -use super::super::model::{Instant, LeaseDomainRuntime, PendingAcquire, SinkLeaseState, Utc}; -use super::WaiterProgress; +//! TTL expiry: reaping timed-out waiters and expired leases, and advancing +//! each key's FIFO wait queue once it becomes free. + +use super::model::{Instant, LeaseDomainRuntime, PendingAcquire, SinkLeaseState, Utc}; use std::collections::VecDeque; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum WaiterProgress { + Unchanged, + Expired, + Consumed, +} + +impl WaiterProgress { + pub(super) const fn changed(self) -> bool { + !matches!(self, Self::Unchanged) + } +} + fn drain_expired_waiters( queue: &mut VecDeque, now: Instant, @@ -19,45 +34,6 @@ fn drain_expired_waiters( } impl LeaseDomainRuntime<'_> { - /// Removes every queued waiter owned by the session before empty queues are dropped. - pub(in crate::domains::lease::sink) fn remove_session_waiters(&self, session_id: u64) -> usize { - let waiter_refs = self - .core - .session_waiters - .lock() - .remove(&session_id) - .map(|waiters| waiters.into_iter().collect::>()) - .unwrap_or_default(); - - if waiter_refs.is_empty() { - return 0; - } - - let mut removed = 0; - let mut pending_acquires = self.core.pending_acquires.lock(); - let mut empty_keys = Vec::new(); - for waiter_ref in waiter_refs { - if let Some(queue) = pending_acquires.get_mut(&waiter_ref.key) { - if let Some(index) = queue - .iter() - .position(|waiter| waiter.queued_token == waiter_ref.queued_token) - { - queue.remove(index); - removed += 1; - } - if queue.is_empty() { - empty_keys.push(waiter_ref.key.clone()); - } - } - } - - for key in empty_keys { - pending_acquires.remove(&key); - } - - removed - } - pub(in crate::domains::lease::sink) fn expire_timed_out_waiters_for_key( &self, key: &crate::domains::lease::protocol::LeaseKey, diff --git a/src/domains/lease/sink/lifecycle_and_admin/lifecycle.rs b/src/domains/lease/sink/facade.rs similarity index 96% rename from src/domains/lease/sink/lifecycle_and_admin/lifecycle.rs rename to src/domains/lease/sink/facade.rs index ca6aa73e..e38cf890 100644 --- a/src/domains/lease/sink/lifecycle_and_admin/lifecycle.rs +++ b/src/domains/lease/sink/facade.rs @@ -1,6 +1,8 @@ +//! Public `LeaseDomainSink` API and actor lifecycle management. + #[cfg(any(test, feature = "benchkit"))] -use super::super::model::LeaseAcquireRequest; -use super::super::model::{ +use super::model::LeaseAcquireRequest; +use super::model::{ Arc, AtomicBool, AtomicU64, HashMap, LeaseDomainActor, LeaseDomainCommand, LeaseDomainCore, LeaseDomainRuntime, LeaseDomainSink, LeaseDomainState, LeaseLiveCounts, LeaseMetrics, Mutex, Ordering, LEASE_ACTOR_REPLY_TIMEOUT, @@ -19,6 +21,9 @@ impl LeaseDomainState { session_leases: Mutex::new(HashMap::new()), pending_acquires: Mutex::new(HashMap::new()), session_waiters: Mutex::new(HashMap::new()), + cleaned_up_sessions: Mutex::new(crate::runtime::CleanedUpSessions::new( + crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, + )), next_token: AtomicU64::new(1), router, families: Mutex::new(HashMap::new()), diff --git a/src/domains/lease/sink/mailbox_sink_impl.rs b/src/domains/lease/sink/ingress.rs similarity index 65% rename from src/domains/lease/sink/mailbox_sink_impl.rs rename to src/domains/lease/sink/ingress.rs index 4c820129..54eb36ce 100644 --- a/src/domains/lease/sink/mailbox_sink_impl.rs +++ b/src/domains/lease/sink/ingress.rs @@ -1,13 +1,12 @@ -use super::model::{ - DeliveryError, Envelope, LeaseAcquireRequest, LeaseDomainActor, LeaseDomainCommand, - LeaseDomainRuntime, LeaseDomainSink, LeaseSubscription, MailboxSink, Ordering, - RoutedSubscriptionSet, -}; +//! Envelope ingress: validate an inbound envelope, parse it into a Lease +//! request, and dispatch to the subscriptions/acquire/response layers. + +use super::model::{DeliveryError, LeaseAcquireRequest, LeaseDomainRuntime}; #[cfg(test)] use crate::dispatch::protocol::frame_context::FrameContext; -use crate::runtime::{Actor, Context}; +use crate::runtime::Envelope; -enum LeaseRequestView<'a> { +pub(super) enum LeaseRequestView<'a> { Borrowed(&'a crate::domains::lease::LeaseClientRequest), #[cfg(test)] Owned(crate::domains::lease::LeaseClientRequest), @@ -31,91 +30,6 @@ impl LeaseRequestView<'_> { } } -impl MailboxSink for LeaseDomainSink { - fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { - self.actor.try_send(LeaseDomainCommand::Deliver(envelope)) - } - - fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { - self.actor - .try_send_high_priority(LeaseDomainCommand::Deliver(envelope)) - } -} - -impl Actor for LeaseDomainActor { - type Message = LeaseDomainCommand; - - fn receive(&mut self, msg: Self::Message, _ctx: &mut Context) { - let runtime = self.state.runtime(); - match msg { - LeaseDomainCommand::Deliver(envelope) => { - if let Err(error) = runtime.deliver_envelope(&envelope) { - tracing::warn!(domain = "lease", error = %error, "Lease actor delivery failed"); - } - } - LeaseDomainCommand::CleanupSession(session_id) => { - runtime.cleanup_session(session_id); - } - LeaseDomainCommand::ReadLiveCounts(reply) => { - let _ = reply.send(runtime.live_counts()); - } - LeaseDomainCommand::ReadWaiters(reply) => { - let _ = reply.send(runtime.admin_waiters()); - } - LeaseDomainCommand::SweepExpiredState => { - runtime.sweep_expired_state(); - } - #[cfg(any(test, feature = "benchkit"))] - LeaseDomainCommand::ApplyAcquireForBench(request, reply) => { - let _ = reply.send(runtime.handle_acquire(request)); - } - #[cfg(any(test, feature = "benchkit"))] - LeaseDomainCommand::ApplyReleaseForBench(key, owner_id, fencing_token, reply) => { - let _ = reply.send(runtime.handle_release(&key, owner_id.as_str(), fencing_token)); - } - #[cfg(test)] - LeaseDomainCommand::ApplyAcquireForTests(request, reply) => { - let _ = reply.send(runtime.handle_acquire(request)); - } - #[cfg(test)] - LeaseDomainCommand::ApplyExtendForTests( - key, - owner_id, - fencing_token, - ttl_secs, - reply, - ) => { - let _ = reply.send(runtime.handle_extend( - &key, - owner_id.as_str(), - fencing_token, - ttl_secs, - )); - } - #[cfg(test)] - LeaseDomainCommand::ExpireLeaseForTests(key, reply) => { - let expired = if let Some(lease) = runtime.core.leases.lock().get_mut(&key) { - lease.expiry = std::time::Instant::now() - .checked_sub(std::time::Duration::from_millis(1)) - .expect("past instant"); - true - } else { - false - }; - let _ = reply.send(expired); - } - #[cfg(test)] - LeaseDomainCommand::ReadPendingWaiterCountForTests(key, reply) => { - let _ = reply.send(runtime.pending_waiter_count(&key)); - } - #[cfg(test)] - LeaseDomainCommand::PanicForTests => { - panic!("test Lease domain actor panic"); - } - } - } -} - impl LeaseDomainRuntime<'_> { pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { if self.handle_cleanup_envelope(envelope) { @@ -155,6 +69,18 @@ impl LeaseDomainRuntime<'_> { return Ok(()); } + // This request was already queued (on the normal lane) before this + // session's disconnect cleanup ran (on the high-priority lane) and + // jumped ahead of it. Reject rather than silently recreating a + // lease, waiter, or subscription for a session that is already gone + // and will never be cleaned up again. + if self.is_cleaned_up_session(meta.session_id) { + let response = Self::error_response("session already closed"); + let response_meta = Self::response_meta_for_source(envelope, meta); + self.route_lease_response(envelope, response_meta, &response, request_started); + return Ok(()); + } + let Some(parsed_frame) = self.parse_request_frame(envelope, meta, request.frame(), request_started) else { @@ -180,6 +106,11 @@ impl LeaseDomainRuntime<'_> { ) { let meta = request.meta; let request_started = self.record_request_start(); + if self.is_cleaned_up_session(meta.session_id) { + let response = Self::error_response("session already closed"); + self.route_lease_response(envelope, meta, &response, request_started); + return; + } let Some(operation) = self.parse_prepared_request_frame(envelope, meta, &request.frame, request_started) else { @@ -195,21 +126,8 @@ impl LeaseDomainRuntime<'_> { self.handle_prepared_operation_frame(envelope, meta, request_started, operation); } - fn handle_cleanup_envelope(&self, envelope: &Envelope) -> bool { - if let Some(cleanup) = envelope.payload::() { - self.cleanup_session(cleanup.session_id); - return true; - } - - false - } - fn ensure_active(&self) -> Result<(), DeliveryError> { - if !self.active.load(Ordering::Relaxed) { - return Err(DeliveryError::ActorStopped); - } - - Ok(()) + crate::runtime::ingress_support::ensure_actor_active(self.active) } fn handle_domain_publish_envelope(&self, envelope: &Envelope) -> bool { @@ -226,11 +144,10 @@ impl LeaseDomainRuntime<'_> { } fn log_delivery(envelope: &Envelope) { - tracing::debug!( - domain = "lease", - destination = %envelope.destination(), - source = ?envelope.source(), - "Lease domain sink: received envelope" + crate::runtime::ingress_support::log_envelope_received( + "lease", + "Lease domain sink: received envelope", + envelope, ); } @@ -307,142 +224,6 @@ impl LeaseDomainRuntime<'_> { } } - fn handle_subscription_frame( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - request_started: Option, - sub_msg: &crate::domains::lease::protocol::LeaseSubscriptionMessage, - ) { - use crate::domains::lease::protocol::LeaseSubscriptionMessage; - - let response = match sub_msg { - LeaseSubscriptionMessage::Subscribe { - family_id, - route, - session_id, - subscriber, - } => self.handle_lease_subscribe( - envelope, - meta, - *family_id, - route, - *session_id, - subscriber, - ), - LeaseSubscriptionMessage::Unsubscribe { - family_id, - route, - session_id, - subscriber, - } => self.handle_lease_unsubscribe( - envelope, - meta, - *family_id, - route, - *session_id, - subscriber, - ), - }; - - self.refresh_metrics_gauges(); - self.route_lease_response(envelope, meta, &response, request_started); - } - - fn handle_lease_subscribe( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - family_id: crate::runtime::routing::RouteFamily, - route: &crate::runtime::routing::Route, - session_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - ) -> crate::domains::lease::protocol::LeaseResponse { - use crate::domains::lease::protocol::LeaseResponse; - - if Self::valid_subscription_request(envelope, meta, family_id, session_id, subscriber) { - let compiled = match Self::compile_exact_lease_subscription_route(route) { - Ok(compiled) => compiled, - Err(response) => return response, - }; - let mut families = self.core.families.lock(); - let state = families - .entry(family_id.as_u64()) - .or_insert_with(RoutedSubscriptionSet::new); - if let Some(subscription_id) = state.find_existing_id(session_id, route.as_str()) { - return LeaseResponse::SubscribeOk { subscription_id }; - } - if let Ok(subscription_id) = self.core.next_sub_id.fetch_update( - Ordering::Relaxed, - Ordering::Relaxed, - |current| current.checked_add(1), - ) { - state.insert( - family_id, - LeaseSubscription { - route: compiled, - session_id, - route_address: subscriber.clone(), - subscription_id, - }, - ); - LeaseResponse::SubscribeOk { subscription_id } - } else { - if state.is_empty() { - families.remove(&family_id.as_u64()); - } - LeaseResponse::Error("subscription ID space exhausted".to_string()) - } - } else { - LeaseResponse::Error("route family mismatch".to_string()) - } - } - - fn handle_lease_unsubscribe( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - family_id: crate::runtime::routing::RouteFamily, - route: &crate::runtime::routing::Route, - session_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - ) -> crate::domains::lease::protocol::LeaseResponse { - use crate::domains::lease::protocol::LeaseResponse; - - if Self::valid_subscription_request(envelope, meta, family_id, session_id, subscriber) { - if let Err(response) = Self::compile_exact_lease_subscription_route(route) { - return response; - } - let mut families = self.core.families.lock(); - let remove_family = if let Some(state) = families.get_mut(&family_id.as_u64()) { - state.remove_session_pattern(family_id, session_id, route.as_str()); - state.is_empty() - } else { - false - }; - if remove_family { - families.remove(&family_id.as_u64()); - } - LeaseResponse::UnsubscribeOk - } else { - LeaseResponse::Error("route family mismatch".to_string()) - } - } - - fn compile_exact_lease_subscription_route( - route: &crate::runtime::routing::Route, - ) -> Result - { - use crate::domains::lease::protocol::LeaseResponse; - - // The exact-only rule lives on the Lease descriptor so ingress and - // this sink reject the same patterns. - crate::runtime::DomainKind::Lease - .descriptor() - .compile_registration_pattern(route.as_str()) - .map_err(LeaseResponse::InvalidSubscriptionRoute) - } - fn handle_actor_operation_frame( &self, envelope: &Envelope, @@ -683,19 +464,6 @@ impl LeaseDomainRuntime<'_> { crate::domains::lease::protocol::LeaseResponse::Error(reason.to_string()) } - fn valid_subscription_request( - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - family_id: crate::runtime::routing::RouteFamily, - session_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - ) -> bool { - family_id == meta.route_family - && *subscriber.family() == family_id - && session_id == meta.session_id - && envelope.source().is_none_or(|source| source == subscriber) - } - fn valid_lease_message( envelope: &Envelope, meta: crate::runtime::ClientFrameMeta, diff --git a/src/domains/lease/sink/lifecycle_and_admin/mod.rs b/src/domains/lease/sink/lifecycle_and_admin/mod.rs deleted file mode 100644 index 477c7803..00000000 --- a/src/domains/lease/sink/lifecycle_and_admin/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod admin_projection; -mod lifecycle; diff --git a/src/domains/lease/sink/mailbox.rs b/src/domains/lease/sink/mailbox.rs new file mode 100644 index 00000000..08bf6657 --- /dev/null +++ b/src/domains/lease/sink/mailbox.rs @@ -0,0 +1,90 @@ +//! Mailbox-lane routing and the domain actor's message loop. + +use super::model::{LeaseDomainActor, LeaseDomainCommand, LeaseDomainSink, MailboxSink}; +use crate::runtime::{Actor, Context}; +use crate::runtime::{DeliveryError, Envelope}; + +impl MailboxSink for LeaseDomainSink { + fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.actor.try_send(LeaseDomainCommand::Deliver(envelope)) + } + + fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.actor + .try_send_high_priority(LeaseDomainCommand::Deliver(envelope)) + } +} + +impl Actor for LeaseDomainActor { + type Message = LeaseDomainCommand; + + fn receive(&mut self, msg: Self::Message, _ctx: &mut Context) { + let runtime = self.state.runtime(); + match msg { + LeaseDomainCommand::Deliver(envelope) => { + if let Err(error) = runtime.deliver_envelope(&envelope) { + tracing::warn!(domain = "lease", error = %error, "Lease actor delivery failed"); + } + } + LeaseDomainCommand::CleanupSession(session_id) => { + runtime.cleanup_session(session_id); + } + LeaseDomainCommand::ReadLiveCounts(reply) => { + let _ = reply.send(runtime.live_counts()); + } + LeaseDomainCommand::ReadWaiters(reply) => { + let _ = reply.send(runtime.admin_waiters()); + } + LeaseDomainCommand::SweepExpiredState => { + runtime.sweep_expired_state(); + } + #[cfg(any(test, feature = "benchkit"))] + LeaseDomainCommand::ApplyAcquireForBench(request, reply) => { + let _ = reply.send(runtime.handle_acquire(request)); + } + #[cfg(any(test, feature = "benchkit"))] + LeaseDomainCommand::ApplyReleaseForBench(key, owner_id, fencing_token, reply) => { + let _ = reply.send(runtime.handle_release(&key, owner_id.as_str(), fencing_token)); + } + #[cfg(test)] + LeaseDomainCommand::ApplyAcquireForTests(request, reply) => { + let _ = reply.send(runtime.handle_acquire(request)); + } + #[cfg(test)] + LeaseDomainCommand::ApplyExtendForTests( + key, + owner_id, + fencing_token, + ttl_secs, + reply, + ) => { + let _ = reply.send(runtime.handle_extend( + &key, + owner_id.as_str(), + fencing_token, + ttl_secs, + )); + } + #[cfg(test)] + LeaseDomainCommand::ExpireLeaseForTests(key, reply) => { + let expired = if let Some(lease) = runtime.core.leases.lock().get_mut(&key) { + lease.expiry = std::time::Instant::now() + .checked_sub(std::time::Duration::from_millis(1)) + .expect("past instant"); + true + } else { + false + }; + let _ = reply.send(expired); + } + #[cfg(test)] + LeaseDomainCommand::ReadPendingWaiterCountForTests(key, reply) => { + let _ = reply.send(runtime.pending_waiter_count(&key)); + } + #[cfg(test)] + LeaseDomainCommand::PanicForTests => { + panic!("test Lease domain actor panic"); + } + } + } +} diff --git a/src/domains/lease/sink/mod.rs b/src/domains/lease/sink/mod.rs index 22b2f0c4..e070e370 100644 --- a/src/domains/lease/sink/mod.rs +++ b/src/domains/lease/sink/mod.rs @@ -1,10 +1,18 @@ -mod domain_sink_impl; -mod lifecycle_and_admin; -mod mailbox_sink_impl; +mod acquire; +mod cleanup; +mod delivery; +mod expiry; +mod facade; +mod ingress; +mod mailbox; mod model; +mod observability; +mod responses; +mod subscriptions; #[cfg(test)] mod test_actor_commands; mod validation; +mod waiter_tracking; pub use model::LeaseDomainSink; diff --git a/src/domains/lease/sink/model.rs b/src/domains/lease/sink/model.rs index 8e796ef7..765a02b9 100644 --- a/src/domains/lease/sink/model.rs +++ b/src/domains/lease/sink/model.rs @@ -91,6 +91,10 @@ pub(super) struct LeaseDomainCore { pub(super) pending_acquires: Mutex>>, pub(super) session_waiters: Mutex>>, + /// Sessions disconnect cleanup has already run for; guards against a + /// stale queued request recreating a lease/waiter/subscription. See + /// `cleanup.rs`. + pub(super) cleaned_up_sessions: Mutex, /// Process-local fencing token counter; resets on broker restart. pub(super) next_token: AtomicU64, pub(super) router: Arc, diff --git a/src/domains/lease/sink/lifecycle_and_admin/admin_projection.rs b/src/domains/lease/sink/observability.rs similarity index 75% rename from src/domains/lease/sink/lifecycle_and_admin/admin_projection.rs rename to src/domains/lease/sink/observability.rs index 314a6a96..87633c37 100644 --- a/src/domains/lease/sink/lifecycle_and_admin/admin_projection.rs +++ b/src/domains/lease/sink/observability.rs @@ -1,7 +1,51 @@ -use super::super::model::{Instant, LeaseDomainRuntime, LeaseLiveCounts, SinkLeaseState, Utc}; +//! Admin read-model projection and metrics glue. + +use super::model::{Instant, LeaseDomainRuntime, LeaseLiveCounts, SinkLeaseState, Utc}; use std::collections::VecDeque; +#[derive(Clone, Copy)] +pub(super) enum DeliveryDropKind { + Response, + Notification, +} + +impl DeliveryDropKind { + const fn label(self) -> &'static str { + match self { + Self::Response => "response", + Self::Notification => "notification", + } + } +} + impl LeaseDomainRuntime<'_> { + pub(super) fn record_dropped_delivery( + &self, + kind: DeliveryDropKind, + session_id: u64, + route_family: crate::runtime::routing::RouteFamily, + error: &impl std::fmt::Display, + ) { + match (self.core.metrics.as_ref(), kind) { + (Some(metrics), DeliveryDropKind::Response) => metrics.record_response_drop(), + (Some(metrics), DeliveryDropKind::Notification) => metrics.record_notify_drop(), + (None, DeliveryDropKind::Response) => crate::observability::counter_inc( + crate::domains::lease::metrics::METRIC_RESPONSE_DROPS_TOTAL, + ), + (None, DeliveryDropKind::Notification) => crate::observability::counter_inc( + crate::domains::lease::metrics::METRIC_NOTIFY_DROPS_TOTAL, + ), + } + tracing::warn!( + domain = "lease", + delivery_kind = kind.label(), + session_id, + route_family = route_family.as_u64(), + error = %error, + "Dropped best-effort Lease delivery" + ); + } + #[cfg(test)] pub(in crate::domains::lease::sink) fn session_inbox_address( route_family: crate::runtime::routing::RouteFamily, @@ -157,4 +201,16 @@ impl LeaseDomainRuntime<'_> { | crate::domains::lease::protocol::LeaseResponse::InvalidSubscriptionRoute(_) ) } + + pub(super) fn lease_count(&self) -> usize { + self.core.leases.lock().len() + } + + pub(super) fn subscription_count(&self) -> usize { + let families = self.core.families.lock(); + families + .values() + .map(crate::domains::subscription_state::RoutedSubscriptionSet::subscription_count) + .sum() + } } diff --git a/src/domains/lease/sink/responses.rs b/src/domains/lease/sink/responses.rs new file mode 100644 index 00000000..3f888f13 --- /dev/null +++ b/src/domains/lease/sink/responses.rs @@ -0,0 +1,116 @@ +//! Response encoding and best-effort routing back to the requester or to a +//! queued waiter. + +use super::model::LeaseDomainRuntime; +#[cfg(test)] +use crate::dispatch::protocol::frame_context::FrameContext; +use crate::runtime::Envelope; + +impl LeaseDomainRuntime<'_> { + pub(in crate::domains::lease::sink) fn send_waiter_response( + &self, + waiter: &super::model::PendingAcquire, + response: &crate::domains::lease::protocol::LeaseResponse, + ) { + #[cfg(test)] + let response_ctx = { + let mut payload_encoder = + crate::dispatch::protocol::payload_codec::PayloadEncoder::with_capacity(128); + let response_bytes = + crate::dispatch::protocol::lease_codec::encode_domain_response_into( + &mut payload_encoder, + response, + ); + FrameContext::new( + waiter.owner_session_id, + crate::protocol::test_support::channel_id_from_client(waiter.channel), + crate::dispatch::protocol::tlv::MessageType::new( + crate::dispatch::protocol::lease_codec::msg_type::ACQUIRE, + ), + bytes::Bytes::from(response_bytes), + waiter.route_family, + ) + }; + + #[cfg(not(test))] + let response_ctx = crate::domains::lease::LeaseClientResponse::new( + crate::runtime::ClientFrameMeta::new( + waiter.owner_session_id, + waiter.channel, + crate::dispatch::protocol::lease_codec::msg_type::ACQUIRE, + waiter.route_family, + ), + response.clone(), + ); + + let response_envelope = Envelope::from_route( + waiter.reply_source.clone(), + waiter.reply_destination.clone(), + response_ctx, + ); + if let Err(error) = self.core.router.route(response_envelope) { + self.record_dropped_delivery( + super::observability::DeliveryDropKind::Response, + waiter.owner_session_id, + waiter.route_family, + &error, + ); + } + } + + pub(in crate::domains::lease::sink) fn route_lease_response( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + response: &crate::domains::lease::protocol::LeaseResponse, + request_started: Option, + ) { + #[cfg(test)] + let response_ctx = { + let response_bytes = + crate::dispatch::protocol::lease_codec::encode_domain_response(response); + FrameContext::new( + meta.session_id, + crate::protocol::test_support::channel_id_from_client(meta.channel), + crate::dispatch::protocol::tlv::MessageType::new(meta.message_type), + bytes::Bytes::from(response_bytes), + meta.route_family, + ) + }; + + #[cfg(not(test))] + let response_ctx = crate::domains::lease::LeaseClientResponse::new(meta, response.clone()); + + if let Some(response_envelope) = envelope.try_reply_to(response_ctx) { + let response_sink = self + .core + .router + .resolve_sink(response_envelope.destination()); + if let Some(sink) = response_sink { + if let Err(error) = sink.deliver(response_envelope) { + self.record_dropped_delivery( + super::observability::DeliveryDropKind::Response, + meta.session_id, + meta.route_family, + &error, + ); + } + } else if let Err(error) = self.core.router.route(response_envelope) { + self.record_dropped_delivery( + super::observability::DeliveryDropKind::Response, + meta.session_id, + meta.route_family, + &error, + ); + } + } + + if let (Some(metrics), Some(started_at)) = (self.core.metrics.as_ref(), request_started) { + if Self::lease_response_is_failure(response) { + metrics.record_failure(started_at); + } else { + metrics.record_success(started_at); + } + } + } +} diff --git a/src/domains/lease/sink/subscriptions.rs b/src/domains/lease/sink/subscriptions.rs new file mode 100644 index 00000000..78d8f3f2 --- /dev/null +++ b/src/domains/lease/sink/subscriptions.rs @@ -0,0 +1,156 @@ +//! Subscribe/unsubscribe message handling: mutation of the live subscription +//! index in response to a client request. + +use super::model::{LeaseDomainRuntime, LeaseSubscription, Ordering, RoutedSubscriptionSet}; +use crate::runtime::Envelope; + +impl LeaseDomainRuntime<'_> { + pub(super) fn handle_subscription_frame( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + request_started: Option, + sub_msg: &crate::domains::lease::protocol::LeaseSubscriptionMessage, + ) { + use crate::domains::lease::protocol::LeaseSubscriptionMessage; + + let response = match sub_msg { + LeaseSubscriptionMessage::Subscribe { + family_id, + route, + session_id, + subscriber, + } => self.handle_lease_subscribe( + envelope, + meta, + *family_id, + route, + *session_id, + subscriber, + ), + LeaseSubscriptionMessage::Unsubscribe { + family_id, + route, + session_id, + subscriber, + } => self.handle_lease_unsubscribe( + envelope, + meta, + *family_id, + route, + *session_id, + subscriber, + ), + }; + + self.refresh_metrics_gauges(); + self.route_lease_response(envelope, meta, &response, request_started); + } + + fn handle_lease_subscribe( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + family_id: crate::runtime::routing::RouteFamily, + route: &crate::runtime::routing::Route, + session_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + ) -> crate::domains::lease::protocol::LeaseResponse { + use crate::domains::lease::protocol::LeaseResponse; + + if Self::valid_subscription_request(envelope, meta, family_id, session_id, subscriber) { + let compiled = match Self::compile_exact_lease_subscription_route(route) { + Ok(compiled) => compiled, + Err(response) => return response, + }; + let mut families = self.core.families.lock(); + let state = families + .entry(family_id.as_u64()) + .or_insert_with(RoutedSubscriptionSet::new); + if let Some(subscription_id) = state.find_existing_id(session_id, route.as_str()) { + return LeaseResponse::SubscribeOk { subscription_id }; + } + if let Ok(subscription_id) = self.core.next_sub_id.fetch_update( + Ordering::Relaxed, + Ordering::Relaxed, + |current| current.checked_add(1), + ) { + state.insert( + family_id, + LeaseSubscription { + route: compiled, + session_id, + route_address: subscriber.clone(), + subscription_id, + }, + ); + LeaseResponse::SubscribeOk { subscription_id } + } else { + if state.is_empty() { + families.remove(&family_id.as_u64()); + } + LeaseResponse::Error("subscription ID space exhausted".to_string()) + } + } else { + LeaseResponse::Error("route family mismatch".to_string()) + } + } + + fn handle_lease_unsubscribe( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + family_id: crate::runtime::routing::RouteFamily, + route: &crate::runtime::routing::Route, + session_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + ) -> crate::domains::lease::protocol::LeaseResponse { + use crate::domains::lease::protocol::LeaseResponse; + + if Self::valid_subscription_request(envelope, meta, family_id, session_id, subscriber) { + if let Err(response) = Self::compile_exact_lease_subscription_route(route) { + return response; + } + let mut families = self.core.families.lock(); + let remove_family = if let Some(state) = families.get_mut(&family_id.as_u64()) { + state.remove_session_pattern(family_id, session_id, route.as_str()); + state.is_empty() + } else { + false + }; + if remove_family { + families.remove(&family_id.as_u64()); + } + LeaseResponse::UnsubscribeOk + } else { + LeaseResponse::Error("route family mismatch".to_string()) + } + } + + fn compile_exact_lease_subscription_route( + route: &crate::runtime::routing::Route, + ) -> Result + { + use crate::domains::lease::protocol::LeaseResponse; + + // The exact-only rule lives on the Lease descriptor so ingress and + // this sink reject the same patterns. + crate::runtime::DomainKind::Lease + .descriptor() + .compile_registration_pattern(route.as_str()) + .map_err(LeaseResponse::InvalidSubscriptionRoute) + } + + fn valid_subscription_request( + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + family_id: crate::runtime::routing::RouteFamily, + session_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + ) -> bool { + family_id == meta.route_family + && *subscriber.family() == family_id + && session_id == meta.session_id + && envelope.source().is_none_or(|source| source == subscriber) + } +} diff --git a/src/domains/lease/sink/tests.rs b/src/domains/lease/sink/tests.rs index aae5f394..3a096cf3 100644 --- a/src/domains/lease/sink/tests.rs +++ b/src/domains/lease/sink/tests.rs @@ -397,6 +397,49 @@ fn should_clear_session_state_given_session_cleanup() { assert!(sink.watch_families_are_empty_for_tests()); } +#[test] +fn should_reject_stale_acquire_after_disconnect_cleanup_marks_session() { + // Arrange + let family = RouteFamily::new(1); + let session_id = 9; + let lease_route = "lease://acme/locks/resource"; + let lease_address = RouteAddress::new(family, Route::new(lease_route)); + let subscriber_address = RouteAddress::new(family, Route::new("inbox://session/9")); + let router = Arc::new(Router::new()); + let subscriber_mailbox = Arc::new(Mailbox::new(8)); + router.register(subscriber_address.clone(), subscriber_mailbox.clone()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = LeaseDomainSink::new(router, admin_read_model); + + // Act: cleanup for this session runs and completes before the stale + // acquire below is processed - equivalent to what the high-priority + // mailbox lane guarantees a real disconnect races against a queued + // normal-lane request. + sink.deliver(Envelope::new( + RouteAddress::new(family, Route::new("lease://cleanup")), + crate::runtime::SessionCleanup { session_id }, + )) + .expect("cleanup session"); + + sink.deliver(Envelope::from_route( + subscriber_address, + lease_address, + FrameContext::new( + session_id, + ChannelId::Sub, + MessageType::new(400), + encode_lease_acquire(lease_route, "", 30), + family, + ), + )) + .expect("deliver stale acquire"); + let _ack = receive_envelope(&subscriber_mailbox, "stale acquire response envelope"); + + // Assert: the stale acquire from the now-cleaned-up session is rejected + // instead of resurrecting a lease for it. + assert_eq!(sink.lease_count(), 0); +} + #[test] fn should_preserve_other_session_leases_given_session_cleanup() { // Arrange diff --git a/src/domains/lease/sink/domain_sink_impl/waiter_tracking.rs b/src/domains/lease/sink/waiter_tracking.rs similarity index 90% rename from src/domains/lease/sink/domain_sink_impl/waiter_tracking.rs rename to src/domains/lease/sink/waiter_tracking.rs index 002992c1..1a9d7f9f 100644 --- a/src/domains/lease/sink/domain_sink_impl/waiter_tracking.rs +++ b/src/domains/lease/sink/waiter_tracking.rs @@ -1,4 +1,7 @@ -use super::super::model::{LeaseDomainRuntime, PendingAcquireRef}; +//! Per-session index of owned leases and queued waiters, used by cleanup and +//! by acquire/expiry bookkeeping to keep both directions in sync. + +use super::model::{LeaseDomainRuntime, PendingAcquireRef}; impl LeaseDomainRuntime<'_> { pub(super) fn track_session_lease( diff --git a/src/domains/notice/metrics.rs b/src/domains/notice/metrics.rs index 526d99c2..5320a559 100644 --- a/src/domains/notice/metrics.rs +++ b/src/domains/notice/metrics.rs @@ -9,6 +9,18 @@ pub const METRIC_SUBSCRIPTIONS_GAUGE: &str = "fitz_notice_subscriptions_gauge"; pub const METRIC_RESPONSE_DROPS_TOTAL: &str = "fitz_notice_response_drops_total"; // Wire name kept as `delivery` rather than `notify` for dashboard compatibility. pub const METRIC_DELIVERY_DROPS_TOTAL: &str = "fitz_notice_delivery_drops_total"; +/// Deliveries handed to a worker that did not confirm within the handoff +/// window. Distinct from a drop: the worker may still complete the delivery, +/// so counting these as drops would overstate loss. Previously the timeout was +/// discarded entirely, leaving the one uncertain outcome invisible while both +/// certain failures were counted. +pub const METRIC_DELIVERY_HANDOFF_TIMEOUTS_TOTAL: &str = + "fitz_notice_delivery_handoff_timeouts_total"; +/// Accepted deliveries whose envelope failed inside the actor. The client was +/// already told the publish was accepted, so there is nobody to report to - +/// but the failure must still be countable. +pub const METRIC_ACCEPTED_DELIVERY_FAILURES_TOTAL: &str = + "fitz_notice_accepted_delivery_failures_total"; #[derive(Clone)] pub struct NoticeMetrics { diff --git a/src/domains/notice/sink.rs b/src/domains/notice/sink.rs deleted file mode 100644 index 662a06c9..00000000 --- a/src/domains/notice/sink.rs +++ /dev/null @@ -1,525 +0,0 @@ -//! Live notice domain sink for the current broker process. -//! -//! Notice subscriptions are broker-local in-memory state only. They are -//! session-scoped, cleaned up on disconnect, and are never replayed or -//! restored after broker restart. - -use crate::domains::notice::NoticeMetrics; -use crate::domains::subscription_state::RoutedSubscriptionSet; -use crate::runtime::{DeliveryError, Envelope, MailboxSink, ManagedActor, Router}; -use chrono::Utc; -use parking_lot::Mutex; -use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -mod actor_runtime; -mod delivery_worker; -mod domain_sink_impl; -mod mailbox_sink_impl; -mod model; -#[cfg(test)] -mod test_channels; -mod validation; - -use actor_runtime::{NoticeDomainActor, NoticeDomainCommand}; -use delivery_worker::{notice_delivery_worker, NoticeDeliveryJob, NOTICE_DELIVERY_HANDOFF_TIMEOUT}; -use model::{ - notice_route_realm, usize_to_u64, NoticeDeliveryTarget, NoticeDeliveryTargets, - NoticeMatchedRoutePatterns, NoticeRouteStats, NoticeRouteStatsKey, NoticeSubscription, -}; -#[cfg(test)] -use test_channels::{test_client_channel_from_protocol, test_protocol_channel_from_client}; -use validation::subscription_limit_error; - -/// Live notice pub/sub state for the current broker process. -/// -/// This core owns the authoritative in-memory subscription index used for -/// delivery and admin snapshots. State disappears on session cleanup or broker -/// restart and is never durably recovered or replayed. -struct NoticeDomainCore { - /// Actor-owned single-writer state. The mutex supports immutable facade - /// methods; production mutation remains serialized by `NoticeDomainActor`. - families: Mutex< - HashMap>, - >, - /// Actor-owned single-writer route telemetry guarded for facade reads. - route_stats: Mutex>, - next_sub_id: AtomicU64, - router: Arc, - admin_read_model: Arc, - admin_snapshot_dirty: AtomicBool, - metrics: Option, - active: AtomicBool, - /// One bounded, ordered delivery lane per route family prevents a blocked - /// subscriber from stalling unrelated families on the Notice actor. - delivery_workers: Mutex< - HashMap>, - >, -} - -pub struct NoticeDomainSink { - core: Arc, - actor: ManagedActor, -} - -impl NoticeDomainSink { - pub fn new( - router: Arc, - admin_read_model: Arc, - ) -> Self { - let core = Arc::new(NoticeDomainCore { - families: Mutex::new(HashMap::new()), - route_stats: Mutex::new(HashMap::with_capacity(64)), - next_sub_id: AtomicU64::new(1), - router, - admin_read_model, - admin_snapshot_dirty: AtomicBool::new(false), - metrics: None, - active: AtomicBool::new(true), - delivery_workers: Mutex::new(HashMap::new()), - }); - let actor = Self::spawn_actor(core.clone()); - Self { core, actor } - } - - fn spawn_actor( - core: Arc, - ) -> crate::runtime::ManagedActor { - let router = core.router.clone(); - crate::runtime::ManagedActor::spawn_fail_closed( - router, - NoticeDomainActor::route_address(), - move || NoticeDomainActor::new(core.clone()), - crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, - ) - } - - fn rebuild_actor(&mut self) { - self.actor.stop(); - self.actor = Self::spawn_actor(self.core.clone()); - } - - fn core_for_builder(&mut self) -> &mut NoticeDomainCore { - Arc::get_mut(&mut self.core).expect("Notice sink builders must run before sharing the sink") - } - - #[must_use] - pub fn with_metrics( - mut self, - collector: crate::observability::metrics::MetricsCollector, - ) -> Self { - self.actor.stop(); - self.core_for_builder().metrics = Some(NoticeMetrics::new(collector)); - self.core.refresh_metrics_gauges(); - self.rebuild_actor(); - self - } - - pub fn stop(&self) { - self.core.active.store(false, Ordering::Relaxed); - self.actor.stop(); - } - - #[cfg(test)] - #[must_use] - pub(super) fn is_active(&self) -> bool { - self.core.active.load(Ordering::Relaxed) - } - - #[cfg(test)] - #[must_use] - pub(super) fn subscription_family_count(&self) -> usize { - self.core.families.lock().len() - } - - #[cfg(test)] - #[must_use] - pub(super) fn route_stats_count(&self) -> usize { - self.core.route_stats.lock().len() - } - - #[cfg(test)] - pub(super) fn is_actor_running(&self) -> bool { - self.actor.is_running() - } - - pub(crate) fn actor_health_snapshot(&self) -> crate::runtime::ManagedActorHealthSnapshot { - self.actor.health_snapshot() - } - - #[cfg(test)] - pub(crate) fn panic_actor_for_tests(&self) { - let _ = self - .actor - .try_send_high_priority(NoticeDomainCommand::PanicForTests); - } - - #[cfg(test)] - pub(super) fn stop_actor_for_tests(&self) { - self.actor.stop(); - } - - #[cfg(test)] - pub(super) fn block_actor_for_tests( - &self, - entered: crossbeam_channel::Sender<()>, - release: crossbeam_channel::Receiver<()>, - ) { - self.actor - .try_send_high_priority(NoticeDomainCommand::BlockForTests(entered, release)) - .expect("enqueue Notice actor test block"); - } - - pub fn refresh_admin_snapshot_if_dirty(&self) { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = self - .actor - .try_send(NoticeDomainCommand::RefreshAdminSnapshotIfDirty(reply_tx)) - { - tracing::warn!( - domain = "notice", - error = %error, - "Notice admin snapshot refresh enqueue failed" - ); - return; - } - - if let Err(error) = reply_rx.recv_timeout(Duration::from_secs(1)) { - tracing::warn!( - domain = "notice", - error = %error, - "Notice admin snapshot refresh reply failed" - ); - } - } - - /// Return the actor-owned live Notice subscription count. - /// - /// # Errors - /// - /// Returns the enqueue failure or `DeliveryError::Timeout` when the live - /// actor does not reply before the bounded query deadline. - pub fn subscription_count(&self) -> Result { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = self - .actor - .try_send_high_priority(NoticeDomainCommand::ReadSubscriptionCount(reply_tx)) - { - tracing::warn!(domain = "notice", error = %error, "Notice subscription-count query enqueue failed"); - return Err(error); - } - - reply_rx - .recv_timeout(Duration::from_secs(1)) - .map_err(|_| DeliveryError::Timeout) - } - - /// Remove every Notice registration owned by one ephemeral session. - /// - /// # Errors - /// - /// Returns the enqueue failure or `DeliveryError::Timeout` when the live - /// actor does not reply before the bounded cleanup deadline. - pub fn unsubscribe_all_for_session(&self, session_id: u64) -> Result { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = - self.actor - .try_send_high_priority(NoticeDomainCommand::UnsubscribeAllForSession( - session_id, reply_tx, - )) - { - tracing::warn!( - domain = "notice", - error = %error, - "Notice session cleanup command enqueue failed" - ); - return Err(error); - } - - reply_rx - .recv_timeout(Duration::from_secs(1)) - .map_err(|_| DeliveryError::Timeout) - } - - fn deliver_to_actor( - &self, - envelope: Envelope, - high_priority: bool, - ) -> Result<(), DeliveryError> { - if Self::can_accept_without_reply(&envelope) { - let command = NoticeDomainCommand::DeliverAccepted(envelope); - return if high_priority { - self.actor.try_send_high_priority(command) - } else { - self.actor.try_send(command) - }; - } - - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - let command = NoticeDomainCommand::Deliver(envelope, reply_tx); - let enqueue_result = if high_priority { - self.actor.try_send_high_priority(command) - } else { - self.actor.try_send(command) - }; - enqueue_result?; - - reply_rx - .recv_timeout(Duration::from_secs(1)) - .unwrap_or(Err(DeliveryError::Timeout)) - } - - fn can_accept_without_reply(envelope: &Envelope) -> bool { - if envelope - .payload::() - .is_some() - { - return true; - } - - envelope - .payload::() - .is_some_and(|request| { - let Ok(crate::domains::notice::protocol::NotificationMessage::Publish(publish)) = - &request.message - else { - return false; - }; - request.meta.route_family == *envelope.destination().family() - && envelope - .source() - .is_none_or(|source| *source.family() == request.meta.route_family) - && publish.family_id == request.meta.route_family - }) - } -} - -impl NoticeDomainCore { - /// Rebuild the admin read model from the current in-memory subscription - /// state only. - fn sync_admin_snapshot(&self) { - let families = self.families.lock(); - let now = Instant::now(); - let created_at = Utc::now().to_rfc3339(); - let mut subscriptions = Vec::new(); - let mut routes: HashMap = HashMap::new(); - for (route_family, state) in families.iter() { - for subscription in state.values() { - let pattern = subscription.pattern.route().to_string(); - if let Some(realm) = notice_route_realm(&pattern) { - subscriptions.push(crate::control::admin::NoticeSubscription::snapshot( - route_family.as_u64(), - subscription.subscription_id, - subscription.session_id, - realm, - pattern.clone(), - &created_at, - )); - let subscribers = routes - .entry((*route_family, Arc::clone(&subscription.pattern_route))) - .or_insert(0); - *subscribers = subscribers.saturating_add(1); - } - } - } - drop(families); - let mut route_stats = self.route_stats.lock(); - route_stats.retain(|route, stats| { - let keep = routes.contains_key(route); - if keep { - stats.prune_recent_publishes(now); - } - keep - }); - self.admin_read_model - .replace_notice_subscriptions(subscriptions); - self.admin_read_model.replace_notice_routes( - routes - .into_iter() - .map(|((route_family, route), subscribers)| { - let (publishes_total, publishes_per_minute) = route_stats - .get_mut(&(route_family, Arc::clone(&route))) - .map_or((0, 0.0), |stats| { - (stats.publishes_total(), stats.publishes_per_minute(now)) - }); - let mut entry = crate::control::admin::NoticeRouteInfo::snapshot( - route_family.as_u64(), - route.to_string(), - subscribers, - ); - entry.publishes_total = publishes_total; - entry.publishes_per_minute = publishes_per_minute; - entry - }) - .collect(), - ); - if let Some(metrics) = &self.metrics { - metrics.set_subscription_count(self.subscription_count()); - } - } - - fn mark_admin_snapshot_dirty(&self) { - self.admin_snapshot_dirty.store(true, Ordering::Relaxed); - self.refresh_metrics_gauges(); - } - - fn refresh_metrics_gauges(&self) { - if let Some(metrics) = &self.metrics { - metrics.set_subscription_count(self.subscription_count()); - } - } - - fn counter_add(&self, name: &str, amount: u64) { - if let Some(metrics) = &self.metrics { - metrics.counter_add(name, amount); - } else { - crate::observability::counter_add(name, amount); - } - } - - pub(super) fn refresh_admin_snapshot_if_dirty(&self) { - if self.admin_snapshot_dirty.swap(false, Ordering::AcqRel) { - self.sync_admin_snapshot(); - } - } - - fn fan_out_notice_event( - &self, - targets: &NoticeDeliveryTargets, - route: &crate::runtime::routing::Route, - payload: &bytes::Bytes, - ) { - for target in targets { - self.route_notice_notify(target, route, payload); - } - } - - fn record_route_publishes( - &self, - route_family: crate::runtime::routing::RouteFamily, - routes: &[Arc], - ) { - if routes.is_empty() { - return; - } - - let now = Instant::now(); - let mut route_stats = self.route_stats.lock(); - for route in routes { - route_stats - .entry((route_family, Arc::clone(route))) - .or_insert_with(NoticeRouteStats::new) - .record_publish(now); - } - } - - fn route_notice_notify( - &self, - target: &NoticeDeliveryTarget, - route: &crate::runtime::routing::Route, - payload: &bytes::Bytes, - ) { - let family = *target.subscriber.family(); - let worker = notice_delivery_worker(&self.delivery_workers, &self.router, family); - let Some(worker) = worker else { - crate::observability::counter_inc( - crate::domains::notice::metrics::METRIC_DELIVERY_DROPS_TOTAL, - ); - return; - }; - let (completed_tx, completed_rx) = crossbeam_channel::bounded(1); - let job = - NoticeDeliveryJob::new(target.clone(), route.clone(), payload.clone(), completed_tx); - if worker.try_send(job).is_err() { - crate::observability::counter_inc( - crate::domains::notice::metrics::METRIC_DELIVERY_DROPS_TOTAL, - ); - return; - } - let _ = completed_rx.recv_timeout(NOTICE_DELIVERY_HANDOFF_TIMEOUT); - } - - fn collect_matching_targets_for_route( - &self, - family_id: crate::runtime::routing::RouteFamily, - route: &str, - ) -> NoticeDeliveryTargets { - let families = self.families.lock(); - let Some(state) = families.get(&family_id) else { - return NoticeDeliveryTargets::new(); - }; - - let mut targets = NoticeDeliveryTargets::with_capacity(state.matching_capacity_hint(route)); - let mut matching_routes = NoticeMatchedRoutePatterns::new(); - state.for_each_matching_route(family_id, route, |subscription| { - targets.push(NoticeDeliveryTarget::from(subscription)); - let pattern_route = subscription.pattern_route.as_ref(); - if !matching_routes - .iter() - .any(|route| route.as_ref() == pattern_route) - { - matching_routes.push(Arc::clone(&subscription.pattern_route)); - } - }); - self.record_route_publishes(family_id, &matching_routes); - targets - } - - fn publish_route_payload( - &self, - family_id: crate::runtime::routing::RouteFamily, - route: &crate::runtime::routing::Route, - payload: &bytes::Bytes, - ) { - let targets = self.collect_matching_targets_for_route(family_id, route.as_str()); - if targets.is_empty() { - return; - } - - self.fan_out_notice_event(&targets, route, payload); - self.mark_admin_snapshot_dirty(); - } - - fn publish_event(&self, event: &crate::runtime::DomainPublishEvent) { - self.publish_route_payload(event.family_id, &event.route, &event.payload); - } - - fn handle_domain_publish(&self, event: &crate::runtime::DomainPublishEvent) { - self.publish_event(event); - } - - pub(super) fn unsubscribe_all_for_session(&self, session_id: u64) -> usize { - let mut families = self.families.lock(); - let mut removed = 0; - for (family_id, state) in families.iter_mut() { - removed += state.remove_session(*family_id, session_id); - } - families.retain(|_, state| !state.is_empty()); - tracing::debug!( - domain = "notice", - session = session_id, - "All notice subscriptions removed for session (disconnect cleanup)" - ); - drop(families); - if removed > 0 { - self.counter_add("fitz_notice_unsubscribes_total", usize_to_u64(removed)); - self.mark_admin_snapshot_dirty(); - } - removed - } - - pub(super) fn subscription_count(&self) -> usize { - let families = self.families.lock(); - families - .values() - .map(RoutedSubscriptionSet::subscription_count) - .sum() - } -} - -#[cfg(test)] -mod tests; -#[cfg(test)] -use crate::dispatch::protocol::frame_context::FrameContext; diff --git a/src/domains/notice/sink/actor_runtime.rs b/src/domains/notice/sink/actor_runtime.rs index 84928b0a..f2324455 100644 --- a/src/domains/notice/sink/actor_runtime.rs +++ b/src/domains/notice/sink/actor_runtime.rs @@ -53,7 +53,20 @@ impl Actor for NoticeDomainActor { let _ = reply.send(runtime.deliver_envelope(&envelope)); } NoticeDomainCommand::DeliverAccepted(envelope) => { - let _ = runtime.deliver_envelope(&envelope); + // The publish was already acknowledged, so there is nobody left + // to report to - but a failure here must still be countable, + // otherwise an accepted publish that never delivered looks + // identical to one that did. + if let Err(error) = runtime.deliver_envelope(&envelope) { + crate::observability::counter_inc( + crate::domains::notice::metrics::METRIC_ACCEPTED_DELIVERY_FAILURES_TOTAL, + ); + tracing::warn!( + domain = "notice", + error = ?error, + "Accepted notice delivery failed after acknowledgement" + ); + } } NoticeDomainCommand::ReadSubscriptionCount(reply) => { let _ = reply.send(runtime.subscription_count()); diff --git a/src/domains/notice/sink/admin_projection.rs b/src/domains/notice/sink/admin_projection.rs new file mode 100644 index 00000000..3b6cec29 --- /dev/null +++ b/src/domains/notice/sink/admin_projection.rs @@ -0,0 +1,97 @@ +//! Admin read-model projection: when and how the Notice subscription index +//! is mirrored into the admin snapshot. +//! +//! Projection failure must never affect domain correctness - it is a +//! dirty-flagged, best-effort reflection of live subscription state, not a +//! source of truth. + +use super::NoticeDomainCore; +use chrono::Utc; +use std::collections::HashMap; +use std::sync::atomic::Ordering; +use std::time::Instant; + +impl NoticeDomainCore { + /// Rebuild the admin read model from the current in-memory subscription + /// state only. + fn sync_admin_snapshot(&self) { + let families = self.families.lock(); + let now = Instant::now(); + let created_at = Utc::now().to_rfc3339(); + let mut subscriptions = Vec::new(); + let mut routes: HashMap = HashMap::new(); + for (route_family, state) in families.iter() { + for subscription in state.values() { + let pattern = subscription.pattern.route().to_string(); + if let Some(realm) = super::notice_route_realm(&pattern) { + subscriptions.push(crate::control::admin::NoticeSubscription::snapshot( + route_family.as_u64(), + subscription.subscription_id, + subscription.session_id, + realm, + pattern.clone(), + &created_at, + )); + let subscribers = routes + .entry(( + *route_family, + std::sync::Arc::clone(&subscription.pattern_route), + )) + .or_insert(0); + *subscribers = subscribers.saturating_add(1); + } + } + } + drop(families); + let mut route_stats = self.route_stats.lock(); + route_stats.retain(|route, stats| { + let keep = routes.contains_key(route); + if keep { + stats.prune_recent_publishes(now); + } + keep + }); + self.admin_read_model + .replace_notice_subscriptions(subscriptions); + self.admin_read_model.replace_notice_routes( + routes + .into_iter() + .map(|((route_family, route), subscribers)| { + let (publishes_total, publishes_per_minute) = route_stats + .get_mut(&(route_family, std::sync::Arc::clone(&route))) + .map_or((0, 0.0), |stats| { + (stats.publishes_total(), stats.publishes_per_minute(now)) + }); + let mut entry = crate::control::admin::NoticeRouteInfo::snapshot( + route_family.as_u64(), + route.to_string(), + subscribers, + ); + entry.publishes_total = publishes_total; + entry.publishes_per_minute = publishes_per_minute; + entry + }) + .collect(), + ); + if let Some(metrics) = &self.metrics { + metrics.set_subscription_count(self.subscription_count()); + } + } + + pub(super) fn mark_admin_snapshot_dirty(&self) { + self.admin_snapshot_dirty.store(true, Ordering::Relaxed); + self.refresh_metrics_gauges(); + } + + pub(super) fn refresh_metrics_gauges(&self) { + if let Some(metrics) = &self.metrics { + metrics.set_subscription_count(self.subscription_count()); + } + } + + pub(super) fn refresh_admin_snapshot_if_dirty(&self) { + if self.admin_snapshot_dirty.swap(false, Ordering::AcqRel) { + self.sync_admin_snapshot(); + } + } +} diff --git a/src/domains/notice/sink/cleanup.rs b/src/domains/notice/sink/cleanup.rs new file mode 100644 index 00000000..8aecd6f5 --- /dev/null +++ b/src/domains/notice/sink/cleanup.rs @@ -0,0 +1,53 @@ +//! Disconnect cleanup and stale queued-request rejection state. +//! +//! `SessionCleanup` is delivered on the high-priority mailbox lane, so it can +//! pass an older, already-queued normal-lane request from the same session. +//! Remembering the cleaned-up session lets that stale request fail instead of +//! silently recreating a subscription for a session that is already gone and +//! will never be cleaned up again. + +use super::{model::usize_to_u64, Envelope, NoticeDomainCore}; + +impl NoticeDomainCore { + pub(super) fn is_cleaned_up_session(&self, session_id: u64) -> bool { + self.cleaned_up_sessions.lock().contains(session_id) + } + + pub(super) fn handle_cleanup_envelope(&self, envelope: &Envelope) -> bool { + if let Some(cleanup) = envelope.payload::() { + // Mark first so an older normal-lane request that cleanup jumped + // over cannot recreate a subscription for this session below. + self.cleaned_up_sessions.lock().mark(cleanup.session_id); + self.unsubscribe_all_for_session(cleanup.session_id); + return true; + } + + false + } + + /// Remove every Notice subscription owned by one session. + /// + /// Shared by disconnect cleanup (`handle_cleanup_envelope`, which marks + /// the session cleaned-up first) and the client-initiated + /// `UnsubscribeAll` request (which does not - a still-connected client is + /// free to subscribe again afterward). + pub(super) fn unsubscribe_all_for_session(&self, session_id: u64) -> usize { + let mut families = self.families.lock(); + let mut removed = 0; + for (family_id, state) in families.iter_mut() { + removed += state.remove_session(*family_id, session_id); + } + families.retain(|_, state| !state.is_empty()); + tracing::debug!( + domain = "notice", + session = session_id, + "All notice subscriptions removed for session (disconnect cleanup)" + ); + drop(families); + if removed > 0 { + self.counter_add("fitz_notice_unsubscribes_total", usize_to_u64(removed)); + self.mark_admin_snapshot_dirty(); + } + removed + } +} diff --git a/src/domains/notice/sink/domain_sink_impl.rs b/src/domains/notice/sink/domain_sink_impl.rs deleted file mode 100644 index ff49db9c..00000000 --- a/src/domains/notice/sink/domain_sink_impl.rs +++ /dev/null @@ -1,472 +0,0 @@ -use super::{ - subscription_limit_error, Arc, DeliveryError, Envelope, Instant, NoticeDomainCore, - NoticeMetrics, NoticeSubscription, Ordering, RoutedSubscriptionSet, -}; -#[cfg(test)] -use super::{test_client_channel_from_protocol, test_protocol_channel_from_client, FrameContext}; - -impl NoticeDomainCore { - pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { - if self.handle_cleanup_envelope(envelope) { - return Ok(()); - } - self.ensure_active()?; - - if self.handle_domain_publish_envelope(envelope) { - return Ok(()); - } - - Self::log_delivery(envelope); - - let Some(request) = Self::extract_request(envelope)? else { - return Ok(()); - }; - let meta = request.meta; - let request_started = self.record_request_start(); - - if !Self::valid_request_envelope(envelope, meta) { - self.reject_with(envelope, meta, "route family mismatch", request_started); - return Ok(()); - } - - Self::log_parse_start(meta); - - let Some(notice_msg) = - self.parse_notice_message(envelope, meta, request.message, request_started) - else { - return Ok(()); - }; - - if !Self::valid_notice_message(envelope, meta, ¬ice_msg) { - self.reject_with(envelope, meta, "route family mismatch", request_started); - return Ok(()); - } - - let (response_opt, should_sync_admin_snapshot) = self.dispatch_notice_message(notice_msg); - if should_sync_admin_snapshot { - self.mark_admin_snapshot_dirty(); - } - - if let Some(response) = response_opt { - self.route_notice_response(envelope, meta, &response, request_started); - } else if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { - metrics.record_success(started_at); - } - - Ok(()) - } - - fn handle_cleanup_envelope(&self, envelope: &Envelope) -> bool { - if let Some(cleanup) = envelope.payload::() { - self.unsubscribe_all_for_session(cleanup.session_id); - return true; - } - - false - } - - fn ensure_active(&self) -> Result<(), DeliveryError> { - if !self.active.load(Ordering::Relaxed) { - return Err(DeliveryError::ActorStopped); - } - - Ok(()) - } - - fn handle_domain_publish_envelope(&self, envelope: &Envelope) -> bool { - if let Some(event) = envelope.payload::() { - if *envelope.destination().family() != event.family_id { - self.counter_add("fitz_notice_publish_family_mismatch_total", 1); - return true; - } - self.handle_domain_publish(event); - return true; - } - - false - } - - fn log_delivery(envelope: &Envelope) { - tracing::debug!( - domain = "notice", - destination = %envelope.destination(), - source = ?envelope.source(), - "Notice domain sink: received envelope" - ); - } - - fn extract_request( - envelope: &Envelope, - ) -> Result, DeliveryError> { - if let Some(request) = Self::request_from_envelope(envelope) { - Ok(Some(request)) - } else { - tracing::warn!( - domain = "notice", - "Envelope payload was not NoticeClientRequest" - ); - Err(DeliveryError::ActorStopped) - } - } - - fn record_request_start(&self) -> Option { - self.metrics - .as_ref() - .map(NoticeMetrics::record_request_start) - } - - fn reject_with( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - reason: &str, - request_started: Option, - ) { - let response = Self::error_response(reason); - let response_meta = Self::response_meta_for_source(envelope, meta); - self.route_notice_response(envelope, response_meta, &response, request_started); - } - - fn log_parse_start(meta: crate::runtime::ClientFrameMeta) { - tracing::debug!( - domain = "notice", - session = meta.session_id, - msg_type = meta.message_type, - "Notice: parsing request" - ); - } - - fn parse_notice_message( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - message: Result, - request_started: Option, - ) -> Option { - match message { - Ok(message) => Some(message), - Err(error) => { - tracing::warn!(domain = "notice", error = %error, "Failed to parse notice message"); - self.reject_with(envelope, meta, &error, request_started); - None - } - } - } - - fn dispatch_notice_message( - &self, - notice_msg: crate::domains::notice::protocol::NotificationMessage, - ) -> (Option, bool) { - use crate::domains::notice::protocol::NotificationMessage; - use crate::domains::notice::NoticeResponse; - - match notice_msg { - NotificationMessage::Publish(pub_msg) => { - self.publish_route_payload(pub_msg.family_id, &pub_msg.route, &pub_msg.payload); - (None, false) - } - NotificationMessage::Subscribe(sub_msg) => self.handle_subscribe_message(&sub_msg), - NotificationMessage::Unsubscribe(unsub_msg) => { - let family_id = unsub_msg.family_id; - let mut families = self.families.lock(); - let removed = if let Some(state) = families.get_mut(&family_id) { - let removed = state.remove_subscription_for_session( - unsub_msg.family_id, - unsub_msg.session_id.0, - unsub_msg.subscription_id, - ); - if state.is_empty() { - families.remove(&family_id); - } - removed - } else { - false - }; - if removed { - self.counter_add("fitz_notice_unsubscribes_total", 1); - } - (Some(NoticeResponse::Ok), removed) - } - NotificationMessage::UnsubscribeAll(unsub_all) => { - let session_id = unsub_all.session_id.0; - let removed = self.unsubscribe_all_for_session(session_id); - tracing::debug!( - domain = "notice", - session = session_id, - "All subscriptions removed for session" - ); - (Some(NoticeResponse::Ok), removed > 0) - } - NotificationMessage::Deliver(_) => (Some(NoticeResponse::Ok), false), - } - } - - fn handle_subscribe_message( - &self, - sub_msg: &crate::domains::notice::protocol::SubscribeMessage, - ) -> (Option, bool) { - if let Some(response) = self.try_reuse_existing(sub_msg) { - return (Some(response), false); - } - let compiled = match Self::compile_pattern(sub_msg) { - Ok(compiled) => compiled, - Err(response) => return (Some(response), false), - }; - let (response, state_changed) = self.allocate_and_insert(sub_msg, compiled); - (Some(response), state_changed) - } - - fn compile_pattern( - sub_msg: &crate::domains::notice::protocol::SubscribeMessage, - ) -> Result { - crate::runtime::DomainKind::Notice - .descriptor() - .compile_registration_pattern(sub_msg.pattern.as_str()) - .map_err(|error| { - tracing::warn!( - domain = "notice", - session = sub_msg.session_id.0, - "Rejected invalid subscription pattern" - ); - crate::domains::notice::NoticeResponse::Error(error) - }) - } - - fn try_reuse_existing( - &self, - sub_msg: &crate::domains::notice::protocol::SubscribeMessage, - ) -> Option { - let families = self.families.lock(); - let id = families.get(&sub_msg.family_id).and_then(|state| { - state.find_existing_id(sub_msg.session_id.0, sub_msg.pattern.as_str()) - })?; - tracing::debug!( - domain = "notice", - session = sub_msg.session_id.0, - subscription_id = id, - pattern = sub_msg.pattern.as_str(), - "Notice subscription already exists (idempotent)" - ); - Some(crate::domains::notice::NoticeResponse::SubscribeOk { - subscription_id: id, - }) - } - - fn allocate_and_insert( - &self, - sub_msg: &crate::domains::notice::protocol::SubscribeMessage, - compiled: crate::runtime::matcher::Pattern, - ) -> (crate::domains::notice::NoticeResponse, bool) { - use crate::domains::notice::NoticeResponse; - - let mut families = self.families.lock(); - let session_subscription_count = families - .values() - .map(|state| state.subscription_count_for_session(sub_msg.session_id.0)) - .sum::(); - let state = families - .entry(sub_msg.family_id) - .or_insert_with(RoutedSubscriptionSet::new); - - let (response, state_changed) = if let Some(error) = - subscription_limit_error(state, session_subscription_count, sub_msg, &compiled) - { - (error, false) - } else { - let Ok(new_id) = - self.next_sub_id - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { - current.checked_add(1) - }) - else { - let state_empty = state.is_empty(); - if state_empty { - families.remove(&sub_msg.family_id); - } - return ( - NoticeResponse::Error("subscription ID space exhausted".to_string()), - false, - ); - }; - state.insert( - sub_msg.family_id, - NoticeSubscription { - pattern: compiled, - pattern_route: Arc::from(sub_msg.pattern.as_str()), - session_id: sub_msg.session_id.0, - subscription_id: new_id, - subscriber: sub_msg.subscriber.clone(), - }, - ); - - tracing::debug!( - domain = "notice", - session = sub_msg.session_id.0, - subscription_id = new_id, - pattern = sub_msg.pattern.as_str(), - "Notice subscription added" - ); - ( - NoticeResponse::SubscribeOk { - subscription_id: new_id, - }, - true, - ) - }; - - (response, state_changed) - } - - fn valid_request_envelope(envelope: &Envelope, meta: crate::runtime::ClientFrameMeta) -> bool { - meta.route_family == *envelope.destination().family() - && envelope - .source() - .is_none_or(|source| *source.family() == meta.route_family) - } - - fn valid_notice_message( - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - message: &crate::domains::notice::protocol::NotificationMessage, - ) -> bool { - use crate::domains::notice::protocol::NotificationMessage; - - match message { - NotificationMessage::Publish(publish) => publish.family_id == meta.route_family, - NotificationMessage::Subscribe(subscribe) => { - subscribe.family_id == meta.route_family - && subscribe.session_id.0 == meta.session_id - && *subscribe.subscriber.family() == subscribe.family_id - && envelope - .source() - .is_none_or(|source| source == &subscribe.subscriber) - } - NotificationMessage::Unsubscribe(unsubscribe) => { - unsubscribe.family_id == meta.route_family - && unsubscribe.session_id.0 == meta.session_id - } - NotificationMessage::UnsubscribeAll(unsubscribe_all) => { - unsubscribe_all.session_id.0 == meta.session_id - && *unsubscribe_all.subscriber.family() == meta.route_family - && envelope - .source() - .is_none_or(|source| source == &unsubscribe_all.subscriber) - } - NotificationMessage::Deliver(_) => false, - } - } - - fn error_response(reason: &str) -> crate::domains::notice::NoticeResponse { - crate::domains::notice::NoticeResponse::Error(reason.to_string()) - } - - fn response_meta_for_source( - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - ) -> crate::runtime::ClientFrameMeta { - envelope.source().map_or(meta, |source| { - let mut response_meta = meta; - response_meta.route_family = *source.family(); - response_meta - }) - } - - fn route_notice_response( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - response: &crate::domains::notice::NoticeResponse, - request_started: Option, - ) { - #[cfg(test)] - let response_ctx = { - let mut payload_encoder = - crate::dispatch::protocol::payload_codec::PayloadEncoder::with_capacity(256); - let response_bytes = crate::dispatch::protocol::notice_codec::encode_response_into( - response, - &mut payload_encoder, - ); - FrameContext::new( - meta.session_id, - test_protocol_channel_from_client(meta.channel), - crate::dispatch::protocol::tlv::MessageType::new(meta.message_type), - bytes::Bytes::from(response_bytes), - meta.route_family, - ) - }; - - #[cfg(not(test))] - let response_ctx = - crate::domains::notice::NoticeClientResponse::new(meta, response.clone()); - - if let Some(response_envelope) = envelope.try_reply_to(response_ctx) { - if let Err(error) = self.router.route(response_envelope) { - if let Some(metrics) = self.metrics.as_ref() { - metrics.record_response_drop(); - } else { - crate::observability::counter_inc( - crate::domains::notice::metrics::METRIC_RESPONSE_DROPS_TOTAL, - ); - } - tracing::warn!( - domain = "notice", - session_id = meta.session_id, - route_family = meta.route_family.as_u64(), - error = %error, - "Dropped best-effort Notice response" - ); - } - } - - if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { - if response.is_failure() { - metrics.record_failure(started_at); - } else { - metrics.record_success(started_at); - } - } - } - - fn request_from_envelope( - envelope: &Envelope, - ) -> Option { - if let Some(request) = envelope.payload::() { - return Some(request.clone()); - } - - #[cfg(test)] - { - let frame_ctx = envelope.payload::()?.clone(); - let subscriber = envelope.source().cloned().unwrap_or_else(|| { - crate::runtime::routing::RouteAddress::new( - *envelope.destination().family(), - crate::runtime::routing::Route::new(format!( - "inbox://session/{}", - frame_ctx.session_id - )), - ) - }); - let meta = crate::runtime::ClientFrameMeta::new( - frame_ctx.session_id, - test_client_channel_from_protocol(frame_ctx.channel_id), - frame_ctx.msg_type.as_u16(), - frame_ctx.route_family, - ); - let parsed = crate::dispatch::protocol::notice_codec::parse_request( - &frame_ctx, - &frame_ctx.payload, - *envelope.destination().family(), - crate::session::SessionId(frame_ctx.session_id), - subscriber, - ); - Some(crate::domains::notice::NoticeClientRequest::new( - meta, parsed, - )) - } - - #[cfg(not(test))] - { - None - } - } -} diff --git a/src/domains/notice/sink/facade.rs b/src/domains/notice/sink/facade.rs new file mode 100644 index 00000000..c395fb2c --- /dev/null +++ b/src/domains/notice/sink/facade.rs @@ -0,0 +1,244 @@ +//! Public `NoticeDomainSink` API and actor lifecycle management. + +use super::{ + DeliveryError, Envelope, NoticeDomainActor, NoticeDomainCommand, NoticeDomainCore, + NoticeDomainSink, NoticeMetrics, +}; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::Duration; + +impl NoticeDomainSink { + pub fn new( + router: Arc, + admin_read_model: Arc, + ) -> Self { + let core = Arc::new(NoticeDomainCore { + families: parking_lot::Mutex::new(std::collections::HashMap::new()), + route_stats: parking_lot::Mutex::new(std::collections::HashMap::with_capacity(64)), + next_sub_id: std::sync::atomic::AtomicU64::new(1), + router, + admin_read_model, + admin_snapshot_dirty: std::sync::atomic::AtomicBool::new(false), + metrics: None, + active: std::sync::atomic::AtomicBool::new(true), + delivery_workers: parking_lot::Mutex::new(std::collections::HashMap::new()), + cleaned_up_sessions: parking_lot::Mutex::new(crate::runtime::CleanedUpSessions::new( + crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, + )), + }); + let actor = Self::spawn_actor(core.clone()); + Self { core, actor } + } + + fn spawn_actor( + core: Arc, + ) -> crate::runtime::ManagedActor { + let router = core.router.clone(); + crate::runtime::ManagedActor::spawn_fail_closed( + router, + NoticeDomainActor::route_address(), + move || NoticeDomainActor::new(core.clone()), + crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, + ) + } + + fn rebuild_actor(&mut self) { + self.actor.stop(); + self.actor = Self::spawn_actor(self.core.clone()); + } + + fn core_for_builder(&mut self) -> &mut NoticeDomainCore { + Arc::get_mut(&mut self.core).expect("Notice sink builders must run before sharing the sink") + } + + #[must_use] + pub fn with_metrics( + mut self, + collector: crate::observability::metrics::MetricsCollector, + ) -> Self { + self.actor.stop(); + self.core_for_builder().metrics = Some(NoticeMetrics::new(collector)); + self.core.refresh_metrics_gauges(); + self.rebuild_actor(); + self + } + + pub fn stop(&self) { + self.core.active.store(false, Ordering::Relaxed); + self.actor.stop(); + } + + #[cfg(test)] + #[must_use] + pub(super) fn is_active(&self) -> bool { + self.core.active.load(Ordering::Relaxed) + } + + #[cfg(test)] + #[must_use] + pub(super) fn subscription_family_count(&self) -> usize { + self.core.families.lock().len() + } + + #[cfg(test)] + #[must_use] + pub(super) fn route_stats_count(&self) -> usize { + self.core.route_stats.lock().len() + } + + #[cfg(test)] + pub(super) fn is_actor_running(&self) -> bool { + self.actor.is_running() + } + + pub(crate) fn actor_health_snapshot(&self) -> crate::runtime::ManagedActorHealthSnapshot { + self.actor.health_snapshot() + } + + #[cfg(test)] + pub(crate) fn panic_actor_for_tests(&self) { + let _ = self + .actor + .try_send_high_priority(NoticeDomainCommand::PanicForTests); + } + + #[cfg(test)] + pub(super) fn stop_actor_for_tests(&self) { + self.actor.stop(); + } + + #[cfg(test)] + pub(super) fn block_actor_for_tests( + &self, + entered: crossbeam_channel::Sender<()>, + release: crossbeam_channel::Receiver<()>, + ) { + self.actor + .try_send_high_priority(NoticeDomainCommand::BlockForTests(entered, release)) + .expect("enqueue Notice actor test block"); + } + + pub fn refresh_admin_snapshot_if_dirty(&self) { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = self + .actor + .try_send(NoticeDomainCommand::RefreshAdminSnapshotIfDirty(reply_tx)) + { + tracing::warn!( + domain = "notice", + error = %error, + "Notice admin snapshot refresh enqueue failed" + ); + return; + } + + if let Err(error) = reply_rx.recv_timeout(Duration::from_secs(1)) { + tracing::warn!( + domain = "notice", + error = %error, + "Notice admin snapshot refresh reply failed" + ); + } + } + + /// Return the actor-owned live Notice subscription count. + /// + /// # Errors + /// + /// Returns the enqueue failure or `DeliveryError::Timeout` when the live + /// actor does not reply before the bounded query deadline. + pub fn subscription_count(&self) -> Result { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = self + .actor + .try_send_high_priority(NoticeDomainCommand::ReadSubscriptionCount(reply_tx)) + { + tracing::warn!(domain = "notice", error = %error, "Notice subscription-count query enqueue failed"); + return Err(error); + } + + reply_rx + .recv_timeout(Duration::from_secs(1)) + .map_err(|_| DeliveryError::Timeout) + } + + /// Remove every Notice registration owned by one ephemeral session. + /// + /// # Errors + /// + /// Returns the enqueue failure or `DeliveryError::Timeout` when the live + /// actor does not reply before the bounded cleanup deadline. + pub fn unsubscribe_all_for_session(&self, session_id: u64) -> Result { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = + self.actor + .try_send_high_priority(NoticeDomainCommand::UnsubscribeAllForSession( + session_id, reply_tx, + )) + { + tracing::warn!( + domain = "notice", + error = %error, + "Notice session cleanup command enqueue failed" + ); + return Err(error); + } + + reply_rx + .recv_timeout(Duration::from_secs(1)) + .map_err(|_| DeliveryError::Timeout) + } + + pub(super) fn deliver_to_actor( + &self, + envelope: Envelope, + high_priority: bool, + ) -> Result<(), DeliveryError> { + if Self::can_accept_without_reply(&envelope) { + let command = NoticeDomainCommand::DeliverAccepted(envelope); + return if high_priority { + self.actor.try_send_high_priority(command) + } else { + self.actor.try_send(command) + }; + } + + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + let command = NoticeDomainCommand::Deliver(envelope, reply_tx); + let enqueue_result = if high_priority { + self.actor.try_send_high_priority(command) + } else { + self.actor.try_send(command) + }; + enqueue_result?; + + reply_rx + .recv_timeout(Duration::from_secs(1)) + .unwrap_or(Err(DeliveryError::Timeout)) + } + + fn can_accept_without_reply(envelope: &Envelope) -> bool { + if envelope + .payload::() + .is_some() + { + return true; + } + + envelope + .payload::() + .is_some_and(|request| { + let Ok(crate::domains::notice::protocol::NotificationMessage::Publish(publish)) = + &request.message + else { + return false; + }; + request.meta.route_family == *envelope.destination().family() + && envelope + .source() + .is_none_or(|source| *source.family() == request.meta.route_family) + && publish.family_id == request.meta.route_family + }) + } +} diff --git a/src/domains/notice/sink/ingress.rs b/src/domains/notice/sink/ingress.rs new file mode 100644 index 00000000..13b07a67 --- /dev/null +++ b/src/domains/notice/sink/ingress.rs @@ -0,0 +1,223 @@ +//! Envelope ingress: validate an inbound envelope, parse it into a Notice +//! request, and dispatch to the subscription/publish/response layers. + +#[cfg(test)] +use super::{test_client_channel_from_protocol, FrameContext}; +use super::{Envelope, NoticeDomainCore, NoticeMetrics}; +use crate::runtime::DeliveryError; +use std::time::Instant; + +impl NoticeDomainCore { + pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { + if self.handle_cleanup_envelope(envelope) { + return Ok(()); + } + self.ensure_active()?; + + if self.handle_domain_publish_envelope(envelope) { + return Ok(()); + } + + Self::log_delivery(envelope); + + let Some(request) = Self::extract_request(envelope)? else { + return Ok(()); + }; + let meta = request.meta; + let request_started = self.record_request_start(); + + if !Self::valid_request_envelope(envelope, meta) { + self.reject_with(envelope, meta, "route family mismatch", request_started); + return Ok(()); + } + + // This request was already queued (on the normal lane) before this + // session's disconnect cleanup ran (on the high-priority lane) and + // jumped ahead of it. Reject rather than silently recreating a + // subscription for a session that is already gone and will never be + // cleaned up again. + if self.is_cleaned_up_session(meta.session_id) { + self.reject_with(envelope, meta, "session already closed", request_started); + return Ok(()); + } + + Self::log_parse_start(meta); + + let Some(notice_msg) = + self.parse_notice_message(envelope, meta, request.message, request_started) + else { + return Ok(()); + }; + + if !Self::valid_notice_message(envelope, meta, ¬ice_msg) { + self.reject_with(envelope, meta, "route family mismatch", request_started); + return Ok(()); + } + + let (response_opt, should_sync_admin_snapshot) = self.dispatch_notice_message(notice_msg); + if should_sync_admin_snapshot { + self.mark_admin_snapshot_dirty(); + } + + if let Some(response) = response_opt { + self.route_notice_response(envelope, meta, &response, request_started); + } else if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { + metrics.record_success(started_at); + } + + Ok(()) + } + + fn ensure_active(&self) -> Result<(), DeliveryError> { + crate::runtime::ingress_support::ensure_actor_active(&self.active) + } + + fn handle_domain_publish_envelope(&self, envelope: &Envelope) -> bool { + if let Some(event) = envelope.payload::() { + if *envelope.destination().family() != event.family_id { + self.counter_add("fitz_notice_publish_family_mismatch_total", 1); + return true; + } + self.handle_domain_publish(event); + return true; + } + + false + } + + fn log_delivery(envelope: &Envelope) { + crate::runtime::ingress_support::log_envelope_received( + "notice", + "Notice domain sink: received envelope", + envelope, + ); + } + + fn extract_request( + envelope: &Envelope, + ) -> Result, DeliveryError> { + if let Some(request) = Self::request_from_envelope(envelope) { + Ok(Some(request)) + } else { + tracing::warn!( + domain = "notice", + "Envelope payload was not NoticeClientRequest" + ); + Err(DeliveryError::ActorStopped) + } + } + + fn record_request_start(&self) -> Option { + self.metrics + .as_ref() + .map(NoticeMetrics::record_request_start) + } + + fn log_parse_start(meta: crate::runtime::ClientFrameMeta) { + tracing::debug!( + domain = "notice", + session = meta.session_id, + msg_type = meta.message_type, + "Notice: parsing request" + ); + } + + fn parse_notice_message( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + message: Result, + request_started: Option, + ) -> Option { + match message { + Ok(message) => Some(message), + Err(error) => { + tracing::warn!(domain = "notice", error = %error, "Failed to parse notice message"); + self.reject_with(envelope, meta, &error, request_started); + None + } + } + } + + fn valid_request_envelope(envelope: &Envelope, meta: crate::runtime::ClientFrameMeta) -> bool { + meta.route_family == *envelope.destination().family() + && envelope + .source() + .is_none_or(|source| *source.family() == meta.route_family) + } + + fn valid_notice_message( + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + message: &crate::domains::notice::protocol::NotificationMessage, + ) -> bool { + use crate::domains::notice::protocol::NotificationMessage; + + match message { + NotificationMessage::Publish(publish) => publish.family_id == meta.route_family, + NotificationMessage::Subscribe(subscribe) => { + subscribe.family_id == meta.route_family + && subscribe.session_id.0 == meta.session_id + && *subscribe.subscriber.family() == subscribe.family_id + && envelope + .source() + .is_none_or(|source| source == &subscribe.subscriber) + } + NotificationMessage::Unsubscribe(unsubscribe) => { + unsubscribe.family_id == meta.route_family + && unsubscribe.session_id.0 == meta.session_id + } + NotificationMessage::UnsubscribeAll(unsubscribe_all) => { + unsubscribe_all.session_id.0 == meta.session_id + && *unsubscribe_all.subscriber.family() == meta.route_family + && envelope + .source() + .is_none_or(|source| source == &unsubscribe_all.subscriber) + } + NotificationMessage::Deliver(_) => false, + } + } + + fn request_from_envelope( + envelope: &Envelope, + ) -> Option { + if let Some(request) = envelope.payload::() { + return Some(request.clone()); + } + + #[cfg(test)] + { + let frame_ctx = envelope.payload::()?.clone(); + let subscriber = envelope.source().cloned().unwrap_or_else(|| { + crate::runtime::routing::RouteAddress::new( + *envelope.destination().family(), + crate::runtime::routing::Route::new(format!( + "inbox://session/{}", + frame_ctx.session_id + )), + ) + }); + let meta = crate::runtime::ClientFrameMeta::new( + frame_ctx.session_id, + test_client_channel_from_protocol(frame_ctx.channel_id), + frame_ctx.msg_type.as_u16(), + frame_ctx.route_family, + ); + let parsed = crate::dispatch::protocol::notice_codec::parse_request( + &frame_ctx, + &frame_ctx.payload, + *envelope.destination().family(), + crate::session::SessionId(frame_ctx.session_id), + subscriber, + ); + Some(crate::domains::notice::NoticeClientRequest::new( + meta, parsed, + )) + } + + #[cfg(not(test))] + { + None + } + } +} diff --git a/src/domains/notice/sink/mod.rs b/src/domains/notice/sink/mod.rs new file mode 100644 index 00000000..c4cfea5d --- /dev/null +++ b/src/domains/notice/sink/mod.rs @@ -0,0 +1,65 @@ +//! Notice domain sink module wiring. +//! +//! See `state.rs` for what's owned (broker-local, session-scoped, never +//! durable), `admin_projection.rs` for the read-model mirror, and +//! `ingress.rs`/`subscriptions.rs`/`publish.rs`/`responses.rs`/`cleanup.rs` +//! for the request lifecycle. + +use crate::domains::notice::NoticeMetrics; +use crate::domains::subscription_state::RoutedSubscriptionSet; +use crate::runtime::{DeliveryError, Envelope, MailboxSink}; +use std::time::Instant; + +mod actor_runtime; +mod admin_projection; +mod cleanup; +mod delivery_worker; +mod facade; +mod ingress; +mod mailbox_sink_impl; +mod model; +mod publish; +mod responses; +mod state; +mod subscriptions; +#[cfg(test)] +mod test_channels; +mod validation; + +use actor_runtime::{NoticeDomainActor, NoticeDomainCommand}; +use delivery_worker::{notice_delivery_worker, NoticeDeliveryJob, NOTICE_DELIVERY_HANDOFF_TIMEOUT}; +use model::{ + notice_route_realm, NoticeDeliveryTarget, NoticeDeliveryTargets, NoticeMatchedRoutePatterns, + NoticeRouteStats, NoticeRouteStatsKey, NoticeSubscription, +}; +use state::NoticeDomainCore; +use std::sync::atomic::Ordering; +use std::sync::Arc; +#[cfg(test)] +use test_channels::test_client_channel_from_protocol; +use validation::subscription_limit_error; + +pub use state::NoticeDomainSink; + +impl NoticeDomainCore { + fn counter_add(&self, name: &str, amount: u64) { + if let Some(metrics) = &self.metrics { + metrics.counter_add(name, amount); + } else { + crate::observability::counter_add(name, amount); + } + } + + pub(super) fn subscription_count(&self) -> usize { + let families = self.families.lock(); + families + .values() + .map(RoutedSubscriptionSet::subscription_count) + .sum() + } +} + +#[cfg(test)] +mod tests; +#[cfg(test)] +use crate::dispatch::protocol::frame_context::FrameContext; diff --git a/src/domains/notice/sink/model.rs b/src/domains/notice/sink/model.rs index 28f42120..e4b90e55 100644 --- a/src/domains/notice/sink/model.rs +++ b/src/domains/notice/sink/model.rs @@ -100,3 +100,57 @@ fn usize_to_f64(value: usize) -> f64 { pub(super) fn usize_to_u64(value: usize) -> u64 { u64::try_from(value).unwrap_or(u64::MAX) } + +/// Record the outcome of waiting for a delivery worker to confirm a handoff. +/// +/// A timeout is not a drop: the worker may still complete the delivery. It is +/// counted separately so an uncertain outcome is visible rather than silently +/// discarded, which is how a saturated worker could look identical to a +/// healthy one. +pub(super) fn record_delivery_handoff_outcome( + outcome: Result<(), crossbeam_channel::RecvTimeoutError>, +) { + if outcome.is_err() { + crate::observability::counter_inc( + crate::domains::notice::metrics::METRIC_DELIVERY_HANDOFF_TIMEOUTS_TOTAL, + ); + } +} + +#[cfg(test)] +mod delivery_handoff_outcome_tests { + use super::record_delivery_handoff_outcome; + use crate::domains::notice::metrics::METRIC_DELIVERY_HANDOFF_TIMEOUTS_TOTAL; + + #[test] + fn should_count_an_unconfirmed_delivery_handoff() { + // Arrange + let metrics = crate::observability::metrics(); + let before = metrics.counter_get(METRIC_DELIVERY_HANDOFF_TIMEOUTS_TOTAL); + + // Act + record_delivery_handoff_outcome(Err(crossbeam_channel::RecvTimeoutError::Timeout)); + + // Assert + assert!( + metrics.counter_get(METRIC_DELIVERY_HANDOFF_TIMEOUTS_TOTAL) > before, + "an unconfirmed handoff must be observable" + ); + } + + #[test] + fn should_not_count_a_confirmed_delivery_handoff() { + // Arrange + let metrics = crate::observability::metrics(); + let before = metrics.counter_get(METRIC_DELIVERY_HANDOFF_TIMEOUTS_TOTAL); + + // Act + record_delivery_handoff_outcome(Ok(())); + + // Assert + assert_eq!( + metrics.counter_get(METRIC_DELIVERY_HANDOFF_TIMEOUTS_TOTAL), + before + ); + } +} diff --git a/src/domains/notice/sink/publish.rs b/src/domains/notice/sink/publish.rs new file mode 100644 index 00000000..95abd075 --- /dev/null +++ b/src/domains/notice/sink/publish.rs @@ -0,0 +1,118 @@ +//! Publish fan-out: matching subscribers to a published route and handing +//! delivery off to the per-route-family delivery workers. + +use super::{ + model, notice_delivery_worker, NoticeDeliveryJob, NoticeDeliveryTarget, NoticeDeliveryTargets, + NoticeDomainCore, NoticeMatchedRoutePatterns, NOTICE_DELIVERY_HANDOFF_TIMEOUT, +}; +use std::sync::Arc; +use std::time::Instant; + +impl NoticeDomainCore { + fn fan_out_notice_event( + &self, + targets: &NoticeDeliveryTargets, + route: &crate::runtime::routing::Route, + payload: &bytes::Bytes, + ) { + for target in targets { + self.route_notice_notify(target, route, payload); + } + } + + fn record_route_publishes( + &self, + route_family: crate::runtime::routing::RouteFamily, + routes: &[Arc], + ) { + if routes.is_empty() { + return; + } + + let now = Instant::now(); + let mut route_stats = self.route_stats.lock(); + for route in routes { + route_stats + .entry((route_family, Arc::clone(route))) + .or_insert_with(super::NoticeRouteStats::new) + .record_publish(now); + } + } + + fn route_notice_notify( + &self, + target: &NoticeDeliveryTarget, + route: &crate::runtime::routing::Route, + payload: &bytes::Bytes, + ) { + let family = *target.subscriber.family(); + let worker = notice_delivery_worker(&self.delivery_workers, &self.router, family); + let Some(worker) = worker else { + crate::observability::counter_inc( + crate::domains::notice::metrics::METRIC_DELIVERY_DROPS_TOTAL, + ); + return; + }; + let (completed_tx, completed_rx) = crossbeam_channel::bounded(1); + let job = + NoticeDeliveryJob::new(target.clone(), route.clone(), payload.clone(), completed_tx); + if worker.try_send(job).is_err() { + crate::observability::counter_inc( + crate::domains::notice::metrics::METRIC_DELIVERY_DROPS_TOTAL, + ); + return; + } + model::record_delivery_handoff_outcome( + completed_rx.recv_timeout(NOTICE_DELIVERY_HANDOFF_TIMEOUT), + ); + } + + fn collect_matching_targets_for_route( + &self, + family_id: crate::runtime::routing::RouteFamily, + route: &str, + ) -> NoticeDeliveryTargets { + let families = self.families.lock(); + let Some(state) = families.get(&family_id) else { + return NoticeDeliveryTargets::new(); + }; + + let mut targets = NoticeDeliveryTargets::with_capacity(state.matching_capacity_hint(route)); + let mut matching_routes = NoticeMatchedRoutePatterns::new(); + state.for_each_matching_route(family_id, route, |subscription| { + targets.push(NoticeDeliveryTarget::from(subscription)); + let pattern_route = subscription.pattern_route.as_ref(); + if !matching_routes + .iter() + .any(|route| route.as_ref() == pattern_route) + { + matching_routes.push(Arc::clone(&subscription.pattern_route)); + } + }); + self.record_route_publishes(family_id, &matching_routes); + targets + } + + pub(super) fn publish_route_payload( + &self, + family_id: crate::runtime::routing::RouteFamily, + route: &crate::runtime::routing::Route, + payload: &bytes::Bytes, + ) { + let targets = self.collect_matching_targets_for_route(family_id, route.as_str()); + if targets.is_empty() { + return; + } + + self.fan_out_notice_event(&targets, route, payload); + self.mark_admin_snapshot_dirty(); + } + + fn publish_event(&self, event: &crate::runtime::DomainPublishEvent) { + self.publish_route_payload(event.family_id, &event.route, &event.payload); + } + + pub(super) fn handle_domain_publish(&self, event: &crate::runtime::DomainPublishEvent) { + self.publish_event(event); + } +} diff --git a/src/domains/notice/sink/responses.rs b/src/domains/notice/sink/responses.rs new file mode 100644 index 00000000..2cdc223a --- /dev/null +++ b/src/domains/notice/sink/responses.rs @@ -0,0 +1,90 @@ +//! Response encoding and best-effort routing back to the requester. + +#[cfg(test)] +use super::FrameContext; +use super::{Envelope, Instant, NoticeDomainCore}; + +impl NoticeDomainCore { + pub(super) fn reject_with( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + reason: &str, + request_started: Option, + ) { + let response = Self::error_response(reason); + let response_meta = Self::response_meta_for_source(envelope, meta); + self.route_notice_response(envelope, response_meta, &response, request_started); + } + + fn error_response(reason: &str) -> crate::domains::notice::NoticeResponse { + crate::domains::notice::NoticeResponse::Error(reason.to_string()) + } + + fn response_meta_for_source( + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + ) -> crate::runtime::ClientFrameMeta { + envelope.source().map_or(meta, |source| { + let mut response_meta = meta; + response_meta.route_family = *source.family(); + response_meta + }) + } + + pub(super) fn route_notice_response( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + response: &crate::domains::notice::NoticeResponse, + request_started: Option, + ) { + #[cfg(test)] + let response_ctx = { + let mut payload_encoder = + crate::dispatch::protocol::payload_codec::PayloadEncoder::with_capacity(256); + let response_bytes = crate::dispatch::protocol::notice_codec::encode_response_into( + response, + &mut payload_encoder, + ); + FrameContext::new( + meta.session_id, + crate::protocol::test_support::channel_id_from_client(meta.channel), + crate::dispatch::protocol::tlv::MessageType::new(meta.message_type), + bytes::Bytes::from(response_bytes), + meta.route_family, + ) + }; + + #[cfg(not(test))] + let response_ctx = + crate::domains::notice::NoticeClientResponse::new(meta, response.clone()); + + if let Some(response_envelope) = envelope.try_reply_to(response_ctx) { + if let Err(error) = self.router.route(response_envelope) { + if let Some(metrics) = self.metrics.as_ref() { + metrics.record_response_drop(); + } else { + crate::observability::counter_inc( + crate::domains::notice::metrics::METRIC_RESPONSE_DROPS_TOTAL, + ); + } + tracing::warn!( + domain = "notice", + session_id = meta.session_id, + route_family = meta.route_family.as_u64(), + error = %error, + "Dropped best-effort Notice response" + ); + } + } + + if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { + if response.is_failure() { + metrics.record_failure(started_at); + } else { + metrics.record_success(started_at); + } + } + } +} diff --git a/src/domains/notice/sink/state.rs b/src/domains/notice/sink/state.rs new file mode 100644 index 00000000..b8c2cf0d --- /dev/null +++ b/src/domains/notice/sink/state.rs @@ -0,0 +1,49 @@ +//! Notice domain sink and core state definitions. +//! +//! Notice subscriptions are broker-local in-memory state only. They are +//! session-scoped, cleaned up on disconnect, and are never replayed or +//! restored after broker restart. + +use super::{ + NoticeDeliveryJob, NoticeDomainCommand, NoticeMetrics, NoticeRouteStats, NoticeRouteStatsKey, + NoticeSubscription, RoutedSubscriptionSet, +}; +use crate::runtime::{CleanedUpSessions, ManagedActor, Router}; +use parking_lot::Mutex; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU64}; +use std::sync::Arc; + +/// Live notice pub/sub state for the current broker process. +/// +/// This core owns the authoritative in-memory subscription index used for +/// delivery and admin snapshots. State disappears on session cleanup or broker +/// restart and is never durably recovered or replayed. +pub(super) struct NoticeDomainCore { + /// Actor-owned single-writer state. The mutex supports immutable facade + /// methods; production mutation remains serialized by `NoticeDomainActor`. + pub(super) families: Mutex< + HashMap>, + >, + /// Actor-owned single-writer route telemetry guarded for facade reads. + pub(super) route_stats: Mutex>, + pub(super) next_sub_id: AtomicU64, + pub(super) router: Arc, + pub(super) admin_read_model: Arc, + pub(super) admin_snapshot_dirty: AtomicBool, + pub(super) metrics: Option, + pub(super) active: AtomicBool, + /// Sessions disconnect cleanup has already run for; guards against a + /// stale queued request recreating a subscription. See `cleanup.rs`. + pub(super) cleaned_up_sessions: Mutex, + /// One bounded, ordered delivery lane per route family prevents a blocked + /// subscriber from stalling unrelated families on the Notice actor. + pub(super) delivery_workers: Mutex< + HashMap>, + >, +} + +pub struct NoticeDomainSink { + pub(super) core: Arc, + pub(super) actor: ManagedActor, +} diff --git a/src/domains/notice/sink/subscriptions.rs b/src/domains/notice/sink/subscriptions.rs new file mode 100644 index 00000000..4bfa6050 --- /dev/null +++ b/src/domains/notice/sink/subscriptions.rs @@ -0,0 +1,173 @@ +//! Subscribe/unsubscribe message handling: mutation of the live subscription +//! index in response to a client request. + +use super::{ + subscription_limit_error, Arc, NoticeDomainCore, NoticeSubscription, Ordering, + RoutedSubscriptionSet, +}; + +impl NoticeDomainCore { + pub(super) fn dispatch_notice_message( + &self, + notice_msg: crate::domains::notice::protocol::NotificationMessage, + ) -> (Option, bool) { + use crate::domains::notice::protocol::NotificationMessage; + use crate::domains::notice::NoticeResponse; + + match notice_msg { + NotificationMessage::Publish(pub_msg) => { + self.publish_route_payload(pub_msg.family_id, &pub_msg.route, &pub_msg.payload); + (None, false) + } + NotificationMessage::Subscribe(sub_msg) => self.handle_subscribe_message(&sub_msg), + NotificationMessage::Unsubscribe(unsub_msg) => { + let family_id = unsub_msg.family_id; + let mut families = self.families.lock(); + let removed = if let Some(state) = families.get_mut(&family_id) { + let removed = state.remove_subscription_for_session( + unsub_msg.family_id, + unsub_msg.session_id.0, + unsub_msg.subscription_id, + ); + if state.is_empty() { + families.remove(&family_id); + } + removed + } else { + false + }; + if removed { + self.counter_add("fitz_notice_unsubscribes_total", 1); + } + (Some(NoticeResponse::Ok), removed) + } + NotificationMessage::UnsubscribeAll(unsub_all) => { + let session_id = unsub_all.session_id.0; + let removed = self.unsubscribe_all_for_session(session_id); + tracing::debug!( + domain = "notice", + session = session_id, + "All subscriptions removed for session" + ); + (Some(NoticeResponse::Ok), removed > 0) + } + NotificationMessage::Deliver(_) => (Some(NoticeResponse::Ok), false), + } + } + + fn handle_subscribe_message( + &self, + sub_msg: &crate::domains::notice::protocol::SubscribeMessage, + ) -> (Option, bool) { + if let Some(response) = self.try_reuse_existing(sub_msg) { + return (Some(response), false); + } + let compiled = match Self::compile_pattern(sub_msg) { + Ok(compiled) => compiled, + Err(response) => return (Some(response), false), + }; + let (response, state_changed) = self.allocate_and_insert(sub_msg, compiled); + (Some(response), state_changed) + } + + fn compile_pattern( + sub_msg: &crate::domains::notice::protocol::SubscribeMessage, + ) -> Result { + crate::runtime::DomainKind::Notice + .descriptor() + .compile_registration_pattern(sub_msg.pattern.as_str()) + .map_err(|error| { + tracing::warn!( + domain = "notice", + session = sub_msg.session_id.0, + "Rejected invalid subscription pattern" + ); + crate::domains::notice::NoticeResponse::Error(error) + }) + } + + fn try_reuse_existing( + &self, + sub_msg: &crate::domains::notice::protocol::SubscribeMessage, + ) -> Option { + let families = self.families.lock(); + let id = families.get(&sub_msg.family_id).and_then(|state| { + state.find_existing_id(sub_msg.session_id.0, sub_msg.pattern.as_str()) + })?; + tracing::debug!( + domain = "notice", + session = sub_msg.session_id.0, + subscription_id = id, + pattern = sub_msg.pattern.as_str(), + "Notice subscription already exists (idempotent)" + ); + Some(crate::domains::notice::NoticeResponse::SubscribeOk { + subscription_id: id, + }) + } + + fn allocate_and_insert( + &self, + sub_msg: &crate::domains::notice::protocol::SubscribeMessage, + compiled: crate::runtime::matcher::Pattern, + ) -> (crate::domains::notice::NoticeResponse, bool) { + use crate::domains::notice::NoticeResponse; + + let mut families = self.families.lock(); + let session_subscription_count = families + .values() + .map(|state| state.subscription_count_for_session(sub_msg.session_id.0)) + .sum::(); + let state = families + .entry(sub_msg.family_id) + .or_insert_with(RoutedSubscriptionSet::new); + + let (response, state_changed) = if let Some(error) = + subscription_limit_error(state, session_subscription_count, sub_msg, &compiled) + { + (error, false) + } else { + let Ok(new_id) = + self.next_sub_id + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + current.checked_add(1) + }) + else { + let state_empty = state.is_empty(); + if state_empty { + families.remove(&sub_msg.family_id); + } + return ( + NoticeResponse::Error("subscription ID space exhausted".to_string()), + false, + ); + }; + state.insert( + sub_msg.family_id, + NoticeSubscription { + pattern: compiled, + pattern_route: Arc::from(sub_msg.pattern.as_str()), + session_id: sub_msg.session_id.0, + subscription_id: new_id, + subscriber: sub_msg.subscriber.clone(), + }, + ); + + tracing::debug!( + domain = "notice", + session = sub_msg.session_id.0, + subscription_id = new_id, + pattern = sub_msg.pattern.as_str(), + "Notice subscription added" + ); + ( + NoticeResponse::SubscribeOk { + subscription_id: new_id, + }, + true, + ) + }; + + (response, state_changed) + } +} diff --git a/src/domains/notice/sink/test_channels.rs b/src/domains/notice/sink/test_channels.rs index 0256712d..1be443fe 100644 --- a/src/domains/notice/sink/test_channels.rs +++ b/src/domains/notice/sink/test_channels.rs @@ -14,20 +14,3 @@ pub(super) fn test_client_channel_from_protocol( } } } - -pub(super) fn test_protocol_channel_from_client( - channel: crate::runtime::ClientChannel, -) -> crate::dispatch::protocol::frame::ChannelId { - match channel { - crate::runtime::ClientChannel::Control => { - crate::dispatch::protocol::frame::ChannelId::Control - } - crate::runtime::ClientChannel::Pub => crate::dispatch::protocol::frame::ChannelId::Pub, - crate::runtime::ClientChannel::Sub => crate::dispatch::protocol::frame::ChannelId::Sub, - crate::runtime::ClientChannel::Rpc => crate::dispatch::protocol::frame::ChannelId::Rpc, - crate::runtime::ClientChannel::Lease => crate::dispatch::protocol::frame::ChannelId::Lease, - crate::runtime::ClientChannel::Internal => { - crate::dispatch::protocol::frame::ChannelId::Internal - } - } -} diff --git a/src/domains/notice/sink/tests.rs b/src/domains/notice/sink/tests.rs index cfa068be..18539858 100644 --- a/src/domains/notice/sink/tests.rs +++ b/src/domains/notice/sink/tests.rs @@ -9,8 +9,11 @@ use crate::domains::subscription_state::{ }; use crate::runtime::mailbox::Mailbox; use crate::runtime::routing::{Route, RouteAddress, RouteFamily}; +use crate::runtime::Router; use bytes::Bytes; +use parking_lot::Mutex; use std::sync::Arc; +use std::time::Duration; mod cleanup; mod correctness; diff --git a/src/domains/notice/sink/tests/cleanup.rs b/src/domains/notice/sink/tests/cleanup.rs index b624085b..776d5a6e 100644 --- a/src/domains/notice/sink/tests/cleanup.rs +++ b/src/domains/notice/sink/tests/cleanup.rs @@ -35,3 +35,55 @@ fn should_route_notice_session_cleanup_command_through_managed_actor() { assert_eq!(removed, Err(DeliveryError::ActorStopped)); assert_eq!(subscription_count, Err(DeliveryError::ActorStopped)); } + +#[test] +fn should_reject_stale_subscribe_after_disconnect_cleanup_marks_session() { + // Arrange + let family = RouteFamily::new(1); + let session_id = 9; + let notice_route = "notice://acme/events"; + let client_address = RouteAddress::new(family, Route::new("inbox://session/9")); + let notice_address = RouteAddress::new(family, Route::new("notice://acme/inbound")); + let router = Arc::new(Router::new()); + let client_mailbox = Arc::new(Mailbox::new(8)); + router.register(client_address.clone(), client_mailbox.clone()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = NoticeDomainSink::new(router, admin_read_model); + subscribe_notice_pattern( + &sink, + &client_address, + ¬ice_address, + session_id, + notice_route, + family, + ); + let _ = decode_notice_response(&client_mailbox); + assert_eq!(sink.subscription_count(), Ok(1)); + + // Act: run disconnect cleanup directly on the core, giving a + // deterministic ordering (cleanup completes, then the stale request + // below is processed) equivalent to what the high-priority mailbox lane + // guarantees a real disconnect races against a queued normal-lane + // request. + sink.core.handle_cleanup_envelope(&Envelope::new( + RouteAddress::new(family, Route::new("notice://cleanup")), + crate::runtime::SessionCleanup { session_id }, + )); + assert_eq!(sink.subscription_count(), Ok(0)); + + subscribe_notice_pattern( + &sink, + &client_address, + ¬ice_address, + session_id, + notice_route, + family, + ); + + // Assert: the stale subscribe from the now-cleaned-up session is + // rejected instead of resurrecting a subscription for it. + let response = decode_notice_response(&client_mailbox); + assert_eq!(response.status, 1); + assert_eq!(response.error.as_deref(), Some("session already closed")); + assert_eq!(sink.subscription_count(), Ok(0)); +} diff --git a/src/domains/queue/actor/enqueue.rs b/src/domains/queue/actor/enqueue.rs index d75c3a4a..25e425aa 100644 --- a/src/domains/queue/actor/enqueue.rs +++ b/src/domains/queue/actor/enqueue.rs @@ -21,8 +21,49 @@ struct BatchSendPlan { } impl QueueActor { + /// Refuse a body that no RESERVE shape could ever return. + /// + /// Accepting it would tell the producer the message is stored and then + /// dead-letter it at reserve time, turning a rejectable write into a loss + /// the consumer discovers instead. + fn validate_deliverable_body(&self, body_len: usize) -> Result<(), String> { + let route_len = "queue://".len() + + self.queue_key.realm.len() + + 1 + + self.queue_key.area.len() + + 1 + + self.queue_key.resource.len(); + let limit = crate::domains::queue::protocol::max_deliverable_body_bytes(route_len); + if body_len > limit { + return Err(format!( + "ERR_MESSAGE_TOO_LARGE: body is {body_len} bytes, exceeding the {limit}-byte \ + limit a reserve response can return for this queue" + )); + } + Ok(()) + } + + /// Enqueue without the deliverable-body check, to stand in for records + /// written before that limit existed. The reserve-time dead-letter path + /// still has to handle those, so it still needs coverage. + #[cfg(test)] + pub(crate) fn handle_send_unvalidated_for_tests( + &mut self, + body: Bytes, + delay_seconds: Option, + ) -> QueueResponse { + self.send_inner(body, delay_seconds) + } + /// Handle send operation pub fn handle_send(&mut self, body: Bytes, delay_seconds: Option) -> QueueResponse { + if let Err(message) = self.validate_deliverable_body(body.len()) { + return QueueResponse::Error { message }; + } + self.send_inner(body, delay_seconds) + } + + fn send_inner(&mut self, body: Bytes, delay_seconds: Option) -> QueueResponse { if !self.has_message_id_capacity(1) { return QueueResponse::Error { message: "queue message id space exhausted".to_string(), @@ -133,6 +174,13 @@ impl QueueActor { if items.is_empty() { return QueueResponse::SentBatch { ids: vec![] }; } + // Validate the whole batch first: a partially-applied batch would + // leave the producer unable to tell which items were stored. + for (body, _) in items { + if let Err(message) = self.validate_deliverable_body(body.len()) { + return QueueResponse::Error { message }; + } + } if !self.has_message_id_capacity(Self::usize_to_u64(items.len())) { return QueueResponse::Error { message: "queue message id space exhausted".to_string(), diff --git a/src/domains/queue/actor/mod.rs b/src/domains/queue/actor/mod.rs index 6442e1e3..598c79f0 100644 --- a/src/domains/queue/actor/mod.rs +++ b/src/domains/queue/actor/mod.rs @@ -163,6 +163,7 @@ enum DlqReason { HydrationFailed = 2, DeliveryAttemptsExhausted = 3, InflightEpochExhausted = 4, + ReserveResponseTooLarge = 5, } #[derive(Clone, Copy)] @@ -182,6 +183,7 @@ impl DlqReason { Self::HydrationFailed => "hydration_failed", Self::DeliveryAttemptsExhausted => "delivery_attempts_exhausted", Self::InflightEpochExhausted => "inflight_epoch_exhausted", + Self::ReserveResponseTooLarge => "reserve_response_too_large", } } @@ -192,6 +194,7 @@ impl DlqReason { 2 => Ok(Some(Self::HydrationFailed)), 3 => Ok(Some(Self::DeliveryAttemptsExhausted)), 4 => Ok(Some(Self::InflightEpochExhausted)), + 5 => Ok(Some(Self::ReserveResponseTooLarge)), other => Err(format!("Unknown DLQ reason {other}")), } } @@ -306,52 +309,6 @@ pub struct Inflight { inflight_epoch: u64, } -/// Queue operations used by live producers and consumers. -pub trait QueueDataPlane {} - -/// Queue operations used only by administration and runtime management. -pub trait QueueAdminPlane { - fn admin_snapshot(&self) -> QueueAdminSnapshot; - fn admin_inflight(&self) -> Vec; - fn admin_dead_letters(&self) -> Vec; - /// Replays a dead letter into the ready queue. - /// - /// # Errors - /// - /// Returns an error when the durable transition cannot be committed. - fn replay_dead_letter(&mut self, id: MessageId) -> Result; - /// Permanently removes a dead letter. - /// - /// # Errors - /// - /// Returns an error when the durable deletion cannot be committed. - fn purge_dead_letter(&mut self, id: MessageId) -> Result; -} - -impl QueueDataPlane for QueueActor {} - -impl QueueAdminPlane for QueueActor { - fn admin_snapshot(&self) -> QueueAdminSnapshot { - QueueActor::admin_snapshot(self) - } - - fn admin_inflight(&self) -> Vec { - QueueActor::admin_inflight(self) - } - - fn admin_dead_letters(&self) -> Vec { - QueueActor::admin_dead_letters(self) - } - - fn replay_dead_letter(&mut self, id: MessageId) -> Result { - QueueActor::replay_dead_letter(self, id) - } - - fn purge_dead_letter(&mut self, id: MessageId) -> Result { - QueueActor::purge_dead_letter(self, id) - } -} - /// Timer event for inflight expiration #[derive(Debug, Clone, PartialEq, Eq)] struct InflightExpiry { diff --git a/src/domains/queue/actor/reserve_and_ack.rs b/src/domains/queue/actor/reserve_and_ack.rs index 57e32cac..93f1be58 100644 --- a/src/domains/queue/actor/reserve_and_ack.rs +++ b/src/domains/queue/actor/reserve_and_ack.rs @@ -14,6 +14,13 @@ enum AckAuthorizationError { Expired, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ReserveWireBudgetDecision { + Reserve(usize), + Skip, + Stop, +} + fn validate_ack_authorization( inflight: &Inflight, token: u64, @@ -43,22 +50,48 @@ impl QueueActor { inflight_seconds: u64, batch_size: Option, ) -> QueueResponse { - self.handle_receive_internal(Some(session_id), inflight_seconds, batch_size) + let mut response_bytes_remaining = usize::MAX; + self.handle_receive_for_session_with_wire_budget( + session_id, + inflight_seconds, + batch_size, + &mut response_bytes_remaining, + 0, + ) + .0 + } + + pub(crate) fn handle_receive_for_session_with_wire_budget( + &mut self, + session_id: u64, + inflight_seconds: u64, + batch_size: Option, + response_bytes_remaining: &mut usize, + message_wire_overhead_bytes: usize, + ) -> (QueueResponse, bool) { + self.handle_receive_internal( + Some(session_id), + inflight_seconds, + batch_size.unwrap_or(1), + response_bytes_remaining, + message_wire_overhead_bytes, + ) } fn handle_receive_internal( &mut self, owner_session_id: Option, inflight_seconds: u64, - batch_size: Option, - ) -> QueueResponse { - let batch_size = batch_size.unwrap_or(1); + batch_size: usize, + response_bytes_remaining: &mut usize, + message_wire_overhead_bytes: usize, + ) -> (QueueResponse, bool) { let now = self.clock.now_instant(); let now_epoch_ms = self.clock.now_epoch_ms(); let (expires_at, expires_at_epoch_ms) = match Self::inflight_expiration(now, now_epoch_ms, inflight_seconds) { Ok(expiration) => expiration, - Err(response) => return response, + Err(response) => return (response, false), }; let mut messages = Vec::with_capacity(self.ready.len().min(batch_size)); @@ -68,6 +101,18 @@ impl QueueActor { break; }; + // Skip hydration (real storage I/O) when this message's fixed + // per-response overhead alone already exceeds what's left of the + // wire budget: no body size, however small, changes that outcome. + // This only short-circuits once a prior message has already been + // reserved this call - an untouched budget failing here instead + // means the message may be fundamentally too large for any + // response, which the divert check below can only decide once + // it knows the body size. + if !messages.is_empty() && message_wire_overhead_bytes > *response_bytes_remaining { + return (QueueResponse::Received { messages }, true); + } + let (body, attempts) = match self.hydrate_record_for_receive(id) { Ok(record) => record, Err(e) => { @@ -103,15 +148,28 @@ impl QueueActor { break; }; + let message_wire_bytes = match self.reserve_wire_budget_decision( + id, + body.len(), + message_wire_overhead_bytes, + *response_bytes_remaining, + now_epoch_ms, + ) { + ReserveWireBudgetDecision::Reserve(bytes) => bytes, + ReserveWireBudgetDecision::Skip => continue, + ReserveWireBudgetDecision::Stop => { + return (QueueResponse::Received { messages }, true); + } + }; + let Some(id) = self.pop_ready() else { break; }; + *response_bytes_remaining -= message_wire_bytes; self.evict_cached_body(id); - // Generate inflight token let token = Self::generate_token(); - // Create inflight entry self.inflight.insert( id, Inflight { @@ -131,18 +189,7 @@ impl QueueActor { Some(now_epoch_ms), ); - // Schedule expiration timer - self.timers.push(Reverse(InflightExpiry { - id, - inflight_epoch, - expires_at, - expires_at_ms: expires_at_epoch_ms, - })); - - // Update deadline cache if this expiration is sooner - if expires_at < self.next_expiration_deadline { - self.next_expiration_deadline = expires_at; - } + self.schedule_inflight_expiration(id, inflight_epoch, expires_at, expires_at_epoch_ms); // Build response message messages.push(ReservedMessage { @@ -154,13 +201,58 @@ impl QueueActor { }); } - // If no messages were reserved, return an empty response (avoid NotFound). - // Clients expect an empty slice when the queue is empty rather than an error. - if messages.is_empty() { - return QueueResponse::Received { messages }; + (QueueResponse::Received { messages }, false) + } + + fn schedule_inflight_expiration( + &mut self, + id: MessageId, + inflight_epoch: u64, + expires_at: Instant, + expires_at_epoch_ms: u64, + ) { + self.timers.push(Reverse(InflightExpiry { + id, + inflight_epoch, + expires_at, + expires_at_ms: expires_at_epoch_ms, + })); + if expires_at < self.next_expiration_deadline { + self.next_expiration_deadline = expires_at; } + } - QueueResponse::Received { messages } + fn reserve_wire_budget_decision( + &mut self, + id: MessageId, + body_bytes: usize, + message_wire_overhead_bytes: usize, + response_bytes_remaining: usize, + now_epoch_ms: u64, + ) -> ReserveWireBudgetDecision { + let message_wire_bytes = message_wire_overhead_bytes.saturating_add(body_bytes); + if message_wire_bytes <= response_bytes_remaining { + return ReserveWireBudgetDecision::Reserve(message_wire_bytes); + } + let empty_response_message_budget = + crate::domains::queue::protocol::MAX_QUEUE_RESPONSE_PAYLOAD_BYTES + - crate::domains::queue::protocol::RECEIVED_RESPONSE_HEADER_BYTES; + // Dead-lettering is permanent, so it must only fire when *no* reserve + // shape could ever carry this body. A wildcard reserve pays extra + // per-message overhead (routing envelope + route string) that a + // concrete reserve does not, so judging by this caller's overhead + // would discard messages a concrete `RESERVE` delivers fine. Charge + // the smallest possible shape instead, and let a wildcard caller that + // cannot fit the message simply `Stop` and leave it ready. + let smallest_possible_message_wire_bytes = + crate::domains::queue::protocol::RESERVED_MESSAGE_WIRE_OVERHEAD_BYTES + .saturating_add(body_bytes); + if smallest_possible_message_wire_bytes > empty_response_message_budget + && self.divert_ready_or_log(id, DlqReason::ReserveResponseTooLarge, now_epoch_ms) + { + return ReserveWireBudgetDecision::Skip; + } + ReserveWireBudgetDecision::Stop } fn divert_ready_or_log( diff --git a/src/domains/queue/actor/tests/inflight_and_delivery.rs b/src/domains/queue/actor/tests/inflight_and_delivery.rs index 5db14d81..5f566e9f 100644 --- a/src/domains/queue/actor/tests/inflight_and_delivery.rs +++ b/src/domains/queue/actor/tests/inflight_and_delivery.rs @@ -133,6 +133,107 @@ fn should_reserve_multiple_messages_in_batch() { } } +#[test] +fn should_bound_reserve_batch_to_tlv_payload_capacity() { + // Arrange + let store = Arc::new( + cntryl_midge::Engine::open( + cntryl_midge::OpenOptions::in_memory() + .build() + .expect("build in-memory test options"), + ) + .expect("Failed to open Midge"), + ); + let queue_key = unique_queue_key("jobs-reserve-wire-capacity"); + let mut actor = QueueActor::new( + RouteFamily::new(0), + queue_key, + store, + None, + crate::utils::idempotency::default_dedup_store(), + ); + for _ in 0..100 { + actor.handle_send(Bytes::from(vec![0x5a; 1024]), None); + } + + // Act + let mut response_bytes_remaining = + crate::domains::queue::protocol::MAX_QUEUE_RESPONSE_PAYLOAD_BYTES + - crate::domains::queue::protocol::RECEIVED_RESPONSE_HEADER_BYTES; + let (response, wire_budget_exhausted) = actor.handle_receive_for_session_with_wire_budget( + TEST_SESSION_ID, + 30, + Some(100), + &mut response_bytes_remaining, + crate::domains::queue::protocol::RESERVED_MESSAGE_WIRE_OVERHEAD_BYTES, + ); + let payload = crate::dispatch::protocol::queue_codec::encode_response(202, &response); + + // Assert + assert!(u16::try_from(payload.len()).is_ok()); + let QueueResponse::Received { messages } = response else { + panic!("Expected Received response"); + }; + assert_eq!(messages.len(), 62); + assert_eq!(actor.ready_len(), 38); + assert_eq!(actor.inflight.len(), 62); + assert!(actor.admin_dead_letters().is_empty()); + assert!(wire_budget_exhausted); +} + +#[test] +fn should_dead_letter_oversized_head_and_reserve_following_message() { + // Arrange + let store = Arc::new( + cntryl_midge::Engine::open( + cntryl_midge::OpenOptions::in_memory() + .build() + .expect("build in-memory test options"), + ) + .expect("Failed to open Midge"), + ); + let queue_key = unique_queue_key("jobs-reserve-oversized-head"); + let mut actor = QueueActor::new( + RouteFamily::new(0), + queue_key, + store, + None, + crate::utils::idempotency::default_dedup_store(), + ); + let response_budget = crate::domains::queue::protocol::MAX_QUEUE_RESPONSE_PAYLOAD_BYTES + - crate::domains::queue::protocol::RECEIVED_RESPONSE_HEADER_BYTES; + let message_overhead = crate::domains::queue::protocol::RESERVED_MESSAGE_WIRE_OVERHEAD_BYTES; + // Stands in for a record written before the SEND limit existed; the + // reserve-time dead-letter path still has to cope with those. + actor.handle_send_unvalidated_for_tests( + Bytes::from(vec![0x5a; response_budget - message_overhead + 1]), + None, + ); + actor.handle_send(Bytes::from_static(b"deliverable"), None); + + // Act + let mut response_bytes_remaining = response_budget; + let (response, _) = actor.handle_receive_for_session_with_wire_budget( + TEST_SESSION_ID, + 30, + Some(1), + &mut response_bytes_remaining, + message_overhead, + ); + + // Assert + let QueueResponse::Received { messages } = response else { + panic!("Expected Received response"); + }; + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].body, Bytes::from_static(b"deliverable")); + assert_eq!(actor.ready_len(), 0); + assert_eq!(actor.inflight.len(), 1); + let dead_letters = actor.admin_dead_letters(); + assert_eq!(dead_letters.len(), 1); + assert_eq!(dead_letters[0].reason, "reserve_response_too_large"); +} + #[test] fn should_ack_multiple_messages_in_batch() { // Arrange @@ -973,3 +1074,229 @@ fn should_allow_unlimited_retries_when_max_attempts_is_none() { _ => panic!("Expected Received response"), } } + +#[test] +fn should_not_dead_letter_message_that_only_a_wildcard_reserve_cannot_carry() { + // Arrange + // A body that fits a concrete reserve response exactly, but not + // once the wildcard routing envelope and route string are added. + let store = Arc::new( + cntryl_midge::Engine::open( + cntryl_midge::OpenOptions::in_memory() + .build() + .expect("build in-memory test options"), + ) + .expect("Failed to open Midge"), + ); + let queue_key = unique_queue_key("jobs-reserve-wildcard-overhead"); + let mut actor = QueueActor::new( + RouteFamily::new(0), + queue_key, + store, + None, + crate::utils::idempotency::default_dedup_store(), + ); + let response_budget = crate::domains::queue::protocol::MAX_QUEUE_RESPONSE_PAYLOAD_BYTES + - crate::domains::queue::protocol::RECEIVED_RESPONSE_HEADER_BYTES; + let concrete_overhead = crate::domains::queue::protocol::RESERVED_MESSAGE_WIRE_OVERHEAD_BYTES; + let route = "queue://acme/jobs/wildcard-overhead"; + let wildcard_overhead = concrete_overhead + + crate::domains::queue::protocol::ROUTED_MESSAGE_WIRE_OVERHEAD_BYTES + + route.len(); + // Above the new SEND limit (which uses the wildcard shape), so seed it as + // a record written before that limit existed. + actor.handle_send_unvalidated_for_tests( + Bytes::from(vec![0x5a; response_budget - concrete_overhead]), + None, + ); + + // Act + // Reserve through the wildcard wire shape. + let mut response_bytes_remaining = response_budget; + let (response, _) = actor.handle_receive_for_session_with_wire_budget( + TEST_SESSION_ID, + 30, + Some(1), + &mut response_bytes_remaining, + wildcard_overhead, + ); + + // Assert + // The wildcard caller gets nothing, but the message stays ready + // for a concrete reserve rather than being permanently dead-lettered. + let QueueResponse::Received { messages } = response else { + panic!("Expected Received response"); + }; + assert!(messages.is_empty()); + assert!( + actor.admin_dead_letters().is_empty(), + "message deliverable by a concrete reserve must not be dead-lettered" + ); + assert_eq!(actor.ready_len(), 1); + + // And a concrete reserve still delivers it. + let mut concrete_bytes_remaining = response_budget; + let (concrete_response, _) = actor.handle_receive_for_session_with_wire_budget( + TEST_SESSION_ID, + 30, + Some(1), + &mut concrete_bytes_remaining, + concrete_overhead, + ); + let QueueResponse::Received { messages } = concrete_response else { + panic!("Expected Received response"); + }; + assert_eq!(messages.len(), 1); +} + +#[test] +fn should_reject_send_of_body_that_no_reserve_shape_could_return() { + // Arrange + // The inbound SEND ceiling and the outbound RESERVE ceiling are the same + // 65_535-byte payload limit, but the response spends part of it on a + // header and a per-message envelope. A body accepted on write above that + // budget can never be handed back, so it must be refused at SEND rather + // than accepted and dead-lettered later, after the producer was told it + // was stored. + let store = Arc::new( + cntryl_midge::Engine::open( + cntryl_midge::OpenOptions::in_memory() + .build() + .expect("build in-memory test options"), + ) + .expect("Failed to open Midge"), + ); + let queue_key = unique_queue_key("jobs-send-oversized"); + let route_len = format!( + "queue://{}/{}/{}", + queue_key.realm, queue_key.area, queue_key.resource + ) + .len(); + let mut actor = QueueActor::new( + RouteFamily::new(0), + queue_key, + store, + None, + crate::utils::idempotency::default_dedup_store(), + ); + // The strictest shape is a wildcard reserve: header, message envelope, + // routing envelope, and the route string. + let max_deliverable_body = crate::domains::queue::protocol::MAX_QUEUE_RESPONSE_PAYLOAD_BYTES + - crate::domains::queue::protocol::RECEIVED_RESPONSE_HEADER_BYTES + - crate::domains::queue::protocol::RESERVED_MESSAGE_WIRE_OVERHEAD_BYTES + - crate::domains::queue::protocol::ROUTED_MESSAGE_WIRE_OVERHEAD_BYTES + - route_len; + + // Act + let accepted = actor.handle_send(Bytes::from(vec![0x5a; max_deliverable_body]), None); + let rejected = actor.handle_send(Bytes::from(vec![0x5a; max_deliverable_body + 1]), None); + let rejected_batch = + actor.handle_send_batch(&[(Bytes::from(vec![0x5a; max_deliverable_body + 1]), None)]); + + // Assert + assert!( + matches!(accepted, QueueResponse::Sent { .. }), + "a body at the deliverable limit must still be accepted, got {accepted:?}" + ); + let QueueResponse::Error { message } = rejected else { + panic!("expected an oversized SEND to be refused, got {rejected:?}"); + }; + assert!( + message.contains("ERR_MESSAGE_TOO_LARGE"), + "unexpected error: {message}" + ); + assert!( + matches!(rejected_batch, QueueResponse::Error { .. }), + "a batch carrying an oversized body must be refused too, got {rejected_batch:?}" + ); + assert_eq!( + actor.ready_len(), + 1, + "the rejected bodies must not be stored" + ); + assert!(actor.admin_dead_letters().is_empty()); +} + +#[test] +fn should_skip_hydration_when_wire_budget_is_already_exhausted() { + // Arrange + // Once the response's wire budget is exhausted by an earlier message, + // the next ready candidate's fixed per-response overhead alone already + // guarantees it cannot fit. Hydrating it anyway would pay real storage + // I/O only to immediately discard the result. Prove this doesn't happen + // by deleting the second message's header out from under the actor: if + // hydration were attempted, it would hit "disappeared from storage" and + // divert the message instead of simply leaving it ready for next time. + let store = Arc::new( + cntryl_midge::Engine::open( + cntryl_midge::OpenOptions::in_memory() + .build() + .expect("build in-memory test options"), + ) + .expect("Failed to open Midge"), + ); + let queue_key = unique_queue_key("jobs-skip-hydration-on-exhausted-budget"); + let mut actor = QueueActor::new( + RouteFamily::new(0), + queue_key.clone(), + store.clone(), + None, + crate::utils::idempotency::default_dedup_store(), + ); + let message_overhead = crate::domains::queue::protocol::RESERVED_MESSAGE_WIRE_OVERHEAD_BYTES; + // A tiny explicit budget stands in for "an earlier message this call + // already consumed most of the response": just enough room for the + // first (1-byte) message, leaving less than `message_overhead` behind - + // too little for the second message to fit no matter its body size. + let initial_budget = message_overhead + 3; + actor.handle_send(Bytes::from_static(b"x"), None); + let second_id = match actor.handle_send(Bytes::from_static(b"never hydrated"), None) { + QueueResponse::Sent { id } => id, + other => panic!("Expected Sent response, found {other:?}"), + }; + + // Force the second message out of every in-memory cache and delete its + // header from storage, so any hydration attempt on it fails loudly. + actor.evict_cached_record(second_id); + actor.evict_cached_body(second_id); + let mut txn = store + .begin_tx( + queue_key.family.id(), + cntryl_midge::TransactionMode::ReadWrite, + ) + .expect("begin write tx"); + txn.delete(QueueActor::header_key(&queue_key, second_id)) + .expect("delete second message's header"); + txn.commit(cntryl_midge::WriteOptions::buffered()) + .expect("commit second message header delete"); + + // Act + let mut response_bytes_remaining = initial_budget; + let (response, wire_budget_exhausted) = actor.handle_receive_for_session_with_wire_budget( + TEST_SESSION_ID, + 30, + Some(2), + &mut response_bytes_remaining, + message_overhead, + ); + + // Assert + let QueueResponse::Received { messages } = response else { + panic!("Expected Received response"); + }; + assert_eq!( + messages.len(), + 1, + "only the first message fits the exhausted budget" + ); + assert!(wire_budget_exhausted); + assert_eq!( + actor.ready_len(), + 1, + "the second message must remain ready, untouched by a failed hydration" + ); + assert!( + actor.admin_dead_letters().is_empty(), + "the second message must not be diverted - it was never hydrated to find out its header is gone" + ); +} diff --git a/src/domains/queue/actor/tests/recovery_and_storage.rs b/src/domains/queue/actor/tests/recovery_and_storage.rs index 5c9a93b7..73f2cb71 100644 --- a/src/domains/queue/actor/tests/recovery_and_storage.rs +++ b/src/domains/queue/actor/tests/recovery_and_storage.rs @@ -161,7 +161,7 @@ fn should_hydrate_oversized_body_from_store_without_caching() { // Act let oversized = Bytes::from(vec![b'x'; QueueActor::BODY_CACHE_LIMIT_BYTES + 1]); - let response = actor.handle_send(oversized.clone(), None); + let response = actor.handle_send_unvalidated_for_tests(oversized.clone(), None); assert!(matches!(response, QueueResponse::Sent { .. })); match actor.handle_receive_for_session(TEST_SESSION_ID, 30, Some(1)) { diff --git a/src/domains/queue/actor/tests/storage_index_and_core.rs b/src/domains/queue/actor/tests/storage_index_and_core.rs index 2e6d1ca4..717d149e 100644 --- a/src/domains/queue/actor/tests/storage_index_and_core.rs +++ b/src/domains/queue/actor/tests/storage_index_and_core.rs @@ -609,11 +609,13 @@ pub(super) fn should_bound_hot_body_cache_total_bytes() { ); // Act + // Bodies this large predate the SEND deliverable-body limit, so seed them + // unvalidated: the cache byte accounting still has to bound them. let body_size = QueueActor::BODY_CACHE_LIMIT_BYTES / 4 + 1; for i in 0..5 { let byte = u8::try_from(i).expect("body byte should fit in u8"); let body = Bytes::from(vec![byte; body_size]); - let response = actor.handle_send(body, None); + let response = actor.handle_send_unvalidated_for_tests(body, None); assert!(matches!(response, QueueResponse::Sent { .. })); } diff --git a/src/domains/queue/metrics.rs b/src/domains/queue/metrics.rs index c9c7ad33..528b832b 100644 --- a/src/domains/queue/metrics.rs +++ b/src/domains/queue/metrics.rs @@ -8,6 +8,12 @@ pub const METRIC_LATENCY_MS: &str = "fitz_queue_latency_ms"; pub const METRIC_READY_GAUGE: &str = "fitz_queue_ready_gauge"; pub const METRIC_DELAYED_GAUGE: &str = "fitz_queue_delayed_gauge"; pub const METRIC_INFLIGHT_GAUGE: &str = "fitz_queue_inflight_gauge"; +/// Responses the actor produced but the transport could not deliver. +/// +/// Success was previously recorded when the actor answered, before the +/// response was routed - so a response that never reached the client still +/// counted as a success and nothing counted the loss. +pub const METRIC_RESPONSE_ROUTE_FAILURES_TOTAL: &str = "fitz_queue_response_route_failures_total"; pub const METRIC_NOTIFY_DROPS_TOTAL: &str = "fitz_queue_notify_drops_total"; // Operation-specific counters diff --git a/src/domains/queue/protocol.rs b/src/domains/queue/protocol.rs index bf1367c7..513ec679 100644 --- a/src/domains/queue/protocol.rs +++ b/src/domains/queue/protocol.rs @@ -44,6 +44,32 @@ pub use super::core::{MessageId, QueueKey, ReservedMessage, RoutedReservedMessag /// Maximum number of messages a client may reserve in one request. pub const MAX_RESERVE_BATCH_SIZE: usize = 1024; +pub(crate) const MAX_QUEUE_RESPONSE_PAYLOAD_BYTES: usize = u16::MAX as usize; +pub(crate) const RECEIVED_RESPONSE_HEADER_BYTES: usize = 1 + 4; +pub(crate) const RESERVED_MESSAGE_WIRE_OVERHEAD_BYTES: usize = 8 + 8 + 4; +pub(crate) const ROUTED_MESSAGE_WIRE_OVERHEAD_BYTES: usize = 4; + +/// Largest message body that every RESERVE shape can still return for a queue +/// reached by `route_len` bytes of route. +/// +/// The inbound SEND ceiling and the outbound RESERVE ceiling are the same +/// payload limit, but the response spends part of it on a header, a +/// per-message envelope, and - for a wildcard reserve - a routing envelope +/// plus the route string. A body accepted above this budget can never be +/// handed back, so it is refused at SEND instead of being accepted and +/// dead-lettered later, after the producer was told it was stored. +/// +/// The strictest (wildcard) shape is used so that whether a message is +/// deliverable never depends on which reserve form a consumer happens to use. +#[must_use] +pub(crate) fn max_deliverable_body_bytes(route_len: usize) -> usize { + MAX_QUEUE_RESPONSE_PAYLOAD_BYTES + .saturating_sub(RECEIVED_RESPONSE_HEADER_BYTES) + .saturating_sub(RESERVED_MESSAGE_WIRE_OVERHEAD_BYTES) + .saturating_sub(ROUTED_MESSAGE_WIRE_OVERHEAD_BYTES) + .saturating_sub(route_len) +} + /// Queue domain messages /// /// All queue operations are asynchronous and return responses via diff --git a/src/domains/queue/sink/actors.rs b/src/domains/queue/sink/actors.rs new file mode 100644 index 00000000..5f9b3a29 --- /dev/null +++ b/src/domains/queue/sink/actors.rs @@ -0,0 +1,473 @@ +//! Per-queue warm actor lifecycle: lookup, idle sweep, fast flush, dead-letter ops. + +use super::model::{ + Arc, HashSet, Instant, Mutex, QueueDomainCore, WarmQueueActor, QUEUE_ACTOR_IDLE_TTL, + QUEUE_DEDUP_SWEEP_INTERVAL, QUEUE_IDLE_SWEEP_BATCH_SIZE, QUEUE_IDLE_SWEEP_INTERVAL, +}; + +impl QueueDomainCore { + pub(super) fn queue_key_for_route( + family_id: crate::runtime::routing::RouteFamily, + route: &crate::runtime::routing::Route, + ) -> Result { + crate::domains::queue::QueueKey::from_route(family_id, route).ok_or_else(|| { + crate::domains::queue::QueueResponse::BadRequest { + reason: format!("invalid queue route: {}", route.as_str()), + } + }) + } + + #[cfg(test)] + pub(super) fn session_inbox_address( + family_id: crate::runtime::routing::RouteFamily, + session_id: u64, + ) -> crate::runtime::routing::RouteAddress { + crate::runtime::routing::RouteAddress::new( + family_id, + crate::runtime::routing::Route::new(format!("inbox://session/{session_id}")), + ) + } + + pub(super) fn matching_queue_keys( + &self, + family: crate::runtime::routing::RouteFamily, + pattern: &crate::runtime::matcher::Pattern, + ) -> Vec { + let mut keys = self + .known_queue_keys + .lock() + .iter() + .filter(|key| key.family == family) + .filter(|key| pattern.matches(&Self::queue_ready_route(key))) + .cloned() + .collect::>(); + keys.sort_by(|left, right| { + (&left.realm, &left.area, &left.resource).cmp(&( + &right.realm, + &right.area, + &right.resource, + )) + }); + keys + } + + pub(super) fn matching_queue_key_count( + &self, + family: crate::runtime::routing::RouteFamily, + pattern: &crate::runtime::matcher::Pattern, + ) -> usize { + self.known_queue_keys + .lock() + .iter() + .filter(|key| key.family == family) + .filter(|key| pattern.matches(&Self::queue_ready_route(key))) + .count() + } + + pub(super) fn inventory_existing_queue_keys( + store: &crate::storage::FitzStorageEngine, + ) -> Result, String> { + let families = store + .list_column_families() + .map_err(|error| format!("list queue inventory families failed: {error:?}"))?; + let mut known_queue_keys = HashSet::new(); + + for family in families { + if family.id() == 0 { + continue; + } + let route_family = crate::runtime::routing::RouteFamily::new(family.id()); + let txn = store + .begin_tx(family.id(), cntryl_midge::TransactionMode::ReadOnly) + .map_err(|error| { + format!( + "queue inventory transaction failed: family={} error={error:?}", + family.id() + ) + })?; + let rows = txn.scan(&cntryl_midge::Query::new()).map_err(|error| { + format!( + "queue inventory scan failed: family={} error={error:?}", + family.id() + ) + })?; + + for row in rows { + let (key, value) = row.map_err(|error| { + format!( + "queue inventory scan failed: family={} error={error:?}", + family.id() + ) + })?; + drop(value); + if let Some(queue_key) = + crate::domains::queue::QueueActor::queue_key_from_authoritative_storage_key( + route_family, + &key, + ) + { + known_queue_keys.insert(queue_key); + } + } + } + + Ok(known_queue_keys) + } + + pub(super) fn record_ready_state( + &self, + key: &crate::domains::queue::QueueKey, + counts: crate::domains::queue::QueueActorLiveCounts, + ) -> Option { + let is_ready = counts.ready > 0; + let mut ready_states = self.ready_states.lock(); + let was_ready = ready_states.get(key).copied().unwrap_or(false); + + if counts.total() == 0 { + ready_states.remove(key); + } else { + ready_states.insert(key.clone(), is_ready); + } + + if !was_ready && is_ready { + Some(super::model::QueueReadyNotification { + family_id: key.family, + counts, + }) + } else { + None + } + } + + pub(super) fn sweep_runtime_state_at(&self, now: Instant) { + #[cfg(test)] + if self + .panic_next_runtime_sweep + .swap(false, std::sync::atomic::Ordering::AcqRel) + { + panic!("test Queue runtime sweep panic"); + } + self.expire_pending_reserves_at(now); + self.sweep_idle_actors_at(now); + self.maybe_cleanup_dedup_at(now); + self.maybe_flush_dirty_fast_families_at(now); + } + + pub(super) fn fast_flush_enabled(&self) -> bool { + self.queue_write_options.is_best_effort() && self.fast_flush_interval.is_some() + } + + pub(super) fn mark_fast_flush_dirty(&self, family_id: crate::runtime::routing::RouteFamily) { + if self.fast_flush_enabled() { + self.dirty_fast_flush_families.lock().insert(family_id.id()); + } + } + + pub(super) fn maybe_flush_dirty_fast_families_at(&self, now: Instant) { + let Some(interval) = self.fast_flush_interval else { + return; + }; + if !self.queue_write_options.is_best_effort() { + return; + } + + let should_flush = { + let mut next_fast_flush_at = self.next_fast_flush_at.lock(); + if now < *next_fast_flush_at { + false + } else { + *next_fast_flush_at = now + interval; + true + } + }; + + if should_flush { + self.flush_dirty_fast_families(); + } + } + + pub(super) fn flush_dirty_fast_families(&self) { + let dirty_family_ids = { + let mut dirty = self.dirty_fast_flush_families.lock(); + dirty.drain().collect::>() + }; + if dirty_family_ids.is_empty() { + return; + } + + let families = match self.store.list_column_families() { + Ok(families) => families, + Err(error) => { + tracing::warn!( + domain = "queue", + error = ?error, + "Failed to list queue column families for fast flush" + ); + self.dirty_fast_flush_families + .lock() + .extend(dirty_family_ids); + return; + } + }; + + let mut retry_family_ids = Vec::new(); + for family_id in dirty_family_ids { + let Some(cf) = families.iter().find(|cf| cf.id() == family_id) else { + tracing::warn!( + domain = "queue", + family = family_id, + "Queue fast flush skipped missing column family" + ); + retry_family_ids.push(family_id); + continue; + }; + + if let Err(error) = self.store.flush_cf(cf) { + tracing::warn!( + domain = "queue", + family = family_id, + error = ?error, + "Queue fast flush failed" + ); + retry_family_ids.push(family_id); + } + } + + if !retry_family_ids.is_empty() { + self.dirty_fast_flush_families + .lock() + .extend(retry_family_ids); + } + } + + pub(super) fn maybe_cleanup_dedup_at(&self, now: Instant) { + let should_cleanup = { + let mut next_dedup_sweep_at = self.next_dedup_sweep_at.lock(); + if now < *next_dedup_sweep_at { + false + } else { + *next_dedup_sweep_at = now + QUEUE_DEDUP_SWEEP_INTERVAL; + true + } + }; + + if should_cleanup { + self.dedup_store.cleanup(); + } + } + + pub(super) fn get_or_create_actor( + &self, + key: &crate::domains::queue::QueueKey, + ) -> Result<(Arc>, bool), String> { + use std::collections::hash_map::Entry; + + let now = Instant::now(); + match self.actors.lock().entry(key.clone()) { + Entry::Occupied(mut entry) => { + entry.get_mut().last_used = now; + Ok((entry.get().actor.clone(), false)) + } + Entry::Vacant(entry) => { + let actor = Arc::new(Mutex::new( + crate::domains::queue::QueueActor::try_new_with_write_options( + key.family, + key.clone(), + self.store.clone_inner(), + None, + self.dedup_store.clone(), + self.queue_write_options, + )?, + )); + entry.insert(WarmQueueActor { + actor: actor.clone(), + last_used: now, + }); + self.idle_sweep_keys.lock().push_back(key.clone()); + Ok((actor, true)) + } + } + } + + pub(super) fn sweep_idle_actors(&self) { + self.sweep_idle_actors_at(Instant::now()); + } + + pub(super) fn maybe_sweep_idle_actors(&self) { + let now = Instant::now(); + + { + let mut next_idle_sweep_at = self.next_idle_sweep_at.lock(); + if now < *next_idle_sweep_at { + return; + } + *next_idle_sweep_at = now + QUEUE_IDLE_SWEEP_INTERVAL; + } + + self.sweep_idle_actors_at(now); + } + + pub(super) fn sweep_idle_actors_at(&self, now: Instant) { + let mut changed = false; + let mut notifications = Vec::new(); + let mut removed_keys = Vec::new(); + let mut empty_removed_keys = Vec::new(); + let mut dirty_families = HashSet::new(); + let sweep_keys = { + let mut idle_sweep_keys = self.idle_sweep_keys.lock(); + let count = idle_sweep_keys.len().min(QUEUE_IDLE_SWEEP_BATCH_SIZE); + idle_sweep_keys.drain(..count).collect::>() + }; + + for key in sweep_keys { + let Some((actor_ref, last_used)) = self + .actors + .lock() + .get(&key) + .map(|warm_actor| (warm_actor.actor.clone(), warm_actor.last_used)) + else { + continue; + }; + let mut actor = actor_ref.lock(); + if actor.process_due_work() { + changed = true; + dirty_families.insert(key.family); + } + let counts = actor.live_counts(); + + if let Some(notification) = self.record_ready_state(&key, counts) { + notifications.push((key.clone(), notification)); + } + + let idle_for = now.saturating_duration_since(last_used); + let should_keep = + idle_for < QUEUE_ACTOR_IDLE_TTL || counts.delayed > 0 || counts.inflight > 0; + drop(actor); + + if should_keep { + self.idle_sweep_keys.lock().push_back(key); + continue; + } + + let removed = { + let mut actors = self.actors.lock(); + let unchanged = actors.get(&key).is_some_and(|warm_actor| { + warm_actor.last_used == last_used && Arc::ptr_eq(&warm_actor.actor, &actor_ref) + }); + unchanged && actors.remove(&key).is_some() + }; + if removed { + changed = true; + removed_keys.push(key.clone()); + if counts.total() == 0 { + empty_removed_keys.push(key); + } + } else { + self.idle_sweep_keys.lock().push_back(key); + } + } + + if !removed_keys.is_empty() { + let mut ready_states = self.ready_states.lock(); + for key in removed_keys { + ready_states.remove(&key); + } + } + if !empty_removed_keys.is_empty() { + let mut known_queue_keys = self.known_queue_keys.lock(); + for key in empty_removed_keys { + known_queue_keys.remove(&key); + } + } + for family in dirty_families { + self.mark_fast_flush_dirty(family); + } + if changed { + self.mark_admin_snapshot_dirty(); + } + for (key, notification) in notifications { + self.route_queue_ready_notification(&key, notification); + let route = Self::queue_ready_route(&key); + self.wake_pending_reserves_for_route(key.family, &route, now); + } + } + + /// Replays a dead-lettered message back into its queue. + /// + /// # Errors + /// + /// Returns an error when the warm queue actor cannot be recovered or the replay fails. + pub(super) fn replay_dead_letter( + &self, + key: &crate::domains::queue::QueueKey, + id: crate::domains::queue::MessageId, + ) -> Result { + let (actor_handle, created_actor) = self.get_or_create_actor(key)?; + let result = { + let mut actor = actor_handle.lock(); + actor.replay_dead_letter(id) + }; + + if matches!(result, Ok(true)) { + self.mark_fast_flush_dirty(key.family); + let counts = actor_handle.lock().live_counts(); + let notification = self.record_ready_state(key, counts); + self.mark_admin_snapshot_dirty(); + if let Some(notification) = notification { + self.route_queue_ready_notification(key, notification); + } + } + + if created_actor { + let should_remove = { + let actor = actor_handle.lock(); + actor.live_counts().total() == 0 + }; + if should_remove { + self.actors.lock().remove(key); + self.ready_states.lock().remove(key); + self.known_queue_keys.lock().remove(key); + self.mark_admin_snapshot_dirty(); + } + } + + result + } + + /// Permanently removes a dead-lettered message from its queue. + /// + /// # Errors + /// + /// Returns an error when the warm queue actor cannot be recovered or the purge fails. + pub(super) fn purge_dead_letter( + &self, + key: &crate::domains::queue::QueueKey, + id: crate::domains::queue::MessageId, + ) -> Result { + let (actor_handle, created_actor) = self.get_or_create_actor(key)?; + let result = { + let mut actor = actor_handle.lock(); + actor.purge_dead_letter(id) + }; + + if matches!(result, Ok(true)) { + self.mark_fast_flush_dirty(key.family); + self.mark_admin_snapshot_dirty(); + } + + if created_actor { + let should_remove = { + let actor = actor_handle.lock(); + actor.live_counts().total() == 0 + }; + if should_remove { + self.actors.lock().remove(key); + self.ready_states.lock().remove(key); + self.known_queue_keys.lock().remove(key); + self.mark_admin_snapshot_dirty(); + } + } + + result + } +} diff --git a/src/domains/queue/sink/cleanup.rs b/src/domains/queue/sink/cleanup.rs new file mode 100644 index 00000000..0125b6f2 --- /dev/null +++ b/src/domains/queue/sink/cleanup.rs @@ -0,0 +1,69 @@ +//! Disconnect cleanup and stale queued-request rejection state. +//! +//! `SessionCleanup` is delivered on the control-plane mailbox lane (see +//! `deliver_to_actor`'s `is_control_plane` check in `mailbox.rs`), so it can +//! pass an older, already-queued normal-lane request from the same session. +//! Remembering the cleaned-up session lets that stale request fail instead +//! of silently recreating a subscription or pending reserve for a session +//! that is already gone and will never be cleaned up again. + +use super::model::{Instant, QueueDomainCore}; + +impl QueueDomainCore { + pub(super) fn is_cleaned_up_session(&self, session_id: u64) -> bool { + self.cleaned_up_sessions.lock().contains(session_id) + } + + pub(super) fn mark_cleaned_up_session(&self, session_id: u64) { + self.cleaned_up_sessions.lock().mark(session_id); + } + + /// Drop all live queue inflight entries owned by the disconnected session and return + /// those accepted messages to the ready queue. Inflight ownership is + /// broker-local runtime state only. + pub(in crate::domains::queue::sink) fn cleanup_session(&self, session_id: u64) { + self.pending_reserves + .lock() + .retain(|pending| pending.meta.session_id != session_id); + let mut released_any = false; + let mut notifications = Vec::new(); + let mut actors = self.actors.lock(); + for (key, warm_actor) in actors.iter_mut() { + let mut actor = warm_actor.actor.lock(); + if actor.cleanup_session_inflight(session_id) > 0 { + released_any = true; + if let Some(notification) = self.record_ready_state(key, actor.live_counts()) { + notifications.push((key.clone(), notification)); + } + } + } + drop(actors); + + let mut families = self.families.lock(); + for (family_id, state) in families.iter_mut() { + state.remove_session( + crate::runtime::routing::RouteFamily::try_from(*family_id) + .expect("queue family IDs originate from RouteFamily"), + session_id, + ); + } + families.retain(|_, state| !state.is_empty()); + drop(families); + + if released_any { + self.mark_admin_snapshot_dirty(); + } + + for (key, notification) in notifications { + self.route_queue_ready_notification(&key, notification); + let route = Self::queue_ready_route(&key); + self.wake_pending_reserves_for_route(key.family, &route, Instant::now()); + } + + tracing::debug!( + domain = "queue", + session = session_id, + "Queue session cleanup completed" + ); + } +} diff --git a/src/domains/queue/sink/delivery.rs b/src/domains/queue/sink/delivery.rs new file mode 100644 index 00000000..cf5d5ed3 --- /dev/null +++ b/src/domains/queue/sink/delivery.rs @@ -0,0 +1,511 @@ +//! Ready-notification fan-out and per-operation actor dispatch. + +use super::model::{ + obs, Envelope, Instant, QueueDomainCore, QueueNotification, QueueReadyNotification, +}; +use crate::runtime::routing::RouteFamily; + +mod pending_reserves; +mod wildcard_receive; + +type ReadyNotificationEvent = (crate::domains::queue::QueueKey, QueueReadyNotification); + +#[derive(Clone, Copy)] +pub(super) struct OperationRequestContext<'a> { + pub(super) envelope: &'a Envelope, + pub(super) meta: crate::runtime::ClientFrameMeta, + pub(super) request_started: Option, +} + +pub(super) struct OperationOutcome { + pub(super) response: crate::domains::queue::QueueResponse, + pub(super) ready_notifications: Vec, + pub(super) mark_admin_snapshot_dirty: bool, +} + +#[derive(Clone, Copy)] +pub(super) struct ExtendOperation { + pub(super) session_id: u64, + pub(super) id: crate::domains::queue::MessageId, + pub(super) token: u64, + pub(super) inflight_seconds: u64, +} + +#[derive(Clone, Copy)] +pub(in crate::domains::queue::sink) enum QueueOpKind { + Send, + Receive, + Extend, + Ack, + InflightExpired, +} + +impl QueueDomainCore { + pub(super) fn queue_ready_route( + key: &crate::domains::queue::QueueKey, + ) -> crate::runtime::routing::Route { + crate::runtime::routing::Route::new(format!( + "queue://{}/{}/{}", + key.realm, key.area, key.resource + )) + } + + pub(super) fn route_queue_notify_to_subscription( + &self, + session_id: u64, + subscription_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + route: &crate::runtime::routing::Route, + counts: crate::domains::queue::QueueActorLiveCounts, + ) { + #[cfg(test)] + { + let payload = crate::dispatch::protocol::queue_codec::encode_notify( + subscription_id, + route, + QueueNotification { + ready_messages: counts.ready as u64, + delayed_messages: counts.delayed as u64, + inflight_messages: counts.inflight as u64, + }, + ); + let notify_ctx = super::model::FrameContext::new( + session_id, + crate::dispatch::protocol::frame::ChannelId::Sub, + crate::dispatch::protocol::tlv::MessageType::new( + crate::dispatch::protocol::queue_codec::msg_type::NOTIFY, + ), + bytes::Bytes::from(payload), + *subscriber.family(), + ); + let notify_envelope = Envelope::new(subscriber.clone(), notify_ctx); + if self.router.route(notify_envelope).is_err() { + crate::observability::counter_inc( + crate::domains::queue::metrics::METRIC_NOTIFY_DROPS_TOTAL, + ); + } + } + + #[cfg(not(test))] + { + let notification = crate::domains::queue::QueueClientNotification::new( + session_id, + *subscriber.family(), + subscription_id, + route.clone(), + QueueNotification { + ready_messages: counts.ready as u64, + delayed_messages: counts.delayed as u64, + inflight_messages: counts.inflight as u64, + }, + ); + let notify_envelope = Envelope::new(subscriber.clone(), notification); + if self.router.route(notify_envelope).is_err() { + crate::observability::counter_inc( + crate::domains::queue::metrics::METRIC_NOTIFY_DROPS_TOTAL, + ); + } + } + } + + pub(super) fn route_queue_ready_notification( + &self, + key: &crate::domains::queue::QueueKey, + notification: QueueReadyNotification, + ) { + let route = Self::queue_ready_route(key); + let targets = { + let families = self.families.lock(); + let mut targets = Vec::new(); + if let Some(state) = families.get(¬ification.family_id.as_u64()) { + state.for_each_matching_route( + notification.family_id, + route.as_str(), + |subscription| { + targets.push(( + subscription.session_id, + subscription.subscription_id, + subscription.subscriber.clone(), + )); + }, + ); + } + targets + }; + + for (session_id, subscription_id, subscriber) in targets { + self.route_queue_notify_to_subscription( + session_id, + subscription_id, + &subscriber, + &route, + notification.counts, + ); + } + } + + pub(super) fn emit_current_ready_notifications_for_watch( + &self, + family_id: crate::runtime::routing::RouteFamily, + pattern: &crate::runtime::matcher::Pattern, + session_id: u64, + subscription_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + ) { + let actors = self.actors.lock(); + let ready_snapshots: Vec<_> = actors + .iter() + .filter(|(key, _)| key.family == family_id) + .filter_map(|(key, warm_actor)| { + let counts = warm_actor.actor.lock().live_counts(); + let route = Self::queue_ready_route(key); + (counts.ready > 0 && pattern.matches(&route)).then_some((route, counts)) + }) + .collect(); + drop(actors); + + for (route, counts) in ready_snapshots { + self.route_queue_notify_to_subscription( + session_id, + subscription_id, + subscriber, + &route, + counts, + ); + } + } + + pub(super) fn dispatch_actor_operation( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + request_started: Option, + queue_msg: crate::domains::queue::protocol::QueueMessage, + ) -> Option { + let request_context = OperationRequestContext { + envelope, + meta, + request_started, + }; + let outcome = match queue_msg { + crate::domains::queue::protocol::QueueMessage::Send { + family_id, + route, + body, + delay_seconds, + } => self.handle_enqueue_operation( + family_id, + &route, + body, + delay_seconds, + request_context, + )?, + crate::domains::queue::protocol::QueueMessage::Receive { + family_id, + route, + inflight_seconds, + batch_size, + wait_seconds: _, + } => self.handle_receive_operation( + family_id, + &route, + meta.session_id, + inflight_seconds, + batch_size, + request_context, + )?, + crate::domains::queue::protocol::QueueMessage::Extend { + family_id, + route, + id, + token, + inflight_seconds, + } => self.handle_extend_operation( + family_id, + &route, + ExtendOperation { + session_id: meta.session_id, + id, + token, + inflight_seconds, + }, + request_context, + )?, + crate::domains::queue::protocol::QueueMessage::Ack { + family_id, + route, + id, + token, + } => self.handle_ack_operation( + family_id, + &route, + meta.session_id, + id, + token, + request_context, + )?, + crate::domains::queue::protocol::QueueMessage::InflightExpired { .. } => { + OperationOutcome { + response: crate::domains::queue::QueueResponse::Error { + message: "InflightExpired is an internal message".to_string(), + }, + ready_notifications: Vec::new(), + mark_admin_snapshot_dirty: false, + } + } + }; + + Some(outcome) + } + + fn handle_enqueue_operation( + &self, + family_id: RouteFamily, + route: &crate::runtime::routing::Route, + body: bytes::Bytes, + delay_seconds: Option, + request_context: OperationRequestContext<'_>, + ) -> Option { + let key = match Self::queue_key_for_route(family_id, route) { + Ok(key) => key, + Err(response) => { + return Some(OperationOutcome { + response, + ready_notifications: Vec::new(), + mark_admin_snapshot_dirty: false, + }); + } + }; + + self.with_actor_for_operation(&key, request_context, |actor| { + actor.handle_send(body, delay_seconds) + }) + .map(|(response, notification)| OperationOutcome { + response, + ready_notifications: notification.into_iter().collect(), + mark_admin_snapshot_dirty: true, + }) + } + + fn handle_receive_operation( + &self, + family_id: RouteFamily, + route: &crate::runtime::routing::Route, + session_id: u64, + inflight_seconds: u64, + batch_size: Option, + request_context: OperationRequestContext<'_>, + ) -> Option { + if let Ok(key) = Self::queue_key_for_route(family_id, route) { + return self + .with_actor_for_operation(&key, request_context, |actor| { + let mut response_bytes_remaining = + crate::domains::queue::protocol::MAX_QUEUE_RESPONSE_PAYLOAD_BYTES + - crate::domains::queue::protocol::RECEIVED_RESPONSE_HEADER_BYTES; + actor + .handle_receive_for_session_with_wire_budget( + session_id, + inflight_seconds, + batch_size, + &mut response_bytes_remaining, + crate::domains::queue::protocol::RESERVED_MESSAGE_WIRE_OVERHEAD_BYTES, + ) + .0 + }) + .map(|(response, notification)| OperationOutcome { + response, + ready_notifications: notification.into_iter().collect(), + mark_admin_snapshot_dirty: true, + }); + } + + let pattern = match Self::wildcard_queue_selector(route) { + Ok(pattern) => pattern, + Err(response) => { + return Some(OperationOutcome { + response, + ready_notifications: Vec::new(), + mark_admin_snapshot_dirty: false, + }); + } + }; + Some(self.handle_wildcard_receive( + family_id, + &pattern, + session_id, + inflight_seconds, + batch_size, + )) + } + + fn wildcard_queue_selector( + route: &crate::runtime::routing::Route, + ) -> Result { + if !route.as_str().contains('*') { + return Err(crate::domains::queue::QueueResponse::BadRequest { + reason: format!("invalid queue route: {}", route.as_str()), + }); + } + let pattern = crate::runtime::DomainKind::Queue + .descriptor() + .compile_registration_pattern(route.as_str()) + .map_err(|reason| crate::domains::queue::QueueResponse::BadRequest { reason })?; + if !pattern.is_wildcard() { + return Err(crate::domains::queue::QueueResponse::BadRequest { + reason: format!("invalid queue route: {}", route.as_str()), + }); + } + Ok(pattern) + } + + fn handle_extend_operation( + &self, + family_id: RouteFamily, + route: &crate::runtime::routing::Route, + extend: ExtendOperation, + request_context: OperationRequestContext<'_>, + ) -> Option { + let key = match Self::queue_key_for_route(family_id, route) { + Ok(key) => key, + Err(response) => { + return Some(OperationOutcome { + response, + ready_notifications: Vec::new(), + mark_admin_snapshot_dirty: false, + }); + } + }; + + self.with_actor_for_operation(&key, request_context, |actor| { + actor.handle_extend_for_session( + extend.session_id, + extend.id, + extend.token, + extend.inflight_seconds, + ) + }) + .map(|(response, notification)| OperationOutcome { + response, + ready_notifications: notification.into_iter().collect(), + mark_admin_snapshot_dirty: true, + }) + } + + fn handle_ack_operation( + &self, + family_id: RouteFamily, + route: &crate::runtime::routing::Route, + session_id: u64, + id: crate::domains::queue::MessageId, + token: u64, + request_context: OperationRequestContext<'_>, + ) -> Option { + let key = match Self::queue_key_for_route(family_id, route) { + Ok(key) => key, + Err(response) => { + return Some(OperationOutcome { + response, + ready_notifications: Vec::new(), + mark_admin_snapshot_dirty: false, + }); + } + }; + + self.with_actor_for_operation(&key, request_context, |actor| { + actor.handle_ack_for_session(session_id, id, token) + }) + .map(|(response, notification)| OperationOutcome { + response, + ready_notifications: notification.into_iter().collect(), + mark_admin_snapshot_dirty: true, + }) + } + + fn with_actor_for_operation( + &self, + key: &crate::domains::queue::QueueKey, + request_context: OperationRequestContext<'_>, + operation: F, + ) -> Option<( + crate::domains::queue::QueueResponse, + Option, + )> + where + F: FnOnce(&mut crate::domains::queue::QueueActor) -> crate::domains::queue::QueueResponse, + { + let actor_lock_start = Instant::now(); + let (actor_handle, _) = match self.get_or_create_actor(key) { + Ok(actor) => actor, + Err(message) => { + self.route_queue_recovery_error( + request_context.envelope, + request_context.meta, + request_context.request_started, + message, + ); + return None; + } + }; + self.observe_histogram_us( + obs::METRIC_QUEUE_ACTOR_LOCK_HOLD_LATENCY, + Self::u128_to_u64_saturating(actor_lock_start.elapsed().as_micros()), + ); + + let mut actor = actor_handle.lock(); + let actor_exec_start = Instant::now(); + actor.process_due_work(); + let response = operation(&mut actor); + let counts = actor.live_counts(); + if counts.total() > 0 { + self.known_queue_keys.lock().insert(key.clone()); + } + let notification = self.record_ready_state(key, counts); + self.observe_histogram_us( + obs::METRIC_QUEUE_ACTOR_EXECUTION_LATENCY, + Self::u128_to_u64_saturating(actor_exec_start.elapsed().as_micros()), + ); + + Some((response, notification.map(|event| (key.clone(), event)))) + } + + pub(super) fn classify_operation( + queue_msg: &crate::domains::queue::protocol::QueueMessage, + ) -> QueueOpKind { + match queue_msg { + crate::domains::queue::protocol::QueueMessage::Send { .. } => QueueOpKind::Send, + crate::domains::queue::protocol::QueueMessage::Receive { .. } => QueueOpKind::Receive, + crate::domains::queue::protocol::QueueMessage::Extend { .. } => QueueOpKind::Extend, + crate::domains::queue::protocol::QueueMessage::Ack { .. } => QueueOpKind::Ack, + crate::domains::queue::protocol::QueueMessage::InflightExpired { .. } => { + QueueOpKind::InflightExpired + } + } + } + + pub(super) fn record_operation_metrics( + &self, + request_started: Option, + response: &crate::domains::queue::QueueResponse, + op_kind: QueueOpKind, + ) { + if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { + if Self::queue_response_is_failure(response) { + metrics.record_failure(started_at); + return; + } + + metrics.record_success(started_at); + match op_kind { + QueueOpKind::Send => metrics.record_enqueue(started_at), + QueueOpKind::Receive => metrics.record_reserve(started_at), + QueueOpKind::Ack => metrics.record_complete(), + QueueOpKind::Extend => metrics.record_extend(), + QueueOpKind::InflightExpired => {} + } + } + } + + fn u128_to_u64_saturating(value: u128) -> u64 { + value.try_into().unwrap_or(u64::MAX) + } +} diff --git a/src/domains/queue/sink/mailbox_sink_impl/pending_reserves.rs b/src/domains/queue/sink/delivery/pending_reserves.rs similarity index 97% rename from src/domains/queue/sink/mailbox_sink_impl/pending_reserves.rs rename to src/domains/queue/sink/delivery/pending_reserves.rs index c6a66142..ddca3173 100644 --- a/src/domains/queue/sink/mailbox_sink_impl/pending_reserves.rs +++ b/src/domains/queue/sink/delivery/pending_reserves.rs @@ -1,6 +1,5 @@ -use super::{ - Instant, OperationOutcome, PendingQueueReserve, QueueDomainCore, QueueOpKind, VecDeque, -}; +use super::{Instant, OperationOutcome, QueueDomainCore, QueueOpKind}; +use crate::domains::queue::sink::model::{PendingQueueReserve, VecDeque}; use crate::runtime::routing::RouteFamily; impl QueueDomainCore { diff --git a/src/domains/queue/sink/mailbox_sink_impl/wildcard_receive.rs b/src/domains/queue/sink/delivery/wildcard_receive.rs similarity index 63% rename from src/domains/queue/sink/mailbox_sink_impl/wildcard_receive.rs rename to src/domains/queue/sink/delivery/wildcard_receive.rs index 07bdcb66..1d8b0d21 100644 --- a/src/domains/queue/sink/mailbox_sink_impl/wildcard_receive.rs +++ b/src/domains/queue/sink/delivery/wildcard_receive.rs @@ -2,6 +2,53 @@ use super::{OperationOutcome, QueueDomainCore}; use crate::runtime::routing::RouteFamily; use std::sync::atomic::Ordering; +use crate::domains::queue::protocol::{ + MAX_QUEUE_RESPONSE_PAYLOAD_BYTES, RECEIVED_RESPONSE_HEADER_BYTES, + RESERVED_MESSAGE_WIRE_OVERHEAD_BYTES, ROUTED_MESSAGE_WIRE_OVERHEAD_BYTES, +}; + +fn receive_with_route_wire_budget( + actor: &mut crate::domains::queue::QueueActor, + session_id: u64, + inflight_seconds: u64, + batch_size: usize, + response_bytes_remaining: &mut usize, + route: &crate::runtime::routing::Route, +) -> (crate::domains::queue::QueueResponse, bool) { + let message_wire_overhead_bytes = RESERVED_MESSAGE_WIRE_OVERHEAD_BYTES + .saturating_add(ROUTED_MESSAGE_WIRE_OVERHEAD_BYTES) + .saturating_add(route.as_str().len()); + actor.handle_receive_for_session_with_wire_budget( + session_id, + inflight_seconds, + Some(batch_size), + response_bytes_remaining, + message_wire_overhead_bytes, + ) +} + +fn empty_wildcard_receive_outcome() -> OperationOutcome { + OperationOutcome { + response: crate::domains::queue::QueueResponse::ReceivedRouted { + messages: Vec::new(), + }, + ready_notifications: Vec::new(), + mark_admin_snapshot_dirty: false, + } +} + +fn route_reserved_messages( + route: crate::runtime::routing::Route, + messages: Vec, +) -> impl Iterator { + messages.into_iter().map( + move |message| crate::domains::queue::RoutedReservedMessage { + route: route.clone(), + message, + }, + ) +} + impl QueueDomainCore { const MAX_WILDCARD_RESERVE_MATCHES: usize = 4096; @@ -53,13 +100,7 @@ impl QueueDomainCore { let keys = self.matching_queue_keys(family_id, pattern); let limit = batch_size.unwrap_or(1); if keys.is_empty() || limit == 0 { - return OperationOutcome { - response: crate::domains::queue::QueueResponse::ReceivedRouted { - messages: Vec::new(), - }, - ready_notifications: Vec::new(), - mark_admin_snapshot_dirty: false, - }; + return empty_wildcard_receive_outcome(); } let start = usize::try_from( @@ -71,6 +112,8 @@ impl QueueDomainCore { let mut routed = Vec::with_capacity(limit); let mut notifications = Vec::new(); let mut state_changed = false; + let mut response_bytes_remaining = + MAX_QUEUE_RESPONSE_PAYLOAD_BYTES - RECEIVED_RESPONSE_HEADER_BYTES; for offset in 0..keys.len() { if routed.len() == limit { @@ -97,29 +140,35 @@ impl QueueDomainCore { break; } }; - let response = { + let (response, wire_budget_exhausted) = { let mut actor = actor_handle.lock(); + let counts_before = actor.live_counts(); state_changed |= actor.process_due_work(); let remaining = limit - routed.len(); - let response = - actor.handle_receive_for_session(session_id, inflight_seconds, Some(remaining)); + let (response, wire_budget_exhausted) = receive_with_route_wire_budget( + &mut actor, + session_id, + inflight_seconds, + remaining, + &mut response_bytes_remaining, + &route, + ); let counts = actor.live_counts(); + state_changed |= counts != counts_before; if counts.total() > 0 { self.known_queue_keys.lock().insert(key.clone()); } if let Some(notification) = self.record_ready_state(key, counts) { notifications.push((key.clone(), notification)); } - response + (response, wire_budget_exhausted) }; match response { crate::domains::queue::QueueResponse::Received { messages } => { - routed.extend(messages.into_iter().map(|message| { - crate::domains::queue::RoutedReservedMessage { - route: route.clone(), - message, - } - })); + routed.extend(route_reserved_messages(route.clone(), messages)); + if wire_budget_exhausted { + break; + } } error if routed.is_empty() => { return OperationOutcome { @@ -146,4 +195,17 @@ impl QueueDomainCore { ready_notifications: notifications, } } + + #[cfg(test)] + pub(in crate::domains::queue::sink) fn handle_wildcard_receive_for_tests( + &self, + family_id: RouteFamily, + pattern: &crate::runtime::matcher::Pattern, + session_id: u64, + inflight_seconds: u64, + batch_size: Option, + ) -> crate::domains::queue::QueueResponse { + self.handle_wildcard_receive(family_id, pattern, session_id, inflight_seconds, batch_size) + .response + } } diff --git a/src/domains/queue/sink/domain_sink_impl/domain_core_impl.rs b/src/domains/queue/sink/domain_sink_impl/domain_core_impl.rs deleted file mode 100644 index 9f42759a..00000000 --- a/src/domains/queue/sink/domain_sink_impl/domain_core_impl.rs +++ /dev/null @@ -1,822 +0,0 @@ -#[cfg(test)] -use super::FrameContext; -use super::{ - Arc, Envelope, HashSet, Instant, Mutex, QueueAdminPlane, QueueDomainCore, QueueLiveCounts, - QueueNotification, QueueProjectionEntry, QueueProjectionState, QueueReadyNotification, - WarmQueueActor, QUEUE_ACTOR_IDLE_TTL, QUEUE_DEDUP_SWEEP_INTERVAL, QUEUE_IDLE_SWEEP_BATCH_SIZE, - QUEUE_IDLE_SWEEP_INTERVAL, -}; - -impl QueueDomainCore { - pub(in crate::domains::queue::sink) fn queue_key_for_route( - family_id: crate::runtime::routing::RouteFamily, - route: &crate::runtime::routing::Route, - ) -> Result { - crate::domains::queue::QueueKey::from_route(family_id, route).ok_or_else(|| { - crate::domains::queue::QueueResponse::BadRequest { - reason: format!("invalid queue route: {}", route.as_str()), - } - }) - } - - #[cfg(test)] - pub(in crate::domains::queue::sink) fn session_inbox_address( - family_id: crate::runtime::routing::RouteFamily, - session_id: u64, - ) -> crate::runtime::routing::RouteAddress { - crate::runtime::routing::RouteAddress::new( - family_id, - crate::runtime::routing::Route::new(format!("inbox://session/{session_id}")), - ) - } - - pub(in crate::domains::queue::sink) fn route_queue_response( - &self, - request_envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - response: &crate::domains::queue::QueueResponse, - ) { - #[cfg(test)] - { - let response_bytes = crate::dispatch::protocol::queue_codec::encode_response( - meta.message_type, - response, - ); - let response_ctx = FrameContext::new( - meta.session_id, - test_protocol_channel_from_client(meta.channel), - crate::dispatch::protocol::tlv::MessageType::new(meta.message_type), - bytes::Bytes::from(response_bytes), - meta.route_family, - ); - if let Some(response_envelope) = request_envelope.try_reply_to(response_ctx) { - if let Err(error) = self.router.route(response_envelope) { - tracing::warn!( - domain = "queue", - session = meta.session_id, - error = ?error, - "Failed to route queue response" - ); - } - } - } - - #[cfg(not(test))] - { - let response = crate::domains::queue::QueueClientResponse::new(meta, response.clone()); - if let Some(response_envelope) = request_envelope.try_reply_to(response) { - if let Err(error) = self.router.route(response_envelope) { - tracing::warn!( - domain = "queue", - session = meta.session_id, - error = ?error, - "Failed to route queue response" - ); - } - } - } - } - - pub(in crate::domains::queue::sink) fn route_queue_recovery_error( - &self, - request_envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - request_started: Option, - message: String, - ) { - tracing::error!( - domain = "queue", - family = meta.route_family.as_u64(), - error = %message, - "Queue actor recovery failed" - ); - let response = crate::domains::queue::QueueResponse::Error { message }; - self.route_queue_response(request_envelope, meta, &response); - if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { - metrics.record_failure(started_at); - } - } - - pub(in crate::domains::queue::sink) fn queue_ready_route( - key: &crate::domains::queue::QueueKey, - ) -> crate::runtime::routing::Route { - crate::runtime::routing::Route::new(format!( - "queue://{}/{}/{}", - key.realm, key.area, key.resource - )) - } - - pub(in crate::domains::queue::sink) fn matching_queue_keys( - &self, - family: crate::runtime::routing::RouteFamily, - pattern: &crate::runtime::matcher::Pattern, - ) -> Vec { - let mut keys = self - .known_queue_keys - .lock() - .iter() - .filter(|key| key.family == family) - .filter(|key| pattern.matches(&Self::queue_ready_route(key))) - .cloned() - .collect::>(); - keys.sort_by(|left, right| { - (&left.realm, &left.area, &left.resource).cmp(&( - &right.realm, - &right.area, - &right.resource, - )) - }); - keys - } - - pub(in crate::domains::queue::sink) fn matching_queue_key_count( - &self, - family: crate::runtime::routing::RouteFamily, - pattern: &crate::runtime::matcher::Pattern, - ) -> usize { - self.known_queue_keys - .lock() - .iter() - .filter(|key| key.family == family) - .filter(|key| pattern.matches(&Self::queue_ready_route(key))) - .count() - } - - pub(in crate::domains::queue::sink) fn inventory_existing_queue_keys( - store: &crate::storage::FitzStorageEngine, - ) -> Result, String> { - let families = store - .list_column_families() - .map_err(|error| format!("list queue inventory families failed: {error:?}"))?; - let mut known_queue_keys = HashSet::new(); - - for family in families { - if family.id() == 0 { - continue; - } - let route_family = crate::runtime::routing::RouteFamily::new(family.id()); - let txn = store - .begin_tx(family.id(), cntryl_midge::TransactionMode::ReadOnly) - .map_err(|error| { - format!( - "queue inventory transaction failed: family={} error={error:?}", - family.id() - ) - })?; - let rows = txn.scan(&cntryl_midge::Query::new()).map_err(|error| { - format!( - "queue inventory scan failed: family={} error={error:?}", - family.id() - ) - })?; - - for row in rows { - let (key, value) = row.map_err(|error| { - format!( - "queue inventory scan failed: family={} error={error:?}", - family.id() - ) - })?; - drop(value); - if let Some(queue_key) = - crate::domains::queue::QueueActor::queue_key_from_authoritative_storage_key( - route_family, - &key, - ) - { - known_queue_keys.insert(queue_key); - } - } - } - - Ok(known_queue_keys) - } - - pub(in crate::domains::queue::sink) fn record_ready_state( - &self, - key: &crate::domains::queue::QueueKey, - counts: crate::domains::queue::QueueActorLiveCounts, - ) -> Option { - let is_ready = counts.ready > 0; - let mut ready_states = self.ready_states.lock(); - let was_ready = ready_states.get(key).copied().unwrap_or(false); - - if counts.total() == 0 { - ready_states.remove(key); - } else { - ready_states.insert(key.clone(), is_ready); - } - - if !was_ready && is_ready { - Some(QueueReadyNotification { - family_id: key.family, - counts, - }) - } else { - None - } - } - - pub(in crate::domains::queue::sink) fn route_queue_notify_to_subscription( - &self, - session_id: u64, - subscription_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - route: &crate::runtime::routing::Route, - counts: crate::domains::queue::QueueActorLiveCounts, - ) { - #[cfg(test)] - { - let payload = crate::dispatch::protocol::queue_codec::encode_notify( - subscription_id, - route, - QueueNotification { - ready_messages: counts.ready as u64, - delayed_messages: counts.delayed as u64, - inflight_messages: counts.inflight as u64, - }, - ); - let notify_ctx = FrameContext::new( - session_id, - crate::dispatch::protocol::frame::ChannelId::Sub, - crate::dispatch::protocol::tlv::MessageType::new( - crate::dispatch::protocol::queue_codec::msg_type::NOTIFY, - ), - bytes::Bytes::from(payload), - *subscriber.family(), - ); - let notify_envelope = Envelope::new(subscriber.clone(), notify_ctx); - if self.router.route(notify_envelope).is_err() { - crate::observability::counter_inc( - crate::domains::queue::metrics::METRIC_NOTIFY_DROPS_TOTAL, - ); - } - } - - #[cfg(not(test))] - { - let notification = crate::domains::queue::QueueClientNotification::new( - session_id, - *subscriber.family(), - subscription_id, - route.clone(), - QueueNotification { - ready_messages: counts.ready as u64, - delayed_messages: counts.delayed as u64, - inflight_messages: counts.inflight as u64, - }, - ); - let notify_envelope = Envelope::new(subscriber.clone(), notification); - if self.router.route(notify_envelope).is_err() { - crate::observability::counter_inc( - crate::domains::queue::metrics::METRIC_NOTIFY_DROPS_TOTAL, - ); - } - } - } - - pub(in crate::domains::queue::sink) fn route_queue_ready_notification( - &self, - key: &crate::domains::queue::QueueKey, - notification: QueueReadyNotification, - ) { - let route = Self::queue_ready_route(key); - let targets = { - let families = self.families.lock(); - let mut targets = Vec::new(); - if let Some(state) = families.get(¬ification.family_id.as_u64()) { - state.for_each_matching_route( - notification.family_id, - route.as_str(), - |subscription| { - targets.push(( - subscription.session_id, - subscription.subscription_id, - subscription.subscriber.clone(), - )); - }, - ); - } - targets - }; - - for (session_id, subscription_id, subscriber) in targets { - self.route_queue_notify_to_subscription( - session_id, - subscription_id, - &subscriber, - &route, - notification.counts, - ); - } - } - - pub(in crate::domains::queue::sink) fn emit_current_ready_notifications_for_watch( - &self, - family_id: crate::runtime::routing::RouteFamily, - pattern: &crate::runtime::matcher::Pattern, - session_id: u64, - subscription_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - ) { - let actors = self.actors.lock(); - let ready_snapshots: Vec<_> = actors - .iter() - .filter(|(key, _)| key.family == family_id) - .filter_map(|(key, warm_actor)| { - let counts = warm_actor.actor.lock().live_counts(); - let route = Self::queue_ready_route(key); - (counts.ready > 0 && pattern.matches(&route)).then_some((route, counts)) - }) - .collect(); - drop(actors); - - for (route, counts) in ready_snapshots { - self.route_queue_notify_to_subscription( - session_id, - subscription_id, - subscriber, - &route, - counts, - ); - } - } - - pub(in crate::domains::queue::sink) fn sweep_runtime_state_at(&self, now: Instant) { - self.expire_pending_reserves_at(now); - self.sweep_idle_actors_at(now); - self.maybe_cleanup_dedup_at(now); - self.maybe_flush_dirty_fast_families_at(now); - } - - pub(in crate::domains::queue::sink) fn fast_flush_enabled(&self) -> bool { - self.queue_write_options.is_best_effort() && self.fast_flush_interval.is_some() - } - - pub(in crate::domains::queue::sink) fn mark_fast_flush_dirty( - &self, - family_id: crate::runtime::routing::RouteFamily, - ) { - if self.fast_flush_enabled() { - self.dirty_fast_flush_families.lock().insert(family_id.id()); - } - } - - pub(in crate::domains::queue::sink) fn maybe_flush_dirty_fast_families_at(&self, now: Instant) { - let Some(interval) = self.fast_flush_interval else { - return; - }; - if !self.queue_write_options.is_best_effort() { - return; - } - - let should_flush = { - let mut next_fast_flush_at = self.next_fast_flush_at.lock(); - if now < *next_fast_flush_at { - false - } else { - *next_fast_flush_at = now + interval; - true - } - }; - - if should_flush { - self.flush_dirty_fast_families(); - } - } - - pub(in crate::domains::queue::sink) fn flush_dirty_fast_families(&self) { - let dirty_family_ids = { - let mut dirty = self.dirty_fast_flush_families.lock(); - dirty.drain().collect::>() - }; - if dirty_family_ids.is_empty() { - return; - } - - let families = match self.store.list_column_families() { - Ok(families) => families, - Err(error) => { - tracing::warn!( - domain = "queue", - error = ?error, - "Failed to list queue column families for fast flush" - ); - self.dirty_fast_flush_families - .lock() - .extend(dirty_family_ids); - return; - } - }; - - let mut retry_family_ids = Vec::new(); - for family_id in dirty_family_ids { - let Some(cf) = families.iter().find(|cf| cf.id() == family_id) else { - tracing::warn!( - domain = "queue", - family = family_id, - "Queue fast flush skipped missing column family" - ); - retry_family_ids.push(family_id); - continue; - }; - - if let Err(error) = self.store.flush_cf(cf) { - tracing::warn!( - domain = "queue", - family = family_id, - error = ?error, - "Queue fast flush failed" - ); - retry_family_ids.push(family_id); - } - } - - if !retry_family_ids.is_empty() { - self.dirty_fast_flush_families - .lock() - .extend(retry_family_ids); - } - } - - pub(in crate::domains::queue::sink) fn maybe_cleanup_dedup_at(&self, now: Instant) { - let should_cleanup = { - let mut next_dedup_sweep_at = self.next_dedup_sweep_at.lock(); - if now < *next_dedup_sweep_at { - false - } else { - *next_dedup_sweep_at = now + QUEUE_DEDUP_SWEEP_INTERVAL; - true - } - }; - - if should_cleanup { - self.dedup_store.cleanup(); - } - } - - pub(in crate::domains::queue::sink) fn get_or_create_actor( - &self, - key: &crate::domains::queue::QueueKey, - ) -> Result<(Arc>, bool), String> { - use std::collections::hash_map::Entry; - - let now = Instant::now(); - match self.actors.lock().entry(key.clone()) { - Entry::Occupied(mut entry) => { - entry.get_mut().last_used = now; - Ok((entry.get().actor.clone(), false)) - } - Entry::Vacant(entry) => { - let actor = Arc::new(Mutex::new( - crate::domains::queue::QueueActor::try_new_with_write_options( - key.family, - key.clone(), - self.store.clone_inner(), - None, - self.dedup_store.clone(), - self.queue_write_options, - )?, - )); - entry.insert(WarmQueueActor { - actor: actor.clone(), - last_used: now, - }); - self.idle_sweep_keys.lock().push_back(key.clone()); - Ok((actor, true)) - } - } - } - - pub(in crate::domains::queue::sink) fn mark_admin_snapshot_dirty(&self) { - self.projection.mark_dirty(); - self.refresh_metrics_gauges(); - } - - pub(in crate::domains::queue::sink) fn refresh_metrics_gauges(&self) { - if let Some(metrics) = &self.metrics { - let counts = self.live_counts(); - metrics.set_ready_messages(counts.ready); - metrics.set_delayed_messages(counts.delayed); - metrics.set_inflight_messages(counts.inflight); - } - } - - pub(in crate::domains::queue::sink) fn observe_histogram_us(&self, name: &str, value_us: u64) { - if let Some(metrics) = &self.metrics { - metrics.histogram_observe_us(name, value_us); - } else { - crate::observability::histogram_observe_us(name, value_us); - } - } - - pub(in crate::domains::queue::sink) fn queue_response_is_failure( - response: &crate::domains::queue::QueueResponse, - ) -> bool { - matches!( - response, - crate::domains::queue::QueueResponse::InvalidToken - | crate::domains::queue::QueueResponse::InflightExpired - | crate::domains::queue::QueueResponse::NotFound - | crate::domains::queue::QueueResponse::QueueNotFound - | crate::domains::queue::QueueResponse::BadRequest { .. } - | crate::domains::queue::QueueResponse::Error { .. } - ) - } - - pub(in crate::domains::queue::sink) fn refresh_admin_snapshot_if_dirty(&self) { - self.sweep_idle_actors(); - self.projection - .refresh_if_dirty(|| self.collect_projection_state()); - } - - pub(in crate::domains::queue::sink) fn collect_projection_state(&self) -> QueueProjectionState { - let actors = self.actors.lock(); - let families = self.families.lock(); - let entries = actors - .iter() - .map(|(key, warm_actor)| { - let actor = warm_actor.actor.lock(); - let ready_route = Self::queue_ready_route(key); - let subscriptions_active = families.get(&key.family.as_u64()).map_or(0, |state| { - state.for_each_matching_route(key.family, ready_route.as_str(), |_| {}) - }); - QueueProjectionEntry { - key: key.clone(), - snapshot: QueueAdminPlane::admin_snapshot(&*actor), - subscriptions_active, - inflight: QueueAdminPlane::admin_inflight(&*actor), - dead_letters: QueueAdminPlane::admin_dead_letters(&*actor), - } - }) - .collect(); - - QueueProjectionState::from_entries(entries) - } - - pub(in crate::domains::queue::sink) fn sweep_idle_actors(&self) { - self.sweep_idle_actors_at(Instant::now()); - } - - pub(in crate::domains::queue::sink) fn maybe_sweep_idle_actors(&self) { - let now = Instant::now(); - - { - let mut next_idle_sweep_at = self.next_idle_sweep_at.lock(); - if now < *next_idle_sweep_at { - return; - } - *next_idle_sweep_at = now + QUEUE_IDLE_SWEEP_INTERVAL; - } - - self.sweep_idle_actors_at(now); - } - - pub(in crate::domains::queue::sink) fn sweep_idle_actors_at(&self, now: Instant) { - let mut changed = false; - let mut notifications = Vec::new(); - let mut removed_keys = Vec::new(); - let mut empty_removed_keys = Vec::new(); - let mut dirty_families = HashSet::new(); - let sweep_keys = { - let mut idle_sweep_keys = self.idle_sweep_keys.lock(); - let count = idle_sweep_keys.len().min(QUEUE_IDLE_SWEEP_BATCH_SIZE); - idle_sweep_keys.drain(..count).collect::>() - }; - - for key in sweep_keys { - let Some((actor_ref, last_used)) = self - .actors - .lock() - .get(&key) - .map(|warm_actor| (warm_actor.actor.clone(), warm_actor.last_used)) - else { - continue; - }; - let mut actor = actor_ref.lock(); - if actor.process_due_work() { - changed = true; - dirty_families.insert(key.family); - } - let counts = actor.live_counts(); - - if let Some(notification) = self.record_ready_state(&key, counts) { - notifications.push((key.clone(), notification)); - } - - let idle_for = now.saturating_duration_since(last_used); - let should_keep = - idle_for < QUEUE_ACTOR_IDLE_TTL || counts.delayed > 0 || counts.inflight > 0; - drop(actor); - - if should_keep { - self.idle_sweep_keys.lock().push_back(key); - continue; - } - - let removed = { - let mut actors = self.actors.lock(); - let unchanged = actors.get(&key).is_some_and(|warm_actor| { - warm_actor.last_used == last_used && Arc::ptr_eq(&warm_actor.actor, &actor_ref) - }); - unchanged && actors.remove(&key).is_some() - }; - if removed { - changed = true; - removed_keys.push(key.clone()); - if counts.total() == 0 { - empty_removed_keys.push(key); - } - } else { - self.idle_sweep_keys.lock().push_back(key); - } - } - - if !removed_keys.is_empty() { - let mut ready_states = self.ready_states.lock(); - for key in removed_keys { - ready_states.remove(&key); - } - } - if !empty_removed_keys.is_empty() { - let mut known_queue_keys = self.known_queue_keys.lock(); - for key in empty_removed_keys { - known_queue_keys.remove(&key); - } - } - for family in dirty_families { - self.mark_fast_flush_dirty(family); - } - if changed { - self.mark_admin_snapshot_dirty(); - } - for (key, notification) in notifications { - self.route_queue_ready_notification(&key, notification); - let route = Self::queue_ready_route(&key); - self.wake_pending_reserves_for_route(key.family, &route, now); - } - } - - /// Drop all live queue inflight entries owned by the disconnected session and return - /// those accepted messages to the ready queue. Inflight ownership is - /// broker-local runtime state only. - pub(in crate::domains::queue::sink) fn cleanup_session(&self, session_id: u64) { - self.pending_reserves - .lock() - .retain(|pending| pending.meta.session_id != session_id); - let mut released_any = false; - let mut notifications = Vec::new(); - let mut actors = self.actors.lock(); - for (key, warm_actor) in actors.iter_mut() { - let mut actor = warm_actor.actor.lock(); - if actor.cleanup_session_inflight(session_id) > 0 { - released_any = true; - if let Some(notification) = self.record_ready_state(key, actor.live_counts()) { - notifications.push((key.clone(), notification)); - } - } - } - drop(actors); - - let mut families = self.families.lock(); - for (family_id, state) in families.iter_mut() { - state.remove_session( - crate::runtime::routing::RouteFamily::try_from(*family_id) - .expect("queue family IDs originate from RouteFamily"), - session_id, - ); - } - families.retain(|_, state| !state.is_empty()); - drop(families); - - if released_any { - self.mark_admin_snapshot_dirty(); - } - - for (key, notification) in notifications { - self.route_queue_ready_notification(&key, notification); - let route = Self::queue_ready_route(&key); - self.wake_pending_reserves_for_route(key.family, &route, Instant::now()); - } - - tracing::debug!( - domain = "queue", - session = session_id, - "Queue session cleanup completed" - ); - } - - pub(in crate::domains::queue::sink) fn live_counts(&self) -> QueueLiveCounts { - let actors = self.actors.lock(); - let mut counts = QueueLiveCounts::default(); - - for warm_actor in actors.values() { - let actor_counts = warm_actor.actor.lock().live_counts(); - counts.ready = counts.ready.saturating_add(actor_counts.ready); - counts.delayed = counts.delayed.saturating_add(actor_counts.delayed); - counts.inflight = counts.inflight.saturating_add(actor_counts.inflight); - counts.dead_letters = counts - .dead_letters - .saturating_add(actor_counts.dead_letters); - } - counts.pending = counts.ready.saturating_add(counts.delayed); - counts - } - - /// Replays a dead-lettered message back into its queue. - /// - /// # Errors - /// - /// Returns an error when the warm queue actor cannot be recovered or the replay fails. - pub(in crate::domains::queue::sink) fn replay_dead_letter( - &self, - key: &crate::domains::queue::QueueKey, - id: crate::domains::queue::MessageId, - ) -> Result { - let (actor_handle, created_actor) = self.get_or_create_actor(key)?; - let result = { - let mut actor = actor_handle.lock(); - QueueAdminPlane::replay_dead_letter(&mut *actor, id) - }; - - if matches!(result, Ok(true)) { - self.mark_fast_flush_dirty(key.family); - let counts = actor_handle.lock().live_counts(); - let notification = self.record_ready_state(key, counts); - self.mark_admin_snapshot_dirty(); - if let Some(notification) = notification { - self.route_queue_ready_notification(key, notification); - } - } - - if created_actor { - let should_remove = { - let actor = actor_handle.lock(); - actor.live_counts().total() == 0 - }; - if should_remove { - self.actors.lock().remove(key); - self.ready_states.lock().remove(key); - self.known_queue_keys.lock().remove(key); - self.mark_admin_snapshot_dirty(); - } - } - - result - } - - /// Permanently removes a dead-lettered message from its queue. - /// - /// # Errors - /// - /// Returns an error when the warm queue actor cannot be recovered or the purge fails. - pub(in crate::domains::queue::sink) fn purge_dead_letter( - &self, - key: &crate::domains::queue::QueueKey, - id: crate::domains::queue::MessageId, - ) -> Result { - let (actor_handle, created_actor) = self.get_or_create_actor(key)?; - let result = { - let mut actor = actor_handle.lock(); - QueueAdminPlane::purge_dead_letter(&mut *actor, id) - }; - - if matches!(result, Ok(true)) { - self.mark_fast_flush_dirty(key.family); - self.mark_admin_snapshot_dirty(); - } - - if created_actor { - let should_remove = { - let actor = actor_handle.lock(); - actor.live_counts().total() == 0 - }; - if should_remove { - self.actors.lock().remove(key); - self.ready_states.lock().remove(key); - self.known_queue_keys.lock().remove(key); - self.mark_admin_snapshot_dirty(); - } - } - - result - } -} - -#[cfg(test)] -fn test_protocol_channel_from_client( - channel: crate::runtime::ClientChannel, -) -> crate::dispatch::protocol::frame::ChannelId { - match channel { - crate::runtime::ClientChannel::Control => { - crate::dispatch::protocol::frame::ChannelId::Control - } - crate::runtime::ClientChannel::Pub => crate::dispatch::protocol::frame::ChannelId::Pub, - crate::runtime::ClientChannel::Sub => crate::dispatch::protocol::frame::ChannelId::Sub, - crate::runtime::ClientChannel::Rpc => crate::dispatch::protocol::frame::ChannelId::Rpc, - crate::runtime::ClientChannel::Lease => crate::dispatch::protocol::frame::ChannelId::Lease, - crate::runtime::ClientChannel::Internal => { - crate::dispatch::protocol::frame::ChannelId::Internal - } - } -} diff --git a/src/domains/queue/sink/domain_sink_impl.rs b/src/domains/queue/sink/facade.rs similarity index 80% rename from src/domains/queue/sink/domain_sink_impl.rs rename to src/domains/queue/sink/facade.rs index 1c5a3e53..6465ac4f 100644 --- a/src/domains/queue/sink/domain_sink_impl.rs +++ b/src/domains/queue/sink/facade.rs @@ -1,18 +1,13 @@ use super::model::{ - AtomicBool, AtomicU64, Duration, Envelope, HashMap, HashSet, Instant, Mutex, Ordering, + AtomicBool, AtomicU64, Duration, HashMap, HashSet, Instant, Mutex, Ordering, QueueAdminProjection, QueueDomainActor, QueueDomainCommand, QueueDomainCore, - QueueDomainRuntime, QueueDomainSink, QueueLiveCounts, QueueMetrics, QueueNotification, - QueueProjectionEntry, QueueProjectionState, QueueReadyNotification, Router, WarmQueueActor, - QUEUE_ACTOR_IDLE_TTL, QUEUE_ACTOR_REPLY_TIMEOUT, QUEUE_DEDUP_SWEEP_INTERVAL, - QUEUE_IDLE_SWEEP_BATCH_SIZE, QUEUE_IDLE_SWEEP_INTERVAL, + QueueDomainRuntime, QueueDomainSink, QueueLiveCounts, QueueMetrics, Router, + QUEUE_ACTOR_REPLY_TIMEOUT, }; #[cfg(test)] -use crate::dispatch::protocol::frame_context::FrameContext; -use crate::domains::queue::actor::QueueAdminPlane; +use super::model::{WarmQueueActor, QUEUE_ACTOR_IDLE_TTL}; use std::{collections::VecDeque, sync::Arc}; -mod domain_core_impl; - #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct QueueCounts { pub pending: usize, @@ -149,6 +144,9 @@ impl QueueDomainSink { inventory_error: Option, ) -> Self { let core = Arc::new(QueueDomainCore { + delivery_service_us: Arc::new(std::sync::atomic::AtomicU64::new( + super::model::assumed_service_us(), + )), store, queue_write_options, dedup_store, @@ -158,6 +156,9 @@ impl QueueDomainSink { inventory_error: Mutex::new(inventory_error), wildcard_reserve_sequence: AtomicU64::new(0), families: Mutex::new(HashMap::new()), + cleaned_up_sessions: Mutex::new(crate::runtime::CleanedUpSessions::new( + crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, + )), next_sub_id: AtomicU64::new(1), ready_states: Mutex::new(HashMap::new()), pending_reserves: Mutex::new(VecDeque::default()), @@ -165,6 +166,9 @@ impl QueueDomainSink { projection: QueueAdminProjection::new(admin_read_model), metrics: None, active: AtomicBool::new(true), + runtime_sweep_pending: AtomicBool::new(false), + #[cfg(test)] + panic_next_runtime_sweep: AtomicBool::new(false), next_idle_sweep_at: Mutex::new(Instant::now()), next_dedup_sweep_at: Mutex::new(Instant::now()), dirty_fast_flush_families: Mutex::new(HashSet::new()), @@ -172,7 +176,11 @@ impl QueueDomainSink { next_fast_flush_at: Mutex::new(Instant::now()), }); let actor = Self::spawn_actor(core.clone()); - Self { core, actor } + Self { + core, + actor, + inflight_client_deliveries: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + } } fn spawn_actor(core: Arc) -> crate::runtime::ManagedActor { @@ -305,7 +313,7 @@ impl QueueDomainSink { .expect("queue key"); let actors = self.core.actors.lock(); let actor = actors.get(&key).expect("warm queue actor").actor.lock(); - QueueAdminPlane::admin_snapshot(&*actor) + actor.admin_snapshot() } #[cfg(test)] @@ -359,20 +367,39 @@ impl QueueDomainSink { *self.core.next_dedup_sweep_at.lock() = now; } + #[cfg(test)] + pub(super) fn panic_next_runtime_sweep_for_tests(&self) { + self.core + .panic_next_runtime_sweep + .store(true, Ordering::Release); + } + + #[cfg(test)] + pub(super) fn runtime_sweep_pending_for_tests(&self) -> bool { + self.core.runtime_sweep_pending.load(Ordering::Acquire) + } + fn send_unit_actor_command( &self, operation: &'static str, build_command: impl FnOnce(crossbeam_channel::Sender<()>) -> QueueDomainCommand, - ) { + ) -> Result<(), crate::runtime::DeliveryError> { let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); if let Err(error) = self.actor.try_send_high_priority(build_command(reply_tx)) { tracing::warn!(domain = "queue", operation, error = %error, "Queue actor command enqueue failed"); - return; + return Err(error); } - if let Err(error) = reply_rx.recv_timeout(QUEUE_ACTOR_REPLY_TIMEOUT) { - tracing::warn!(domain = "queue", operation, error = %error, "Queue actor command reply failed"); - } + // Returning the outcome rather than swallowing it: callers previously + // could not tell a completed command from one that timed out, so a + // silently dropped session cleanup looked identical to a successful + // one. + reply_rx + .recv_timeout(QUEUE_ACTOR_REPLY_TIMEOUT) + .map_err(|error| { + tracing::warn!(domain = "queue", operation, error = %error, "Queue actor command reply failed"); + crate::runtime::reply_wait::map_reply_wait_error(error) + }) } fn send_bool_actor_command( @@ -397,7 +424,9 @@ impl QueueDomainSink { } pub fn refresh_admin_snapshot_if_dirty(&self) { - self.send_unit_actor_command( + // Best effort: the snapshot refreshes again on the next tick, so a + // missed one is not worth surfacing. + let _ = self.send_unit_actor_command( "refresh_admin_snapshot_if_dirty", QueueDomainCommand::RefreshAdminSnapshotIfDirty, ); @@ -415,7 +444,10 @@ impl QueueDomainSink { reply_rx .recv_timeout(QUEUE_ACTOR_REPLY_TIMEOUT) - .unwrap_or_default() + .unwrap_or_else(|error| { + tracing::warn!(domain = "queue", error = %error, "Queue live-count query reply failed"); + QueueLiveCounts::default() + }) } pub fn counts(&self) -> QueueCounts { @@ -429,22 +461,53 @@ impl QueueDomainSink { } } - pub fn cleanup_session(&self, session_id: u64) { + /// Run session cleanup on the actor, reporting whether it completed. + /// + /// The outcome must reach the caller: swallowing it made a cleanup that + /// never ran indistinguishable from one that succeeded, so the ingress + /// retry-ticket machinery never saw a queue cleanup failure at all. + /// + /// # Errors + /// + /// Returns the delivery failure when the command could not be enqueued, or + /// when the actor did not reply before its deadline. + #[must_use = "a dropped cleanup failure is indistinguishable from a cleanup that succeeded"] + pub fn cleanup_session(&self, session_id: u64) -> Result<(), crate::runtime::DeliveryError> { self.send_unit_actor_command("cleanup_session", |reply| { QueueDomainCommand::CleanupSession(session_id, reply) - }); + }) } pub(crate) fn sweep_runtime_state(&self) { - self.sweep_runtime_state_at(Instant::now()); + self.request_runtime_sweep_at(Instant::now()); } + #[cfg(test)] pub(super) fn sweep_runtime_state_at(&self, now: Instant) { - self.send_unit_actor_command("sweep_runtime_state", |reply| { - QueueDomainCommand::SweepRuntimeStateAt(now, reply) + let _ = self.send_unit_actor_command("sweep_runtime_state", |reply| { + QueueDomainCommand::SweepRuntimeStateAt(now, Some(reply)) }); } + pub(super) fn request_runtime_sweep_at(&self, now: Instant) -> bool { + if self.core.runtime_sweep_pending.swap(true, Ordering::AcqRel) { + return false; + } + + if let Err(error) = self + .actor + .try_send_high_priority(QueueDomainCommand::SweepRuntimeStateAt(now, None)) + { + self.core + .runtime_sweep_pending + .store(false, Ordering::Release); + tracing::warn!(domain = "queue", operation = "sweep_runtime_state", error = %error, "Queue actor command enqueue failed"); + return false; + } + + true + } + /// Replays a dead-lettered message back into its queue. /// /// # Errors diff --git a/src/domains/queue/sink/ingress.rs b/src/domains/queue/sink/ingress.rs new file mode 100644 index 00000000..64ec98c0 --- /dev/null +++ b/src/domains/queue/sink/ingress.rs @@ -0,0 +1,330 @@ +//! Envelope intake: cleanup/request extraction, frame parsing, and operation +//! dispatch entry point. + +#[cfg(test)] +use super::model::FrameContext; +use super::model::{ + DeliveryError, Duration, Envelope, Instant, PendingQueueReserve, QueueClientFrame, + QueueClientRequest, QueueDomainCore, +}; +use crate::runtime::routing::RouteFamily; + +impl QueueDomainCore { + pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { + if self.handle_cleanup_envelope(envelope) { + return Ok(()); + } + self.ensure_active()?; + Self::log_delivery(envelope); + + let Some(request) = Self::extract_request(envelope)? else { + return Ok(()); + }; + let meta = request.meta; + let route_family = *envelope.destination().family(); + let request_started = self.record_request_start(); + + if meta.route_family != route_family + || envelope + .source() + .is_some_and(|source| *source.family() != meta.route_family) + { + let response = crate::domains::queue::QueueResponse::BadRequest { + reason: "route family mismatch".to_string(), + }; + let response_meta = envelope.source().map_or(meta, |source| { + let mut response_meta = meta; + response_meta.route_family = *source.family(); + response_meta + }); + self.route_queue_response(envelope, response_meta, &response); + if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { + metrics.record_failure(started_at); + } + return Ok(()); + } + + // This request was already queued (on the normal lane) before this + // session's disconnect cleanup ran (on the high-priority/control-plane + // lane) and jumped ahead of it. Reject rather than silently + // recreating a subscription or pending reserve for a session that is + // already gone and will never be cleaned up again. + if self.is_cleaned_up_session(meta.session_id) { + let response = crate::domains::queue::QueueResponse::BadRequest { + reason: "session already closed".to_string(), + }; + self.route_queue_response(envelope, meta, &response); + if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { + metrics.record_failure(started_at); + } + return Ok(()); + } + + let Some(parsed_frame) = + self.parse_request_frame(envelope, meta, request.frame, request_started) + else { + return Ok(()); + }; + + self.maybe_sweep_idle_actors(); + + match parsed_frame { + QueueClientFrame::Sub(sub_msg) => { + self.handle_subscription_frame(envelope, meta, request_started, sub_msg); + Ok(()) + } + QueueClientFrame::Op(queue_msg) => { + self.handle_actor_operation_frame( + envelope, + meta, + request_started, + route_family, + queue_msg, + ); + Ok(()) + } + } + } + + fn handle_cleanup_envelope(&self, envelope: &Envelope) -> bool { + // `QueueDomainCore::cleanup_session` runs inline rather than through an + // actor command, so there is no reply deadline to surface here. + if let Some(cleanup) = envelope.payload::() { + // Mark first so an older normal-lane request that cleanup jumped + // over cannot recreate a subscription or pending reserve for + // this session below. `cleanup.rs` is the sole source of truth + // for cleaned-up-session state. + self.mark_cleaned_up_session(cleanup.session_id); + self.cleanup_session(cleanup.session_id); + return true; + } + + false + } + + fn ensure_active(&self) -> Result<(), DeliveryError> { + crate::runtime::ingress_support::ensure_actor_active(&self.active) + } + + fn log_delivery(envelope: &Envelope) { + crate::runtime::ingress_support::log_envelope_received( + "queue", + "Queue domain sink: received envelope", + envelope, + ); + } + + fn extract_request(envelope: &Envelope) -> Result, DeliveryError> { + if let Some(request) = Self::request_from_envelope(envelope) { + return Ok(Some(request)); + } + + tracing::warn!( + domain = "queue", + "Envelope payload was not QueueClientRequest" + ); + Err(DeliveryError::ActorStopped) + } + + fn record_request_start(&self) -> Option { + self.metrics + .as_ref() + .map(crate::domains::queue::QueueMetrics::record_request_start) + } + + fn parse_request_frame( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + frame: Result, + request_started: Option, + ) -> Option { + let parsed_frame = match frame { + Ok(frame) => frame, + Err(reason) => { + let response = crate::domains::queue::QueueResponse::BadRequest { reason }; + self.route_queue_response(envelope, meta, &response); + if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) + { + metrics.record_failure(started_at); + } + return None; + } + }; + + tracing::debug!( + domain = "queue", + session = meta.session_id, + msg_type = meta.message_type, + "Parsed Queue message successfully" + ); + + Some(parsed_frame) + } + + fn handle_actor_operation_frame( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + request_started: Option, + route_family: RouteFamily, + queue_msg: crate::domains::queue::protocol::QueueMessage, + ) { + if Self::queue_message_family(&queue_msg) + .is_some_and(|family_id| family_id != meta.route_family) + { + let response = crate::domains::queue::QueueResponse::BadRequest { + reason: "route family mismatch".to_string(), + }; + self.route_queue_response(envelope, meta, &response); + if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { + metrics.record_failure(started_at); + } + return; + } + + let op_kind = Self::classify_operation(&queue_msg); + let wait_seconds = match &queue_msg { + crate::domains::queue::protocol::QueueMessage::Receive { wait_seconds, .. } => { + *wait_seconds + } + _ => None, + }; + let wake_route = match &queue_msg { + crate::domains::queue::protocol::QueueMessage::Send { route, .. } => { + Some(route.clone()) + } + _ => None, + }; + let pending_message = queue_msg.clone(); + let Some(outcome) = + self.dispatch_actor_operation(envelope, meta, request_started, queue_msg) + else { + return; + }; + + if matches!( + &outcome.response, + crate::domains::queue::QueueResponse::Received { messages } if messages.is_empty() + ) || matches!( + &outcome.response, + crate::domains::queue::QueueResponse::ReceivedRouted { messages } if messages.is_empty() + ) { + if let Some(wait_seconds) = wait_seconds.filter(|seconds| *seconds > 0) { + if let Some(source) = envelope.source() { + let mut message = pending_message; + if let crate::domains::queue::protocol::QueueMessage::Receive { + wait_seconds, + .. + } = &mut message + { + *wait_seconds = None; + } + let deadline = Instant::now() + .checked_add(Duration::from_secs(wait_seconds)) + .unwrap_or_else(Instant::now); + self.pending_reserves.lock().push_back(PendingQueueReserve { + envelope: Envelope::from_route( + source.clone(), + envelope.destination().clone(), + (), + ), + meta, + request_started, + message, + deadline, + }); + return; + } + } + } + + if outcome.mark_admin_snapshot_dirty { + self.mark_admin_snapshot_dirty(); + self.mark_fast_flush_dirty(route_family); + } + + for (key, notification) in outcome.ready_notifications { + self.route_queue_ready_notification(&key, notification); + } + + self.route_queue_response(envelope, meta, &outcome.response); + self.record_operation_metrics(request_started, &outcome.response, op_kind); + if let Some(route) = wake_route.as_ref() { + self.wake_pending_reserves_for_route(meta.route_family, route, Instant::now()); + } + } + + pub(in crate::domains::queue::sink) fn queue_message_family( + queue_msg: &crate::domains::queue::protocol::QueueMessage, + ) -> Option { + match queue_msg { + crate::domains::queue::protocol::QueueMessage::Send { family_id, .. } + | crate::domains::queue::protocol::QueueMessage::Receive { family_id, .. } + | crate::domains::queue::protocol::QueueMessage::Extend { family_id, .. } + | crate::domains::queue::protocol::QueueMessage::Ack { family_id, .. } => { + Some(*family_id) + } + crate::domains::queue::protocol::QueueMessage::InflightExpired { .. } => None, + } + } + + fn request_from_envelope(envelope: &Envelope) -> Option { + if let Some(request) = envelope.payload::() { + return Some(request.clone()); + } + + #[cfg(test)] + { + let frame_ctx = envelope.payload::()?.clone(); + let subscriber = envelope.source().cloned().unwrap_or_else(|| { + Self::session_inbox_address(frame_ctx.route_family, frame_ctx.session_id) + }); + let meta = crate::runtime::ClientFrameMeta::new( + frame_ctx.session_id, + test_client_channel_from_protocol(frame_ctx.channel_id), + frame_ctx.msg_type.as_u16(), + frame_ctx.route_family, + ); + let parsed = crate::dispatch::protocol::queue_codec::parse_frame( + &frame_ctx, + &frame_ctx.payload, + frame_ctx.route_family, + frame_ctx.session_id, + subscriber, + ) + .map(|frame| match frame { + crate::dispatch::protocol::queue_codec::ParsedQueueFrame::Op(message) => { + QueueClientFrame::Op(message) + } + crate::dispatch::protocol::queue_codec::ParsedQueueFrame::Sub(message) => { + QueueClientFrame::Sub(message) + } + }); + Some(QueueClientRequest::new(meta, parsed)) + } + + #[cfg(not(test))] + { + None + } + } +} + +#[cfg(test)] +fn test_client_channel_from_protocol( + channel: crate::dispatch::protocol::frame::ChannelId, +) -> crate::runtime::ClientChannel { + match channel { + crate::dispatch::protocol::frame::ChannelId::Control => { + crate::runtime::ClientChannel::Control + } + crate::dispatch::protocol::frame::ChannelId::Pub => crate::runtime::ClientChannel::Pub, + crate::dispatch::protocol::frame::ChannelId::Sub => crate::runtime::ClientChannel::Sub, + crate::dispatch::protocol::frame::ChannelId::Rpc => crate::runtime::ClientChannel::Rpc, + crate::dispatch::protocol::frame::ChannelId::Lease => crate::runtime::ClientChannel::Lease, + crate::dispatch::protocol::frame::ChannelId::Internal => { + crate::runtime::ClientChannel::Internal + } + } +} diff --git a/src/domains/queue/sink/mailbox.rs b/src/domains/queue/sink/mailbox.rs new file mode 100644 index 00000000..6b1b4357 --- /dev/null +++ b/src/domains/queue/sink/mailbox.rs @@ -0,0 +1,151 @@ +//! Mailbox entry points: `MailboxSink`, the domain actor's `receive` loop, and +//! the thin runtime-to-core delegation used by both. + +use super::model::{ + DeliveryError, Envelope, Instant, MailboxSink, Ordering, QueueDomainActor, QueueDomainCommand, + QueueDomainRuntime, QueueDomainSink, QueueLiveCounts, +}; +use crate::runtime::{Actor, Context}; + +pub(super) struct RuntimeSweepPendingReset<'a>(pub(super) &'a std::sync::atomic::AtomicBool); + +impl Drop for RuntimeSweepPendingReset<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } +} + +impl MailboxSink for QueueDomainSink { + fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.deliver_to_actor(envelope, false) + } + + fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.deliver_to_actor(envelope, true) + } +} + +impl Actor for QueueDomainActor { + type Message = QueueDomainCommand; + + fn receive(&mut self, msg: Self::Message, _ctx: &mut Context) { + let runtime = self.runtime(); + match msg { + QueueDomainCommand::Deliver(envelope, reply, admission) => { + let started_at = Instant::now(); + let outcome = runtime.deliver_envelope(&envelope); + super::model::record_service_sample(&self.core.delivery_service_us, started_at); + let _ = reply.send(outcome); + // Explicit: the slot is released here, once the work is + // actually done, and not when the caller gave up waiting. + drop(admission); + } + QueueDomainCommand::RefreshAdminSnapshotIfDirty(reply) => { + runtime.refresh_admin_snapshot_if_dirty(); + let _ = reply.send(()); + } + QueueDomainCommand::ReadLiveCounts(reply) => { + let _ = reply.send(runtime.live_counts()); + } + QueueDomainCommand::CleanupSession(session_id, reply) => { + runtime.cleanup_session(session_id); + let _ = reply.send(()); + } + QueueDomainCommand::SweepRuntimeStateAt(now, Some(reply)) => { + runtime.sweep_runtime_state_at(now); + let _ = reply.send(()); + } + QueueDomainCommand::SweepRuntimeStateAt(now, None) => { + let _pending_reset = RuntimeSweepPendingReset(&runtime.runtime_sweep_pending); + runtime.sweep_runtime_state_at(now); + } + QueueDomainCommand::ReplayDeadLetter(key, id, reply) => { + let _ = reply.send(runtime.replay_dead_letter(&key, id)); + } + QueueDomainCommand::PurgeDeadLetter(key, id, reply) => { + let _ = reply.send(runtime.purge_dead_letter(&key, id)); + } + #[cfg(test)] + QueueDomainCommand::PanicForTests => { + panic!("test Queue domain actor panic"); + } + } + } +} + +impl QueueDomainSink { + fn deliver_to_actor( + &self, + envelope: Envelope, + high_priority: bool, + ) -> Result<(), DeliveryError> { + // Admit BEFORE enqueueing so surplus load is refused as never-enqueued + // (retryable) rather than accepted then timed out. Control-plane work + // bypasses the window - cleanup arrives on the normal lane yet must + // never be rationed by client load. See `admit_client_delivery`. + let is_control_plane = high_priority + || envelope + .payload::() + .is_some(); + let admission = if is_control_plane { + None + } else { + Some(super::model::admit_client_delivery( + &self.inflight_client_deliveries, + &self.core.delivery_service_us, + self.actor.is_running(), + )?) + }; + + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + let command = QueueDomainCommand::Deliver(envelope, reply_tx, admission); + let enqueue_result = if high_priority { + self.actor.try_send_high_priority(command) + } else { + self.actor.try_send(command) + }; + enqueue_result?; + + reply_rx + .recv_timeout(super::model::QUEUE_ACTOR_REPLY_TIMEOUT) + .unwrap_or_else(|error| Err(crate::runtime::reply_wait::map_reply_wait_error(error))) + } +} + +impl QueueDomainRuntime<'_> { + pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { + self.core.deliver_envelope(envelope) + } + + pub(super) fn refresh_admin_snapshot_if_dirty(&self) { + self.core.refresh_admin_snapshot_if_dirty(); + } + + pub(super) fn live_counts(&self) -> QueueLiveCounts { + self.core.live_counts() + } + + pub(super) fn cleanup_session(&self, session_id: u64) { + self.core.cleanup_session(session_id); + } + + pub(super) fn sweep_runtime_state_at(&self, now: Instant) { + self.core.sweep_runtime_state_at(now); + } + + pub(super) fn replay_dead_letter( + &self, + key: &crate::domains::queue::QueueKey, + id: crate::domains::queue::MessageId, + ) -> Result { + self.core.replay_dead_letter(key, id) + } + + pub(super) fn purge_dead_letter( + &self, + key: &crate::domains::queue::QueueKey, + id: crate::domains::queue::MessageId, + ) -> Result { + self.core.purge_dead_letter(key, id) + } +} diff --git a/src/domains/queue/sink/mailbox_sink_impl.rs b/src/domains/queue/sink/mailbox_sink_impl.rs deleted file mode 100644 index a07aa6da..00000000 --- a/src/domains/queue/sink/mailbox_sink_impl.rs +++ /dev/null @@ -1,949 +0,0 @@ -use super::model::{ - obs, DeliveryError, Duration, Envelope, Instant, MailboxSink, Ordering, PendingQueueReserve, - QueueClientFrame, QueueClientRequest, QueueDomainActor, QueueDomainCommand, QueueDomainCore, - QueueDomainRuntime, QueueDomainSink, QueueLiveCounts, QueueReadyNotification, - QueueSubscription, QueueSubscriptionMessage, RoutedSubscriptionSet, VecDeque, - QUEUE_ACTOR_REPLY_TIMEOUT, -}; -#[cfg(test)] -use crate::dispatch::protocol::frame_context::FrameContext; -use crate::runtime::routing::RouteFamily; -use crate::runtime::{Actor, Context}; - -type ReadyNotificationEvent = (crate::domains::queue::QueueKey, QueueReadyNotification); - -mod pending_reserves; -mod runtime_adapter; -mod wildcard_receive; - -#[derive(Clone, Copy)] -struct OperationRequestContext<'a> { - envelope: &'a Envelope, - meta: crate::runtime::ClientFrameMeta, - request_started: Option, -} - -struct OperationOutcome { - response: crate::domains::queue::QueueResponse, - ready_notifications: Vec, - mark_admin_snapshot_dirty: bool, -} - -type SubscriptionOutcome = ( - crate::domains::queue::QueueResponse, - Option<( - RouteFamily, - crate::runtime::matcher::Pattern, - u64, - u64, - crate::runtime::routing::RouteAddress, - )>, - bool, -); - -#[derive(Clone, Copy)] -struct ExtendOperation { - session_id: u64, - id: crate::domains::queue::MessageId, - token: u64, - inflight_seconds: u64, -} - -#[derive(Clone, Copy)] -enum QueueOpKind { - Send, - Receive, - Extend, - Ack, - InflightExpired, -} - -impl MailboxSink for QueueDomainSink { - fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { - self.deliver_to_actor(envelope, false) - } - - fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { - self.deliver_to_actor(envelope, true) - } -} - -impl Actor for QueueDomainActor { - type Message = QueueDomainCommand; - - fn receive(&mut self, msg: Self::Message, _ctx: &mut Context) { - let runtime = self.runtime(); - match msg { - QueueDomainCommand::Deliver(envelope, reply) => { - let _ = reply.send(runtime.deliver_envelope(&envelope)); - } - QueueDomainCommand::RefreshAdminSnapshotIfDirty(reply) => { - runtime.refresh_admin_snapshot_if_dirty(); - let _ = reply.send(()); - } - QueueDomainCommand::ReadLiveCounts(reply) => { - let _ = reply.send(runtime.live_counts()); - } - QueueDomainCommand::CleanupSession(session_id, reply) => { - runtime.cleanup_session(session_id); - let _ = reply.send(()); - } - QueueDomainCommand::SweepRuntimeStateAt(now, reply) => { - runtime.sweep_runtime_state_at(now); - let _ = reply.send(()); - } - QueueDomainCommand::ReplayDeadLetter(key, id, reply) => { - let _ = reply.send(runtime.replay_dead_letter(&key, id)); - } - QueueDomainCommand::PurgeDeadLetter(key, id, reply) => { - let _ = reply.send(runtime.purge_dead_letter(&key, id)); - } - #[cfg(test)] - QueueDomainCommand::PanicForTests => { - panic!("test Queue domain actor panic"); - } - } - } -} - -impl QueueDomainSink { - fn deliver_to_actor( - &self, - envelope: Envelope, - high_priority: bool, - ) -> Result<(), DeliveryError> { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - let command = QueueDomainCommand::Deliver(envelope, reply_tx); - let enqueue_result = if high_priority { - self.actor.try_send_high_priority(command) - } else { - self.actor.try_send(command) - }; - enqueue_result?; - - reply_rx - .recv_timeout(QUEUE_ACTOR_REPLY_TIMEOUT) - .unwrap_or(Err(DeliveryError::ActorStopped)) - } -} - -impl QueueDomainCore { - fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { - if self.handle_cleanup_envelope(envelope) { - return Ok(()); - } - self.ensure_active()?; - Self::log_delivery(envelope); - - let Some(request) = Self::extract_request(envelope)? else { - return Ok(()); - }; - let meta = request.meta; - let route_family = *envelope.destination().family(); - let request_started = self.record_request_start(); - - if meta.route_family != route_family - || envelope - .source() - .is_some_and(|source| *source.family() != meta.route_family) - { - let response = crate::domains::queue::QueueResponse::BadRequest { - reason: "route family mismatch".to_string(), - }; - let response_meta = envelope.source().map_or(meta, |source| { - let mut response_meta = meta; - response_meta.route_family = *source.family(); - response_meta - }); - self.route_queue_response(envelope, response_meta, &response); - if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { - metrics.record_failure(started_at); - } - return Ok(()); - } - - let Some(parsed_frame) = - self.parse_request_frame(envelope, meta, request.frame, request_started) - else { - return Ok(()); - }; - - self.maybe_sweep_idle_actors(); - - match parsed_frame { - QueueClientFrame::Sub(sub_msg) => { - self.handle_subscription_frame(envelope, meta, request_started, sub_msg); - Ok(()) - } - QueueClientFrame::Op(queue_msg) => { - self.handle_actor_operation_frame( - envelope, - meta, - request_started, - route_family, - queue_msg, - ); - Ok(()) - } - } - } - - fn handle_cleanup_envelope(&self, envelope: &Envelope) -> bool { - if let Some(cleanup) = envelope.payload::() { - self.cleanup_session(cleanup.session_id); - return true; - } - - false - } - - fn ensure_active(&self) -> Result<(), DeliveryError> { - if !self.active.load(Ordering::Relaxed) { - return Err(DeliveryError::ActorStopped); - } - - Ok(()) - } - - fn log_delivery(envelope: &Envelope) { - tracing::debug!( - domain = "queue", - destination = %envelope.destination(), - source = ?envelope.source(), - "Queue domain sink: received envelope" - ); - } - - fn extract_request(envelope: &Envelope) -> Result, DeliveryError> { - if let Some(request) = Self::request_from_envelope(envelope) { - return Ok(Some(request)); - } - - tracing::warn!( - domain = "queue", - "Envelope payload was not QueueClientRequest" - ); - Err(DeliveryError::ActorStopped) - } - - fn record_request_start(&self) -> Option { - self.metrics - .as_ref() - .map(crate::domains::queue::QueueMetrics::record_request_start) - } - - fn parse_request_frame( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - frame: Result, - request_started: Option, - ) -> Option { - let parsed_frame = match frame { - Ok(frame) => frame, - Err(reason) => { - let response = crate::domains::queue::QueueResponse::BadRequest { reason }; - self.route_queue_response(envelope, meta, &response); - if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) - { - metrics.record_failure(started_at); - } - return None; - } - }; - - tracing::debug!( - domain = "queue", - session = meta.session_id, - msg_type = meta.message_type, - "Parsed Queue message successfully" - ); - - Some(parsed_frame) - } - - fn handle_subscription_frame( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - request_started: Option, - sub_msg: QueueSubscriptionMessage, - ) { - let (response, initial_watch_snapshot, state_changed) = match sub_msg { - QueueSubscriptionMessage::Watch { - family_id, - pattern, - session_id, - subscriber, - } => self.handle_watch_subscription( - envelope, meta, family_id, &pattern, session_id, subscriber, - ), - QueueSubscriptionMessage::Unwatch { - family_id, - pattern, - session_id, - subscriber, - } => self.handle_unwatch_subscription( - envelope, - meta, - family_id, - &pattern, - session_id, - &subscriber, - ), - }; - - self.route_queue_response(envelope, meta, &response); - if state_changed { - self.mark_admin_snapshot_dirty(); - } - if let Some((family_id, pattern, session_id, subscription_id, subscriber)) = - initial_watch_snapshot - { - self.emit_current_ready_notifications_for_watch( - family_id, - &pattern, - session_id, - subscription_id, - &subscriber, - ); - } - - self.record_operation_metrics(request_started, &response, QueueOpKind::InflightExpired); - } - - fn handle_watch_subscription( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - family_id: RouteFamily, - pattern: &crate::runtime::routing::Route, - session_id: u64, - subscriber: crate::runtime::routing::RouteAddress, - ) -> SubscriptionOutcome { - if !Self::valid_subscription_request(envelope, meta, family_id, session_id, &subscriber) { - return ( - crate::domains::queue::QueueResponse::BadRequest { - reason: "route family mismatch".to_string(), - }, - None, - false, - ); - } - let pattern_str = pattern.as_str(); - let parsed_pattern = match crate::runtime::DomainKind::Queue - .descriptor() - .compile_registration_pattern(pattern_str) - { - Ok(pattern) => pattern, - Err(reason) => { - return ( - crate::domains::queue::QueueResponse::InvalidSubscriptionPattern { reason }, - None, - false, - ); - } - }; - let (subscription_id, state_changed) = { - let mut families = self.families.lock(); - let state = families - .entry(family_id.as_u64()) - .or_insert_with(RoutedSubscriptionSet::new); - - if let Some(id) = state.find_existing_id(session_id, pattern_str) { - (id, false) - } else { - if state.wildcard_registration_limit_reached(session_id, &parsed_pattern) { - return ( - crate::domains::queue::QueueResponse::SubscriptionLimit, - None, - false, - ); - } - let Ok(id) = self.next_sub_id.fetch_update( - Ordering::Relaxed, - Ordering::Relaxed, - |current| current.checked_add(1), - ) else { - let state_empty = state.is_empty(); - if state_empty { - families.remove(&family_id.as_u64()); - } - return ( - crate::domains::queue::QueueResponse::BadRequest { - reason: "subscription ID space exhausted".to_string(), - }, - None, - false, - ); - }; - state.insert( - family_id, - QueueSubscription { - pattern: parsed_pattern.clone(), - session_id, - subscription_id: id, - subscriber: subscriber.clone(), - }, - ); - (id, true) - } - }; - - ( - crate::domains::queue::QueueResponse::WatchOk { subscription_id }, - Some(( - family_id, - parsed_pattern, - session_id, - subscription_id, - subscriber, - )), - state_changed, - ) - } - - fn handle_unwatch_subscription( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - family_id: RouteFamily, - pattern: &crate::runtime::routing::Route, - session_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - ) -> SubscriptionOutcome { - if !Self::valid_subscription_request(envelope, meta, family_id, session_id, subscriber) { - return ( - crate::domains::queue::QueueResponse::BadRequest { - reason: "route family mismatch".to_string(), - }, - None, - false, - ); - } - if let Err(reason) = crate::runtime::DomainKind::Queue - .descriptor() - .compile_registration_pattern(pattern.as_str()) - { - return ( - crate::domains::queue::QueueResponse::InvalidSubscriptionPattern { reason }, - None, - false, - ); - } - - let mut families = self.families.lock(); - let remove_family = if let Some(state) = families.get_mut(&family_id.as_u64()) { - state.remove_session_pattern(family_id, session_id, pattern.as_str()); - state.is_empty() - } else { - false - }; - if remove_family { - families.remove(&family_id.as_u64()); - } - (crate::domains::queue::QueueResponse::UnwatchOk, None, true) - } - - fn valid_subscription_request( - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - family_id: RouteFamily, - session_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - ) -> bool { - family_id == meta.route_family - && *subscriber.family() == family_id - && session_id == meta.session_id - && envelope.source().is_none_or(|source| source == subscriber) - } - - fn handle_actor_operation_frame( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - request_started: Option, - route_family: RouteFamily, - queue_msg: crate::domains::queue::protocol::QueueMessage, - ) { - if Self::queue_message_family(&queue_msg) - .is_some_and(|family_id| family_id != meta.route_family) - { - let response = crate::domains::queue::QueueResponse::BadRequest { - reason: "route family mismatch".to_string(), - }; - self.route_queue_response(envelope, meta, &response); - if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { - metrics.record_failure(started_at); - } - return; - } - - let op_kind = Self::classify_operation(&queue_msg); - let wait_seconds = match &queue_msg { - crate::domains::queue::protocol::QueueMessage::Receive { wait_seconds, .. } => { - *wait_seconds - } - _ => None, - }; - let wake_route = match &queue_msg { - crate::domains::queue::protocol::QueueMessage::Send { route, .. } => { - Some(route.clone()) - } - _ => None, - }; - let pending_message = queue_msg.clone(); - let Some(outcome) = - self.dispatch_actor_operation(envelope, meta, request_started, queue_msg) - else { - return; - }; - - if matches!( - &outcome.response, - crate::domains::queue::QueueResponse::Received { messages } if messages.is_empty() - ) || matches!( - &outcome.response, - crate::domains::queue::QueueResponse::ReceivedRouted { messages } if messages.is_empty() - ) { - if let Some(wait_seconds) = wait_seconds.filter(|seconds| *seconds > 0) { - if let Some(source) = envelope.source() { - let mut message = pending_message; - if let crate::domains::queue::protocol::QueueMessage::Receive { - wait_seconds, - .. - } = &mut message - { - *wait_seconds = None; - } - let deadline = Instant::now() - .checked_add(Duration::from_secs(wait_seconds)) - .unwrap_or_else(Instant::now); - self.pending_reserves.lock().push_back(PendingQueueReserve { - envelope: Envelope::from_route( - source.clone(), - envelope.destination().clone(), - (), - ), - meta, - request_started, - message, - deadline, - }); - return; - } - } - } - - if outcome.mark_admin_snapshot_dirty { - self.mark_admin_snapshot_dirty(); - self.mark_fast_flush_dirty(route_family); - } - - for (key, notification) in outcome.ready_notifications { - self.route_queue_ready_notification(&key, notification); - } - - self.route_queue_response(envelope, meta, &outcome.response); - self.record_operation_metrics(request_started, &outcome.response, op_kind); - if let Some(route) = wake_route.as_ref() { - self.wake_pending_reserves_for_route(meta.route_family, route, Instant::now()); - } - } - - fn queue_message_family( - queue_msg: &crate::domains::queue::protocol::QueueMessage, - ) -> Option { - match queue_msg { - crate::domains::queue::protocol::QueueMessage::Send { family_id, .. } - | crate::domains::queue::protocol::QueueMessage::Receive { family_id, .. } - | crate::domains::queue::protocol::QueueMessage::Extend { family_id, .. } - | crate::domains::queue::protocol::QueueMessage::Ack { family_id, .. } => { - Some(*family_id) - } - crate::domains::queue::protocol::QueueMessage::InflightExpired { .. } => None, - } - } - - fn dispatch_actor_operation( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - request_started: Option, - queue_msg: crate::domains::queue::protocol::QueueMessage, - ) -> Option { - let request_context = OperationRequestContext { - envelope, - meta, - request_started, - }; - let outcome = match queue_msg { - crate::domains::queue::protocol::QueueMessage::Send { - family_id, - route, - body, - delay_seconds, - } => self.handle_enqueue_operation( - family_id, - &route, - body, - delay_seconds, - request_context, - )?, - crate::domains::queue::protocol::QueueMessage::Receive { - family_id, - route, - inflight_seconds, - batch_size, - wait_seconds: _, - } => self.handle_receive_operation( - family_id, - &route, - meta.session_id, - inflight_seconds, - batch_size, - request_context, - )?, - crate::domains::queue::protocol::QueueMessage::Extend { - family_id, - route, - id, - token, - inflight_seconds, - } => self.handle_extend_operation( - family_id, - &route, - ExtendOperation { - session_id: meta.session_id, - id, - token, - inflight_seconds, - }, - request_context, - )?, - crate::domains::queue::protocol::QueueMessage::Ack { - family_id, - route, - id, - token, - } => self.handle_ack_operation( - family_id, - &route, - meta.session_id, - id, - token, - request_context, - )?, - crate::domains::queue::protocol::QueueMessage::InflightExpired { .. } => { - OperationOutcome { - response: crate::domains::queue::QueueResponse::Error { - message: "InflightExpired is an internal message".to_string(), - }, - ready_notifications: Vec::new(), - mark_admin_snapshot_dirty: false, - } - } - }; - - Some(outcome) - } - - fn handle_enqueue_operation( - &self, - family_id: RouteFamily, - route: &crate::runtime::routing::Route, - body: bytes::Bytes, - delay_seconds: Option, - request_context: OperationRequestContext<'_>, - ) -> Option { - let key = match Self::queue_key_for_route(family_id, route) { - Ok(key) => key, - Err(response) => { - return Some(OperationOutcome { - response, - ready_notifications: Vec::new(), - mark_admin_snapshot_dirty: false, - }); - } - }; - - self.with_actor_for_operation(&key, request_context, |actor| { - actor.handle_send(body, delay_seconds) - }) - .map(|(response, notification)| OperationOutcome { - response, - ready_notifications: notification.into_iter().collect(), - mark_admin_snapshot_dirty: true, - }) - } - - fn handle_receive_operation( - &self, - family_id: RouteFamily, - route: &crate::runtime::routing::Route, - session_id: u64, - inflight_seconds: u64, - batch_size: Option, - request_context: OperationRequestContext<'_>, - ) -> Option { - if let Ok(key) = Self::queue_key_for_route(family_id, route) { - return self - .with_actor_for_operation(&key, request_context, |actor| { - actor.handle_receive_for_session(session_id, inflight_seconds, batch_size) - }) - .map(|(response, notification)| OperationOutcome { - response, - ready_notifications: notification.into_iter().collect(), - mark_admin_snapshot_dirty: true, - }); - } - - let pattern = match Self::wildcard_queue_selector(route) { - Ok(pattern) => pattern, - Err(response) => { - return Some(OperationOutcome { - response, - ready_notifications: Vec::new(), - mark_admin_snapshot_dirty: false, - }); - } - }; - Some(self.handle_wildcard_receive( - family_id, - &pattern, - session_id, - inflight_seconds, - batch_size, - )) - } - - fn wildcard_queue_selector( - route: &crate::runtime::routing::Route, - ) -> Result { - if !route.as_str().contains('*') { - return Err(crate::domains::queue::QueueResponse::BadRequest { - reason: format!("invalid queue route: {}", route.as_str()), - }); - } - let pattern = crate::runtime::DomainKind::Queue - .descriptor() - .compile_registration_pattern(route.as_str()) - .map_err(|reason| crate::domains::queue::QueueResponse::BadRequest { reason })?; - if !pattern.is_wildcard() { - return Err(crate::domains::queue::QueueResponse::BadRequest { - reason: format!("invalid queue route: {}", route.as_str()), - }); - } - Ok(pattern) - } - - fn handle_extend_operation( - &self, - family_id: RouteFamily, - route: &crate::runtime::routing::Route, - extend: ExtendOperation, - request_context: OperationRequestContext<'_>, - ) -> Option { - let key = match Self::queue_key_for_route(family_id, route) { - Ok(key) => key, - Err(response) => { - return Some(OperationOutcome { - response, - ready_notifications: Vec::new(), - mark_admin_snapshot_dirty: false, - }); - } - }; - - self.with_actor_for_operation(&key, request_context, |actor| { - actor.handle_extend_for_session( - extend.session_id, - extend.id, - extend.token, - extend.inflight_seconds, - ) - }) - .map(|(response, notification)| OperationOutcome { - response, - ready_notifications: notification.into_iter().collect(), - mark_admin_snapshot_dirty: true, - }) - } - - fn handle_ack_operation( - &self, - family_id: RouteFamily, - route: &crate::runtime::routing::Route, - session_id: u64, - id: crate::domains::queue::MessageId, - token: u64, - request_context: OperationRequestContext<'_>, - ) -> Option { - let key = match Self::queue_key_for_route(family_id, route) { - Ok(key) => key, - Err(response) => { - return Some(OperationOutcome { - response, - ready_notifications: Vec::new(), - mark_admin_snapshot_dirty: false, - }); - } - }; - - self.with_actor_for_operation(&key, request_context, |actor| { - actor.handle_ack_for_session(session_id, id, token) - }) - .map(|(response, notification)| OperationOutcome { - response, - ready_notifications: notification.into_iter().collect(), - mark_admin_snapshot_dirty: true, - }) - } - - fn with_actor_for_operation( - &self, - key: &crate::domains::queue::QueueKey, - request_context: OperationRequestContext<'_>, - operation: F, - ) -> Option<( - crate::domains::queue::QueueResponse, - Option, - )> - where - F: FnOnce(&mut crate::domains::queue::QueueActor) -> crate::domains::queue::QueueResponse, - { - let actor_lock_start = Instant::now(); - let (actor_handle, _) = match self.get_or_create_actor(key) { - Ok(actor) => actor, - Err(message) => { - self.route_queue_recovery_error( - request_context.envelope, - request_context.meta, - request_context.request_started, - message, - ); - return None; - } - }; - self.observe_histogram_us( - obs::METRIC_QUEUE_ACTOR_LOCK_HOLD_LATENCY, - Self::u128_to_u64_saturating(actor_lock_start.elapsed().as_micros()), - ); - - let mut actor = actor_handle.lock(); - let actor_exec_start = Instant::now(); - actor.process_due_work(); - let response = operation(&mut actor); - let counts = actor.live_counts(); - if counts.total() > 0 { - self.known_queue_keys.lock().insert(key.clone()); - } - let notification = self.record_ready_state(key, counts); - self.observe_histogram_us( - obs::METRIC_QUEUE_ACTOR_EXECUTION_LATENCY, - Self::u128_to_u64_saturating(actor_exec_start.elapsed().as_micros()), - ); - - Some((response, notification.map(|event| (key.clone(), event)))) - } - - fn classify_operation( - queue_msg: &crate::domains::queue::protocol::QueueMessage, - ) -> QueueOpKind { - match queue_msg { - crate::domains::queue::protocol::QueueMessage::Send { .. } => QueueOpKind::Send, - crate::domains::queue::protocol::QueueMessage::Receive { .. } => QueueOpKind::Receive, - crate::domains::queue::protocol::QueueMessage::Extend { .. } => QueueOpKind::Extend, - crate::domains::queue::protocol::QueueMessage::Ack { .. } => QueueOpKind::Ack, - crate::domains::queue::protocol::QueueMessage::InflightExpired { .. } => { - QueueOpKind::InflightExpired - } - } - } - - fn record_operation_metrics( - &self, - request_started: Option, - response: &crate::domains::queue::QueueResponse, - op_kind: QueueOpKind, - ) { - if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { - if Self::queue_response_is_failure(response) { - metrics.record_failure(started_at); - return; - } - - metrics.record_success(started_at); - match op_kind { - QueueOpKind::Send => metrics.record_enqueue(started_at), - QueueOpKind::Receive => metrics.record_reserve(started_at), - QueueOpKind::Ack => metrics.record_complete(), - QueueOpKind::Extend => metrics.record_extend(), - QueueOpKind::InflightExpired => {} - } - } - } - - fn u128_to_u64_saturating(value: u128) -> u64 { - value.try_into().unwrap_or(u64::MAX) - } - - fn request_from_envelope(envelope: &Envelope) -> Option { - if let Some(request) = envelope.payload::() { - return Some(request.clone()); - } - - #[cfg(test)] - { - let frame_ctx = envelope.payload::()?.clone(); - let subscriber = envelope.source().cloned().unwrap_or_else(|| { - Self::session_inbox_address(frame_ctx.route_family, frame_ctx.session_id) - }); - let meta = crate::runtime::ClientFrameMeta::new( - frame_ctx.session_id, - test_client_channel_from_protocol(frame_ctx.channel_id), - frame_ctx.msg_type.as_u16(), - frame_ctx.route_family, - ); - let parsed = crate::dispatch::protocol::queue_codec::parse_frame( - &frame_ctx, - &frame_ctx.payload, - frame_ctx.route_family, - frame_ctx.session_id, - subscriber, - ) - .map(|frame| match frame { - crate::dispatch::protocol::queue_codec::ParsedQueueFrame::Op(message) => { - QueueClientFrame::Op(message) - } - crate::dispatch::protocol::queue_codec::ParsedQueueFrame::Sub(message) => { - QueueClientFrame::Sub(message) - } - }); - Some(QueueClientRequest::new(meta, parsed)) - } - - #[cfg(not(test))] - { - None - } - } -} - -#[cfg(test)] -fn test_client_channel_from_protocol( - channel: crate::dispatch::protocol::frame::ChannelId, -) -> crate::runtime::ClientChannel { - match channel { - crate::dispatch::protocol::frame::ChannelId::Control => { - crate::runtime::ClientChannel::Control - } - crate::dispatch::protocol::frame::ChannelId::Pub => crate::runtime::ClientChannel::Pub, - crate::dispatch::protocol::frame::ChannelId::Sub => crate::runtime::ClientChannel::Sub, - crate::dispatch::protocol::frame::ChannelId::Rpc => crate::runtime::ClientChannel::Rpc, - crate::dispatch::protocol::frame::ChannelId::Lease => crate::runtime::ClientChannel::Lease, - crate::dispatch::protocol::frame::ChannelId::Internal => { - crate::runtime::ClientChannel::Internal - } - } -} diff --git a/src/domains/queue/sink/mailbox_sink_impl/runtime_adapter.rs b/src/domains/queue/sink/mailbox_sink_impl/runtime_adapter.rs deleted file mode 100644 index 740df32a..00000000 --- a/src/domains/queue/sink/mailbox_sink_impl/runtime_adapter.rs +++ /dev/null @@ -1,41 +0,0 @@ -//! Thin adapter from the managed queue runtime to the queue core. - -use super::{DeliveryError, Envelope, Instant, QueueDomainRuntime, QueueLiveCounts}; - -impl QueueDomainRuntime<'_> { - pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { - self.core.deliver_envelope(envelope) - } - - pub(super) fn refresh_admin_snapshot_if_dirty(&self) { - self.core.refresh_admin_snapshot_if_dirty(); - } - - pub(super) fn live_counts(&self) -> QueueLiveCounts { - self.core.live_counts() - } - - pub(super) fn cleanup_session(&self, session_id: u64) { - self.core.cleanup_session(session_id); - } - - pub(super) fn sweep_runtime_state_at(&self, now: Instant) { - self.core.sweep_runtime_state_at(now); - } - - pub(super) fn replay_dead_letter( - &self, - key: &crate::domains::queue::QueueKey, - id: crate::domains::queue::MessageId, - ) -> Result { - self.core.replay_dead_letter(key, id) - } - - pub(super) fn purge_dead_letter( - &self, - key: &crate::domains::queue::QueueKey, - id: crate::domains::queue::MessageId, - ) -> Result { - self.core.purge_dead_letter(key, id) - } -} diff --git a/src/domains/queue/sink/mod.rs b/src/domains/queue/sink/mod.rs index 0b487c27..952c78f6 100644 --- a/src/domains/queue/sink/mod.rs +++ b/src/domains/queue/sink/mod.rs @@ -1,8 +1,15 @@ -mod domain_sink_impl; -mod mailbox_sink_impl; +mod actors; +mod cleanup; +mod delivery; +mod facade; +mod ingress; +mod mailbox; mod model; +mod observability; +mod responses; +mod subscriptions; -pub use domain_sink_impl::QueueCounts; +pub use facade::QueueCounts; pub use model::QueueDomainSink; #[cfg(test)] diff --git a/src/domains/queue/sink/model.rs b/src/domains/queue/sink/model.rs index a6ea9379..8e4e5ef2 100644 --- a/src/domains/queue/sink/model.rs +++ b/src/domains/queue/sink/model.rs @@ -70,6 +70,9 @@ pub(super) use crate::domains::queue::actor::QUEUE_ACTOR_REPLY_TIMEOUT; /// - Tracks queue-local watch subscriptions for the current broker process /// - Exposes only warm in-memory queue/admin state for the current broker process pub(super) struct QueueDomainCore { + /// Measured delivery service time in microseconds, written by the actor + /// and read by admission to size its window. + pub(super) delivery_service_us: ServiceEstimateUs, /// Fitz storage facade over the current Midge engine. pub(super) store: crate::storage::FitzStorageEngine, /// Commit policy for queue persistence on this runtime. @@ -88,6 +91,10 @@ pub(super) struct QueueDomainCore { pub(super) wildcard_reserve_sequence: AtomicU64, /// Queue-local watch subscriptions scoped to this broker process. pub(super) families: Mutex>>, + /// Sessions disconnect cleanup has already run for; guards against a + /// stale queued request recreating a subscription or pending reserve. + /// See `cleanup.rs`. + pub(super) cleaned_up_sessions: Mutex, pub(super) next_sub_id: AtomicU64, pub(super) ready_states: Mutex>, /// FIFO long-poll RESERVE requests waiting for a matching ready message. @@ -97,6 +104,9 @@ pub(super) struct QueueDomainCore { pub(super) projection: QueueAdminProjection, pub(super) metrics: Option, pub(super) active: AtomicBool, + pub(super) runtime_sweep_pending: AtomicBool, + #[cfg(test)] + pub(super) panic_next_runtime_sweep: AtomicBool, pub(super) next_idle_sweep_at: Mutex, pub(super) next_dedup_sweep_at: Mutex, pub(super) dirty_fast_flush_families: Mutex>, @@ -108,11 +118,14 @@ pub(super) enum QueueDomainCommand { Deliver( Envelope, crossbeam_channel::Sender>, + // Released when this command is finished with, not when the caller + // stops waiting for it. + Option, ), RefreshAdminSnapshotIfDirty(crossbeam_channel::Sender<()>), ReadLiveCounts(crossbeam_channel::Sender), CleanupSession(u64, crossbeam_channel::Sender<()>), - SweepRuntimeStateAt(Instant, crossbeam_channel::Sender<()>), + SweepRuntimeStateAt(Instant, Option>), ReplayDeadLetter( QueueKey, MessageId, @@ -136,6 +149,10 @@ pub(super) struct QueueLiveCounts { pub(super) dead_letters: usize, } +/// Running estimate of how long one delivery takes the actor to serve, +/// in microseconds. Written by the actor, read by admission. +pub(super) type ServiceEstimateUs = Arc; + pub(super) struct QueueDomainActor { pub(super) core: Arc, } @@ -148,6 +165,8 @@ pub(super) struct QueueDomainRuntime<'a> { pub struct QueueDomainSink { pub(super) core: Arc, pub(super) actor: ManagedActor, + /// Client requests currently blocked on the actor's reply. + pub(super) inflight_client_deliveries: Arc, } impl std::ops::Deref for QueueDomainRuntime<'_> { @@ -157,3 +176,154 @@ impl std::ops::Deref for QueueDomainRuntime<'_> { self.core } } + +/// Hard ceiling on concurrent client requests, whatever the measured service +/// time suggests. +pub(super) const QUEUE_ADMISSION_MAX_WINDOW: usize = 64; + +/// Fraction of the reply deadline the admitted backlog may consume, leaving +/// headroom for enqueue, scheduling, and a slower-than-average commit. +const QUEUE_ADMISSION_BUDGET_NUMERATOR: u32 = 4; +const QUEUE_ADMISSION_BUDGET_DENOMINATOR: u32 = 5; + +/// Assumed per-delivery service time until the actor has measured one. +const QUEUE_ADMISSION_ASSUMED_SERVICE_US: u64 = 5_000; + +/// How many client requests may be in flight against the queue actor. +/// +/// Queued concurrency adds no throughput: the actor serves deliveries one at a +/// time, so admitting `n` requests commits the tail caller to `n x +/// service_time`. A fixed window therefore cannot bound the deadline - at 20ms +/// per synchronous commit, a 64-deep window needs 1.28s and the tail caller +/// times out with an indeterminate outcome while its command still executes, +/// which is precisely what admission exists to prevent. +/// +/// The window is derived from observed service time instead, so the admitted +/// backlog stays inside the reply deadline as the backend gets slower. It never +/// drops below 1: the active operation is always admitted, or nothing would +/// ever run to produce a new measurement. +pub(super) fn queue_admission_window(service_us: u64) -> usize { + let deadline_us = u64::try_from(QUEUE_ACTOR_REPLY_TIMEOUT.as_micros()).unwrap_or(u64::MAX); + let budget_us = deadline_us.saturating_mul(u64::from(QUEUE_ADMISSION_BUDGET_NUMERATOR)) + / u64::from(QUEUE_ADMISSION_BUDGET_DENOMINATOR); + let service_us = service_us.max(1); + let window = usize::try_from(budget_us / service_us).unwrap_or(QUEUE_ADMISSION_MAX_WINDOW); + window.clamp(1, QUEUE_ADMISSION_MAX_WINDOW) +} + +/// Blend a fresh delivery duration into the running service estimate. +/// +/// A simple exponential average: fast enough to react to a backend slowdown +/// within a few deliveries, damped enough that one outlier does not slam the +/// window shut. +pub(super) fn blend_service_estimate(previous_us: u64, observed_us: u64) -> u64 { + if previous_us == 0 { + return observed_us.max(1); + } + ((previous_us * 3) + observed_us.max(1)) / 4 +} + +/// Admit one client delivery, sizing the window from measured service time and +/// preferring terminal actor failure over a retryable rejection. +/// +/// # Errors +/// +/// `MailboxFull` when the live actor already has as much work as its deadline +/// can serve, or `ActorStopped` when the actor has terminated. +pub(super) fn admit_client_delivery( + inflight: &Arc, + service: &ServiceEstimateUs, + actor_running: bool, +) -> Result { + let window = queue_admission_window(service.load(std::sync::atomic::Ordering::Relaxed)); + try_admit_queue_delivery(inflight, window) + .map_err(|error| classify_admission_failure(error, actor_running)) +} + +/// Fold one observed delivery duration into the shared estimate. +pub(super) fn record_service_sample(estimate: &ServiceEstimateUs, started_at: Instant) { + use std::sync::atomic::Ordering; + + let observed_us = u64::try_from(started_at.elapsed().as_micros()).unwrap_or(u64::MAX); + let previous = estimate.load(Ordering::Relaxed); + estimate.store( + blend_service_estimate(previous, observed_us), + Ordering::Relaxed, + ); +} + +/// A full window means "retry later" only while the actor is alive. +/// +/// A worker that fails closed leaves its mailbox - and every admitted slot - +/// alive for the sink's lifetime, so admission would keep answering +/// `MailboxFull` and clients would retry a dead domain forever. +pub(super) fn classify_admission_failure( + error: crate::runtime::DeliveryError, + actor_running: bool, +) -> crate::runtime::DeliveryError { + if actor_running { + error + } else { + crate::runtime::DeliveryError::ActorStopped + } +} + +/// Starting value for the service estimate before anything is measured. +pub(super) const fn assumed_service_us() -> u64 { + QUEUE_ADMISSION_ASSUMED_SERVICE_US +} + +/// Holds an admission slot until the queued command is finished with. +/// +/// The slot travels with the command rather than with the blocked caller. +/// A caller that gives up on `recv_timeout` has NOT cancelled anything: the +/// `Deliver` command is still queued and the actor will still run +/// `deliver_envelope`, only to find the reply channel gone. Releasing on +/// caller timeout would therefore recycle slots while the work they admitted +/// is still pending, letting a sustained burst pile accepted - and +/// indeterminate - mutations up to the full mailbox depth. +/// +/// Dropping with the command covers completion, actor death, and mailbox +/// teardown alike, so a slot can never leak. +#[derive(Debug)] +pub(super) struct QueueAdmissionSlot { + inflight: Arc, +} + +impl Drop for QueueAdmissionSlot { + fn drop(&mut self) { + self.inflight + .fetch_sub(1, std::sync::atomic::Ordering::AcqRel); + } +} + +/// Reserve an in-flight slot, or refuse the request. +/// +/// The reservation is a single atomic compare-and-update, so the limit holds +/// under concurrency: callers cannot collectively exceed it by all observing +/// room before any of them commits. +/// +/// # Errors +/// +/// Returns `MailboxFull` when the actor already has as many blocked callers as +/// its deadline can serve. That is deliberately the same error an actually-full +/// mailbox produces: nothing was enqueued, so ingress answers with a retryable +/// code and the client may safely re-send. +pub(super) fn try_admit_queue_delivery( + inflight: &Arc, + window: usize, +) -> Result { + use std::sync::atomic::Ordering; + + inflight + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < window).then_some(current + 1) + }) + .map(|_| QueueAdmissionSlot { + inflight: Arc::clone(inflight), + }) + .map_err(|current| crate::runtime::DeliveryError::MailboxFull { + capacity: window, + current_len: current, + }) +} diff --git a/src/domains/queue/sink/observability.rs b/src/domains/queue/sink/observability.rs new file mode 100644 index 00000000..91e93bb8 --- /dev/null +++ b/src/domains/queue/sink/observability.rs @@ -0,0 +1,74 @@ +//! Admin snapshot and metrics upkeep for the queue domain core. + +use super::model::{QueueDomainCore, QueueLiveCounts, QueueProjectionEntry, QueueProjectionState}; + +impl QueueDomainCore { + pub(super) fn mark_admin_snapshot_dirty(&self) { + self.projection.mark_dirty(); + self.refresh_metrics_gauges(); + } + + pub(super) fn refresh_metrics_gauges(&self) { + if let Some(metrics) = &self.metrics { + let counts = self.live_counts(); + metrics.set_ready_messages(counts.ready); + metrics.set_delayed_messages(counts.delayed); + metrics.set_inflight_messages(counts.inflight); + } + } + + pub(super) fn observe_histogram_us(&self, name: &str, value_us: u64) { + if let Some(metrics) = &self.metrics { + metrics.histogram_observe_us(name, value_us); + } else { + crate::observability::histogram_observe_us(name, value_us); + } + } + + pub(super) fn refresh_admin_snapshot_if_dirty(&self) { + self.sweep_idle_actors(); + self.projection + .refresh_if_dirty(|| self.collect_projection_state()); + } + + pub(super) fn collect_projection_state(&self) -> QueueProjectionState { + let actors = self.actors.lock(); + let families = self.families.lock(); + let entries = actors + .iter() + .map(|(key, warm_actor)| { + let actor = warm_actor.actor.lock(); + let ready_route = Self::queue_ready_route(key); + let subscriptions_active = families.get(&key.family.as_u64()).map_or(0, |state| { + state.for_each_matching_route(key.family, ready_route.as_str(), |_| {}) + }); + QueueProjectionEntry { + key: key.clone(), + snapshot: actor.admin_snapshot(), + subscriptions_active, + inflight: actor.admin_inflight(), + dead_letters: actor.admin_dead_letters(), + } + }) + .collect(); + + QueueProjectionState::from_entries(entries) + } + + pub(super) fn live_counts(&self) -> QueueLiveCounts { + let actors = self.actors.lock(); + let mut counts = QueueLiveCounts::default(); + + for warm_actor in actors.values() { + let actor_counts = warm_actor.actor.lock().live_counts(); + counts.ready = counts.ready.saturating_add(actor_counts.ready); + counts.delayed = counts.delayed.saturating_add(actor_counts.delayed); + counts.inflight = counts.inflight.saturating_add(actor_counts.inflight); + counts.dead_letters = counts + .dead_letters + .saturating_add(actor_counts.dead_letters); + } + counts.pending = counts.ready.saturating_add(counts.delayed); + counts + } +} diff --git a/src/domains/queue/sink/responses.rs b/src/domains/queue/sink/responses.rs new file mode 100644 index 00000000..3f52cace --- /dev/null +++ b/src/domains/queue/sink/responses.rs @@ -0,0 +1,98 @@ +//! Response and recovery-error routing back to clients. + +#[cfg(test)] +use super::model::FrameContext; +use super::model::{Envelope, Instant, QueueDomainCore}; + +impl QueueDomainCore { + /// Count a response the actor produced but the transport could not carry. + fn record_response_route_failure(&self) { + if let Some(metrics) = self.metrics.as_ref() { + metrics + .counter_inc(crate::domains::queue::metrics::METRIC_RESPONSE_ROUTE_FAILURES_TOTAL); + } + } + + pub(super) fn route_queue_response( + &self, + request_envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + response: &crate::domains::queue::QueueResponse, + ) { + #[cfg(test)] + { + let response_bytes = crate::dispatch::protocol::queue_codec::encode_response( + meta.message_type, + response, + ); + let response_ctx = FrameContext::new( + meta.session_id, + crate::protocol::test_support::channel_id_from_client(meta.channel), + crate::dispatch::protocol::tlv::MessageType::new(meta.message_type), + bytes::Bytes::from(response_bytes), + meta.route_family, + ); + if let Some(response_envelope) = request_envelope.try_reply_to(response_ctx) { + if let Err(error) = self.router.route(response_envelope) { + self.record_response_route_failure(); + tracing::warn!( + domain = "queue", + session = meta.session_id, + error = ?error, + "Failed to route queue response" + ); + } + } + } + + #[cfg(not(test))] + { + let response = crate::domains::queue::QueueClientResponse::new(meta, response.clone()); + if let Some(response_envelope) = request_envelope.try_reply_to(response) { + if let Err(error) = self.router.route(response_envelope) { + self.record_response_route_failure(); + tracing::warn!( + domain = "queue", + session = meta.session_id, + error = ?error, + "Failed to route queue response" + ); + } + } + } + } + + pub(super) fn route_queue_recovery_error( + &self, + request_envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + request_started: Option, + message: String, + ) { + tracing::error!( + domain = "queue", + family = meta.route_family.as_u64(), + error = %message, + "Queue actor recovery failed" + ); + let response = crate::domains::queue::QueueResponse::Error { message }; + self.route_queue_response(request_envelope, meta, &response); + if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { + metrics.record_failure(started_at); + } + } + + pub(super) fn queue_response_is_failure( + response: &crate::domains::queue::QueueResponse, + ) -> bool { + matches!( + response, + crate::domains::queue::QueueResponse::InvalidToken + | crate::domains::queue::QueueResponse::InflightExpired + | crate::domains::queue::QueueResponse::NotFound + | crate::domains::queue::QueueResponse::QueueNotFound + | crate::domains::queue::QueueResponse::BadRequest { .. } + | crate::domains::queue::QueueResponse::Error { .. } + ) + } +} diff --git a/src/domains/queue/sink/subscriptions.rs b/src/domains/queue/sink/subscriptions.rs new file mode 100644 index 00000000..d1d3544f --- /dev/null +++ b/src/domains/queue/sink/subscriptions.rs @@ -0,0 +1,219 @@ +//! Watch/unwatch subscription handling for queue domain frames. + +use super::delivery::QueueOpKind; +use super::model::{ + Envelope, Instant, QueueDomainCore, QueueSubscription, QueueSubscriptionMessage, + RoutedSubscriptionSet, +}; +use crate::runtime::routing::RouteFamily; +use std::sync::atomic::Ordering; + +type SubscriptionOutcome = ( + crate::domains::queue::QueueResponse, + Option<( + RouteFamily, + crate::runtime::matcher::Pattern, + u64, + u64, + crate::runtime::routing::RouteAddress, + )>, + bool, +); + +impl QueueDomainCore { + pub(super) fn handle_subscription_frame( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + request_started: Option, + sub_msg: QueueSubscriptionMessage, + ) { + let (response, initial_watch_snapshot, state_changed) = match sub_msg { + QueueSubscriptionMessage::Watch { + family_id, + pattern, + session_id, + subscriber, + } => self.handle_watch_subscription( + envelope, meta, family_id, &pattern, session_id, subscriber, + ), + QueueSubscriptionMessage::Unwatch { + family_id, + pattern, + session_id, + subscriber, + } => self.handle_unwatch_subscription( + envelope, + meta, + family_id, + &pattern, + session_id, + &subscriber, + ), + }; + + self.route_queue_response(envelope, meta, &response); + if state_changed { + self.mark_admin_snapshot_dirty(); + } + if let Some((family_id, pattern, session_id, subscription_id, subscriber)) = + initial_watch_snapshot + { + self.emit_current_ready_notifications_for_watch( + family_id, + &pattern, + session_id, + subscription_id, + &subscriber, + ); + } + + self.record_operation_metrics(request_started, &response, QueueOpKind::InflightExpired); + } + + fn handle_watch_subscription( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + family_id: RouteFamily, + pattern: &crate::runtime::routing::Route, + session_id: u64, + subscriber: crate::runtime::routing::RouteAddress, + ) -> SubscriptionOutcome { + if !Self::valid_subscription_request(envelope, meta, family_id, session_id, &subscriber) { + return ( + crate::domains::queue::QueueResponse::BadRequest { + reason: "route family mismatch".to_string(), + }, + None, + false, + ); + } + let pattern_str = pattern.as_str(); + let parsed_pattern = match crate::runtime::DomainKind::Queue + .descriptor() + .compile_registration_pattern(pattern_str) + { + Ok(pattern) => pattern, + Err(reason) => { + return ( + crate::domains::queue::QueueResponse::InvalidSubscriptionPattern { reason }, + None, + false, + ); + } + }; + let (subscription_id, state_changed) = { + let mut families = self.families.lock(); + let state = families + .entry(family_id.as_u64()) + .or_insert_with(RoutedSubscriptionSet::new); + + if let Some(id) = state.find_existing_id(session_id, pattern_str) { + (id, false) + } else { + if state.wildcard_registration_limit_reached(session_id, &parsed_pattern) { + return ( + crate::domains::queue::QueueResponse::SubscriptionLimit, + None, + false, + ); + } + let Ok(id) = self.next_sub_id.fetch_update( + Ordering::Relaxed, + Ordering::Relaxed, + |current| current.checked_add(1), + ) else { + let state_empty = state.is_empty(); + if state_empty { + families.remove(&family_id.as_u64()); + } + return ( + crate::domains::queue::QueueResponse::BadRequest { + reason: "subscription ID space exhausted".to_string(), + }, + None, + false, + ); + }; + state.insert( + family_id, + QueueSubscription { + pattern: parsed_pattern.clone(), + session_id, + subscription_id: id, + subscriber: subscriber.clone(), + }, + ); + (id, true) + } + }; + + ( + crate::domains::queue::QueueResponse::WatchOk { subscription_id }, + Some(( + family_id, + parsed_pattern, + session_id, + subscription_id, + subscriber, + )), + state_changed, + ) + } + + fn handle_unwatch_subscription( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + family_id: RouteFamily, + pattern: &crate::runtime::routing::Route, + session_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + ) -> SubscriptionOutcome { + if !Self::valid_subscription_request(envelope, meta, family_id, session_id, subscriber) { + return ( + crate::domains::queue::QueueResponse::BadRequest { + reason: "route family mismatch".to_string(), + }, + None, + false, + ); + } + if let Err(reason) = crate::runtime::DomainKind::Queue + .descriptor() + .compile_registration_pattern(pattern.as_str()) + { + return ( + crate::domains::queue::QueueResponse::InvalidSubscriptionPattern { reason }, + None, + false, + ); + } + + let mut families = self.families.lock(); + let remove_family = if let Some(state) = families.get_mut(&family_id.as_u64()) { + state.remove_session_pattern(family_id, session_id, pattern.as_str()); + state.is_empty() + } else { + false + }; + if remove_family { + families.remove(&family_id.as_u64()); + } + (crate::domains::queue::QueueResponse::UnwatchOk, None, true) + } + + fn valid_subscription_request( + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + family_id: RouteFamily, + session_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + ) -> bool { + family_id == meta.route_family + && *subscriber.family() == family_id + && session_id == meta.session_id + && envelope.source().is_none_or(|source| source == subscriber) + } +} diff --git a/src/domains/queue/sink/tests/actor_delivery.rs b/src/domains/queue/sink/tests/actor_delivery.rs index bfa2db0c..c64f4beb 100644 --- a/src/domains/queue/sink/tests/actor_delivery.rs +++ b/src/domains/queue/sink/tests/actor_delivery.rs @@ -424,6 +424,86 @@ fn should_reserve_concrete_items_given_wildcards_in_unknown_queue_segments() { ); } +#[test] +fn should_stop_wildcard_reserve_after_wire_budget_exhaustion() { + // Arrange + let family = RouteFamily::new(1); + let keys = ["a", "b", "c"].map(|resource| crate::domains::queue::QueueKey { + family, + realm: "acme".to_string(), + area: "jobs".to_string(), + resource: resource.to_string(), + }); + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let mut first = crate::domains::queue::QueueActor::new( + family, + keys[0].clone(), + store.clone(), + None, + crate::utils::idempotency::default_dedup_store(), + ); + let first_route = crate::domains::queue::sink::QueueDomainCore::queue_ready_route(&keys[0]); + let first_body_bytes = crate::domains::queue::protocol::MAX_QUEUE_RESPONSE_PAYLOAD_BYTES + - crate::domains::queue::protocol::RECEIVED_RESPONSE_HEADER_BYTES + - crate::domains::queue::protocol::RESERVED_MESSAGE_WIRE_OVERHEAD_BYTES + - crate::domains::queue::protocol::ROUTED_MESSAGE_WIRE_OVERHEAD_BYTES + - first_route.as_str().len() + - 1; + first.handle_send(Bytes::from(vec![0x5a; first_body_bytes]), None); + let mut blocked = crate::domains::queue::QueueActor::new( + family, + keys[1].clone(), + store.clone(), + None, + crate::utils::idempotency::default_dedup_store(), + ); + blocked.handle_send(Bytes::from_static(b"blocked"), None); + let clock = DlqSeedClock::new(); + let mut untouched = crate::domains::queue::QueueActor::with_clock( + family, + keys[2].clone(), + store.clone(), + Box::new(clock.clone()), + None, + crate::utils::idempotency::default_dedup_store(), + ); + untouched.handle_send(Bytes::from_static(b"due"), Some(1)); + let sink = new_queue_domain_sink( + store, + Arc::new(Router::new()), + crate::control::admin::read_model::AdminReadModel::new(), + cntryl_midge::WriteOptions::buffered(), + ); + for (key, actor) in keys.iter().cloned().zip([first, blocked, untouched]) { + sink.install_actor_for_tests(key, actor); + } + assert_eq!( + queue_snapshot(&sink, family, "queue://acme/jobs/c").messages_delayed, + 1 + ); + sink.stop_actor_for_tests(); + clock.advance(Duration::from_secs(2)); + + // Act + let response = sink.core.handle_wildcard_receive_for_tests( + family, + &crate::runtime::matcher::Pattern::new("queue://acme/jobs/*"), + 8, + 30, + Some(3), + ); + + // Assert + let crate::domains::queue::QueueResponse::ReceivedRouted { messages } = response else { + panic!("expected routed queue response"); + }; + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].route, first_route); + let untouched_snapshot = queue_snapshot(&sink, family, "queue://acme/jobs/c"); + assert_eq!(untouched_snapshot.messages_delayed, 1); + assert_eq!(untouched_snapshot.messages_ready, 0); +} + #[test] fn should_surface_startup_inventory_failure_to_wildcard_reserve() { // Arrange @@ -779,6 +859,65 @@ fn should_route_queue_live_counts_through_managed_actor() { assert_eq!(ready_messages, 0); } +#[test] +fn should_bound_concrete_reserve_response_before_messages_become_inflight() { + // Arrange + let family = RouteFamily::new(1); + let queue_route = "queue://acme/jobs/wire-capacity"; + let sender_address = RouteAddress::new(family, Route::new("inbox://session/7")); + let worker_address = RouteAddress::new(family, Route::new("inbox://session/8")); + let queue_address = RouteAddress::new(family, Route::new("queue://inbound")); + let sender_mailbox = Arc::new(Mailbox::new(2)); + let worker_mailbox = Arc::new(Mailbox::new(2)); + let router = Arc::new(Router::new()); + router.register(sender_address.clone(), sender_mailbox.clone()); + router.register(worker_address.clone(), worker_mailbox.clone()); + let sink = new_queue_domain_sink( + crate::testkit::create_test_engine_with_cfs(vec![1]), + router, + crate::control::admin::read_model::AdminReadModel::new(), + cntryl_midge::WriteOptions::buffered(), + ); + let body = vec![0x5a; 1024]; + for _ in 0..100 { + sink.deliver(Envelope::from_route( + sender_address.clone(), + queue_address.clone(), + FrameContext::new( + 7, + ChannelId::Pub, + MessageType::new(200), + encode_queue_send(queue_route, &body), + family, + ), + )) + .expect("enqueue queue message"); + let _response = receive_queue_frame(&sender_mailbox, "enqueue response"); + } + + // Act + sink.deliver(Envelope::from_route( + worker_address, + queue_address, + FrameContext::new( + 8, + ChannelId::Pub, + MessageType::new(202), + encode_queue_reserve(queue_route, 30, 100), + family, + ), + )) + .expect("reserve queue batch"); + let response = receive_queue_frame(&worker_mailbox, "reserve response"); + + // Assert + assert!(u16::try_from(response.payload.len()).is_ok()); + assert_eq!(decode_concrete_reserve_response(&response).len(), 62); + let snapshot = queue_snapshot(&sink, family, queue_route); + assert_eq!(snapshot.messages_ready, 38); + assert_eq!(snapshot.messages_inflight, 62); +} + #[test] fn should_route_queue_cleanup_through_managed_actor() { // Arrange @@ -839,10 +978,19 @@ fn should_route_queue_cleanup_through_managed_actor() { // Act sink.stop_actor_for_tests(); - sink.cleanup_session(worker_session_id); + let cleanup_result = sink.cleanup_session(worker_session_id); let snapshot = queue_snapshot(&sink, family, queue_route); // Assert + // The command cannot run against a stopped actor, and the caller is now + // told so rather than being handed a silent success. + assert!( + matches!( + cleanup_result, + Err(crate::runtime::DeliveryError::ActorStopped) + ), + "expected a reported failure, got {cleanup_result:?}" + ); assert!(!sink.is_actor_running()); assert_eq!(snapshot.messages_inflight, 1); assert_eq!(snapshot.messages_ready, 0); @@ -890,6 +1038,50 @@ fn should_route_queue_runtime_sweep_through_managed_actor() { assert!(sink.dirty_fast_flush_contains_family_for_tests(1)); } +#[test] +fn should_coalesce_queue_runtime_sweeps_while_actor_is_busy() { + // Arrange + let sink = new_queue_domain_sink( + crate::testkit::create_test_engine_with_cfs(vec![1]), + Arc::new(Router::new()), + crate::control::admin::read_model::AdminReadModel::new(), + cntryl_midge::WriteOptions::best_effort(), + ); + let pending_reserves = sink.core.pending_reserves.lock(); + + // Act + let first_enqueued = sink.request_runtime_sweep_at(Instant::now()); + let second_enqueued = sink.request_runtime_sweep_at(Instant::now()); + + // Assert + assert!(first_enqueued); + assert!(!second_enqueued); + drop(pending_reserves); +} + +#[test] +fn should_clear_runtime_sweep_pending_when_sweep_panics() { + // Arrange + let sink = new_queue_domain_sink( + crate::testkit::create_test_engine_with_cfs(vec![1]), + Arc::new(Router::new()), + crate::control::admin::read_model::AdminReadModel::new(), + cntryl_midge::WriteOptions::best_effort(), + ); + sink.panic_next_runtime_sweep_for_tests(); + + // Act + assert!(sink.request_runtime_sweep_at(Instant::now())); + let deadline = Instant::now() + Duration::from_secs(1); + while sink.actor_health_snapshot().running && Instant::now() < deadline { + std::thread::yield_now(); + } + + // Assert + assert!(!sink.actor_health_snapshot().running); + assert!(!sink.runtime_sweep_pending_for_tests()); +} + #[test] fn should_route_queue_dead_letter_replay_through_managed_actor() { // Arrange @@ -955,3 +1147,218 @@ fn should_route_queue_dead_letter_purge_through_managed_actor() { assert_eq!(dead_letters, 1); assert!(sink.actors_are_empty_for_tests()); } + +#[test] +fn should_reject_surplus_queue_load_instead_of_accepting_then_timing_out() { + // Arrange + // Queue is the only domain that commits storage synchronously per request + // inside a single-threaded actor while a caller blocks on the reply. Work + // admitted beyond what the deadline can serve becomes an indeterminate + // outcome the client dare not retry; refusing it keeps it retryable. + use crate::domains::queue::sink::model::{queue_admission_window, try_admit_queue_delivery}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let inflight = Arc::new(AtomicUsize::new(0)); + let window = queue_admission_window(super::super::model::assumed_service_us()); + + // Act + let held = (0..window) + .map(|_| try_admit_queue_delivery(&inflight, window).expect("slot within the limit")) + .collect::>(); + let refused = try_admit_queue_delivery(&inflight, window); + + // Assert + assert!( + matches!(refused, Err(DeliveryError::MailboxFull { .. })), + "surplus must be refused as never-enqueued, got {refused:?}" + ); + assert_eq!(inflight.load(Ordering::Acquire), window); + + // Slots are released when the COMMAND is finished with, not when a caller + // stops waiting: a `recv_timeout` cancels nothing, so recycling on caller + // timeout would admit fresh work on top of still-pending mutations. + drop(held); + assert_eq!(inflight.load(Ordering::Acquire), 0); + assert!(try_admit_queue_delivery(&inflight, window).is_ok()); +} + +#[test] +fn should_hold_queue_admission_limit_under_concurrent_callers() { + // Arrange + // Sampling a depth and then enqueueing is check-then-act: concurrent + // callers can all observe room before any of them commits, and collectively + // blow past the limit in exactly the burst the limit exists to bound. The + // reservation must therefore be a single atomic step. + use crate::domains::queue::sink::model::{queue_admission_window, try_admit_queue_delivery}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let inflight = Arc::new(AtomicUsize::new(0)); + let window = queue_admission_window(super::super::model::assumed_service_us()); + let admitted = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(std::sync::Barrier::new(32)); + + // Act + let handles = (0..32) + .map(|_| { + let inflight = inflight.clone(); + let admitted = admitted.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + // Each thread tries for more slots than the limit allows, and + // holds every one it wins for the duration. + let mut held = Vec::new(); + for _ in 0..8 { + if let Ok(slot) = try_admit_queue_delivery(&inflight, window) { + admitted.fetch_add(1, Ordering::AcqRel); + held.push(slot); + } + } + // Keep them until every thread has finished competing. + barrier.wait(); + drop(held); + }) + }) + .collect::>(); + for handle in handles { + handle.join().unwrap(); + } + + // Assert + assert_eq!( + admitted.load(Ordering::Acquire), + window, + "32 concurrent callers must not collectively exceed the admission limit" + ); + assert_eq!( + inflight.load(Ordering::Acquire), + 0, + "every slot must be released" + ); +} + +#[test] +fn should_hold_queue_admission_while_a_timed_out_command_is_still_pending() { + // Arrange + // A caller that gives up on `recv_timeout` has cancelled nothing: its + // `Deliver` command stays queued and the actor will still run + // `deliver_envelope`. If the slot were tied to the caller, a sustained + // burst would recycle slots while accepted mutations were still pending + // and pile indeterminate work up to the full mailbox depth. + use crate::domains::queue::sink::model::{try_admit_queue_delivery, QueueAdmissionSlot}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let inflight = Arc::new(AtomicUsize::new(0)); + let slot = try_admit_queue_delivery(&inflight, 1).expect("slot"); + + // Act + // Hand the slot to the queued command and let the caller's scope end, + // exactly as a timed-out delivery does. + let queued: Option = Some(slot); + let caller_gave_up = inflight.load(Ordering::Acquire); + + // Assert + assert_eq!( + caller_gave_up, 1, + "the slot must still be held while the command is queued" + ); + drop(queued); + assert_eq!( + inflight.load(Ordering::Acquire), + 0, + "the slot must be released when the command is finished with" + ); +} + +#[test] +fn should_size_queue_admission_to_the_reply_deadline() { + // Arrange + // A fixed window cannot bound the deadline. Queued concurrency adds no + // throughput - the actor serves commands one at a time - so admitting N + // requests commits the tail caller to N x service_time. At 20ms per + // synchronous commit a 64-deep window needs 1.28s, past + // QUEUE_ACTOR_REPLY_TIMEOUT, so the tail times out with an indeterminate + // outcome and its command still executes: exactly what the gate exists to + // prevent. The window must therefore follow observed service time. + use crate::domains::queue::sink::model::{queue_admission_window, QUEUE_ADMISSION_MAX_WINDOW}; + + // Act + let slow = queue_admission_window(20_000); + let quick = queue_admission_window(1_000); + let pathological = queue_admission_window(5_000_000); + + // Assert + let deadline_us = + u64::try_from(crate::domains::queue::sink::model::QUEUE_ACTOR_REPLY_TIMEOUT.as_micros()) + .expect("deadline fits u64"); + assert!( + u64::try_from(slow).unwrap() * 20_000 <= deadline_us, + "a {slow}-deep window at 20ms each overruns the {deadline_us}us deadline" + ); + assert!( + slow < QUEUE_ADMISSION_MAX_WINDOW, + "slow service must shrink the window below the cap" + ); + assert!(quick > slow, "faster service should admit more"); + assert!(quick <= QUEUE_ADMISSION_MAX_WINDOW); + assert_eq!( + pathological, 1, + "service slower than the whole deadline still admits the active operation" + ); +} + +#[test] +fn should_report_a_stopped_actor_rather_than_admission_backpressure() { + // Arrange + // A worker that fails closed leaves its mailbox, and every admitted slot, + // alive for the sink's lifetime. Admission would then keep answering + // MailboxFull - a retryable code - so clients would retry a dead domain + // forever instead of seeing a terminal failure. + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = new_queue_domain_sink( + store, + Arc::new(Router::new()), + admin_read_model, + cntryl_midge::WriteOptions::buffered(), + ); + let window = crate::domains::queue::sink::model::queue_admission_window( + sink.core + .delivery_service_us + .load(std::sync::atomic::Ordering::Relaxed), + ); + let held = (0..window) + .map(|_| { + crate::domains::queue::sink::model::try_admit_queue_delivery( + &sink.inflight_client_deliveries, + window, + ) + .expect("slot") + }) + .collect::>(); + + // Act + sink.stop_actor_for_tests(); + let refused = sink.deliver(Envelope::new( + RouteAddress::new(RouteFamily::new(1), Route::new("queue://acme/jobs/dead")), + crate::runtime::SessionCleanup { session_id: 1 }, + )); + let client_refused = sink.deliver(Envelope::new( + RouteAddress::new(RouteFamily::new(1), Route::new("queue://acme/jobs/dead")), + crate::dispatch::protocol::frame_context::FrameContext::new( + 1, + crate::dispatch::protocol::frame::ChannelId::Pub, + crate::dispatch::protocol::tlv::MessageType::new(401), + bytes::Bytes::new(), + RouteFamily::new(1), + ), + )); + + // Assert + assert!( + matches!(client_refused, Err(DeliveryError::ActorStopped)), + "a dead actor must be terminal, not retryable, got {client_refused:?}" + ); + let _ = refused; + drop(held); +} diff --git a/src/domains/queue/sink/tests/cleanup_and_eviction.rs b/src/domains/queue/sink/tests/cleanup_and_eviction.rs index 8f3bc052..1eac49bf 100644 --- a/src/domains/queue/sink/tests/cleanup_and_eviction.rs +++ b/src/domains/queue/sink/tests/cleanup_and_eviction.rs @@ -217,6 +217,60 @@ fn should_cleanup_queue_inflight_for_disconnected_session() { assert!(admin_read_model.queue_inflight(None).is_empty()); } +#[test] +fn should_reject_stale_reserve_after_disconnect_cleanup_marks_session() { + // Arrange + let family = RouteFamily::new(1); + let session_id = 9; + let queue_route = "queue://acme/jobs/emails"; + let queue_address = RouteAddress::new(family, Route::new("queue://inbound")); + let worker_address = RouteAddress::new(family, Route::new("inbox://session/9")); + let worker_mailbox = Arc::new(Mailbox::new(8)); + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let router = Arc::new(Router::new()); + router.register(worker_address.clone(), worker_mailbox.clone()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = new_queue_domain_sink( + store, + router, + admin_read_model, + cntryl_midge::WriteOptions::buffered(), + ); + + // Act: cleanup for this session runs and completes before the stale + // reserve below is processed - equivalent to what the control-plane + // mailbox lane guarantees a real disconnect races against a queued + // normal-lane request. + sink.deliver(Envelope::new( + RouteAddress::new(family, Route::new("queue://cleanup")), + crate::runtime::SessionCleanup { session_id }, + )) + .expect("cleanup queue session"); + + deliver_reserve( + &sink, + worker_address, + queue_address, + session_id, + queue_route, + family, + ); + + // Assert: the stale reserve from the now-cleaned-up session is rejected + // instead of being accepted as a pending long-poll reserve for it. + let response_envelope = worker_mailbox + .receiver() + .try_recv() + .expect("stale reserve response"); + let frame = response_envelope + .into_payload::() + .expect("queue response frame"); + let (_code, message) = + crate::dispatch::protocol::error_codes::decode_error_body(frame.payload.as_ref()) + .expect("bad request error body"); + assert_eq!(message, "session already closed"); +} + #[test] fn should_reject_queue_inflight_followups_from_non_owner_session() { // Arrange @@ -610,3 +664,52 @@ fn should_not_evict_idle_queue_actor_with_live_inflight() { "actors with live inflight entries must stay warm until the inflight entry is gone" ); } + +#[test] +fn should_admit_session_cleanup_even_when_client_admission_is_exhausted() { + // Arrange + // Disconnect cleanup arrives through `router.route`, so it lands on the + // same normal-lane `deliver` as client traffic. If the admission gate can + // refuse it, a stalled queue starves cleanup for the whole retry window and + // the ticket is abandoned - leaving that session's inflight reservations + // and watches held with no queued command left to release them. Cleanup is + // control-plane work and must not be rationed by client load. + let store = crate::testkit::create_test_engine_with_cfs(vec![1]); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = new_queue_domain_sink( + store, + Arc::new(Router::new()), + admin_read_model, + cntryl_midge::WriteOptions::buffered(), + ); + let family = RouteFamily::new(1); + + // Hold every admission slot, as a stalled actor under load would. + let window = crate::domains::queue::sink::model::queue_admission_window( + sink.core + .delivery_service_us + .load(std::sync::atomic::Ordering::Relaxed), + ); + let held = (0..window) + .map(|_| { + crate::domains::queue::sink::model::try_admit_queue_delivery( + &sink.inflight_client_deliveries, + window, + ) + .expect("slot") + }) + .collect::>(); + + // Act + let cleanup = sink.deliver(Envelope::new( + RouteAddress::new(family, Route::new("queue://cleanup-under-pressure")), + crate::runtime::SessionCleanup { session_id: 4_242 }, + )); + + // Assert + assert!( + cleanup.is_ok(), + "session cleanup must not be refused by client admission, got {cleanup:?}" + ); + drop(held); +} diff --git a/src/domains/queue/sink/tests/routing_watch_and_admin/admin.rs b/src/domains/queue/sink/tests/routing_watch_and_admin/admin.rs index d1da15d2..cd44c908 100644 --- a/src/domains/queue/sink/tests/routing_watch_and_admin/admin.rs +++ b/src/domains/queue/sink/tests/routing_watch_and_admin/admin.rs @@ -77,7 +77,7 @@ pub(super) fn should_delay_ready_notification_until_delayed_message_is_promoted( // Act std::thread::sleep(Duration::from_millis(1_100)); - harness.sink.sweep_runtime_state(); + harness.sink.sweep_runtime_state_at(Instant::now()); let (delivered_subscription_id, delivered_route, ready, delayed, inflight) = harness.next_watch_notification(); diff --git a/src/domains/rpc/metrics.rs b/src/domains/rpc/metrics.rs index 5a6b05de..f118ad8e 100644 --- a/src/domains/rpc/metrics.rs +++ b/src/domains/rpc/metrics.rs @@ -8,6 +8,12 @@ pub const METRIC_LATENCY_MS: &str = "fitz_rpc_latency_ms"; pub const METRIC_WORKERS_GAUGE: &str = "fitz_rpc_workers_gauge"; pub const METRIC_PENDING_GAUGE: &str = "fitz_rpc_pending_gauge"; pub const METRIC_RESPONSE_DROPS_TOTAL: &str = "fitz_rpc_response_drops_total"; +/// Incremented once per route family whose handler panics and fails closed. +/// Non-fatal and scoped to that family only (see +/// `FamilyActorPoolRuntime::is_family_running`) — this is the only +/// operator-visible signal for a permanently degraded realm, since a +/// per-family failure deliberately does not flip domain-wide health/liveness. +pub const METRIC_FAMILY_FAILED_CLOSED_TOTAL: &str = "fitz_rpc_family_failed_closed_total"; #[derive(Clone)] pub struct RpcMetrics { diff --git a/src/domains/rpc/sink/cleanup.rs b/src/domains/rpc/sink/cleanup.rs new file mode 100644 index 00000000..6d5d8f36 --- /dev/null +++ b/src/domains/rpc/sink/cleanup.rs @@ -0,0 +1,167 @@ +//! Session/worker disconnect cleanup and stale queued-request rejection. +//! +//! `SessionCleanup` is delivered on the high-priority mailbox lane, so it can +//! pass an older, already-queued normal-lane request from the same session. +//! Remembering the cleaned-up session lets that stale request fail instead of +//! silently recreating a worker registration or pending request for a session +//! that is already gone and will never be cleaned up again. + +use super::response_forwarder::RpcResponseForwarder; +use super::state_model::{ + Envelope, RouteAddress, RpcDomainRuntime, RpcPendingErrorDelivery, RpcSessionCleanupResult, + RpcWorkerCleanupResult, RPC_WORKER_NOT_FOUND_ERROR, +}; + +impl RpcDomainRuntime<'_> { + pub(super) fn is_cleaned_up_session(&self, session_id: u64) -> bool { + self.cleaned_up_sessions.lock().contains(session_id) + } + + pub(super) fn handle_cleanup_envelope(&self, envelope: &Envelope) -> bool { + if let Some(cleanup) = envelope.payload::() { + // Mark first so an older normal-lane request that cleanup jumped + // over cannot recreate a worker registration or pending request + // for this session below. + self.cleaned_up_sessions.lock().mark(cleanup.session_id); + let cleanup_result = self.apply_session_cleanup(cleanup.session_id); + self.forward_worker_disconnect_errors(cleanup_result.disconnect_deliveries); + return true; + } + + false + } + + pub(super) fn apply_session_cleanup(&self, session_id: u64) -> RpcSessionCleanupResult { + let cleanup_result = { + let mut state = self.state.lock(); + state.cleanup_session(session_id) + }; + + self.gauge_set("rpc_pending_requests", cleanup_result.pending_len as u64); + self.release_global_pending(cleanup_result.removed_pending); + if cleanup_result.removed_registrations > 0 { + self.counter_add( + "rpc_cleanup_workers_removed_total", + cleanup_result.removed_registrations as u64, + ); + } + if cleanup_result.detached_callers > 0 { + self.counter_add( + "rpc_cleanup_callers_detached_total", + cleanup_result.detached_callers as u64, + ); + } + if cleanup_result.removed_pending > 0 { + self.counter_add( + "rpc_cleanup_pending_removed_total", + cleanup_result.removed_pending as u64, + ); + } + if cleanup_result.removed_registrations > 0 + || cleanup_result.detached_callers > 0 + || cleanup_result.removed_pending > 0 + { + self.schedule_admin_snapshot(false); + } + self.refresh_metrics_gauges(); + + tracing::debug!( + domain = "rpc", + session_id, + removed_workers = cleanup_result.removed_registrations, + detached_callers = cleanup_result.detached_callers, + removed_pending = cleanup_result.removed_pending, + pending_len = cleanup_result.pending_len, + "RPC session cleanup applied" + ); + + cleanup_result + } + + pub(super) fn apply_worker_unsubscribe( + &self, + worker_addr: &RouteAddress, + session_id: u64, + ) -> RpcWorkerCleanupResult { + let cleanup_result = { + let mut state = self.state.lock(); + state.unregister_registration(worker_addr, session_id) + }; + + self.gauge_set("rpc_pending_requests", cleanup_result.pending_len as u64); + self.release_global_pending(cleanup_result.removed_pending); + if cleanup_result.removed_registrations > 0 { + self.counter_add( + "rpc_cleanup_workers_removed_total", + cleanup_result.removed_registrations as u64, + ); + } + if cleanup_result.removed_pending > 0 { + self.counter_add( + "rpc_cleanup_pending_removed_total", + cleanup_result.removed_pending as u64, + ); + } + if cleanup_result.removed_registrations > 0 || cleanup_result.removed_pending > 0 { + self.schedule_admin_snapshot(false); + } + self.refresh_metrics_gauges(); + + tracing::debug!( + domain = "rpc", + worker = worker_addr.route().as_str(), + session_id, + removed_workers = cleanup_result.removed_registrations, + removed_pending = cleanup_result.removed_pending, + pending_len = cleanup_result.pending_len, + "RPC worker cleanup applied" + ); + + cleanup_result + } + + pub(super) fn forward_pending_error_deliveries( + &self, + error_deliveries: Vec, + error_code: u16, + error_message: &'static str, + forwarded_counter: &str, + dropped_counter: &str, + ) { + if error_deliveries.is_empty() { + return; + } + + for delivery in error_deliveries { + let correlation_id = delivery.correlation_id; + let response_envelope = + RpcResponseForwarder::terminal_error_envelope(delivery, error_code, error_message); + + if let Err(error) = self.router.route(response_envelope) { + self.counter_inc(dropped_counter); + tracing::warn!( + domain = "rpc", + correlation_id = %correlation_id, + error_code, + error = ?error, + "Failed to forward RPC terminal error to requester" + ); + } else { + self.counter_inc(forwarded_counter); + } + } + } + + pub(super) fn forward_worker_disconnect_errors( + &self, + disconnect_deliveries: Vec, + ) { + self.forward_pending_error_deliveries( + disconnect_deliveries, + crate::dispatch::protocol::error_codes::rpc::ERR_WORKER_NOT_FOUND, + RPC_WORKER_NOT_FOUND_ERROR, + "rpc_worker_disconnect_errors_forwarded_total", + "rpc_worker_disconnect_errors_dropped_total", + ); + } +} diff --git a/src/domains/rpc/sink/delivery.rs b/src/domains/rpc/sink/delivery.rs new file mode 100644 index 00000000..400a05aa --- /dev/null +++ b/src/domains/rpc/sink/delivery.rs @@ -0,0 +1,458 @@ +//! Request admission decisions and delivery to workers. +//! +//! Covers the path from "a parsed `Request` message" through admission +//! (accept/queue/reject), forwarding to a worker, and draining the +//! route-local queue once a worker becomes available again. + +#[cfg(test)] +use super::state_model::RPC_MSG_TYPE_REQUEST; +use super::state_model::{ + session_inbox_address, DeliveryError, Envelope, Instant, RpcDeliveryOutcome as DeliveryOutcome, + RpcDomainRuntime, RpcPendingErrorDelivery, RpcPendingRequest, RpcQueuedDispatch, + RpcRequestRejection, RpcRequestState, RpcWorkerDispatch, RPC_BACKPRESSURE_ERROR, + RPC_DUPLICATE_CORRELATION_ERROR, RPC_MAX_PENDING_REQUESTS, RPC_NO_WORKERS_ERROR, + RPC_WORKER_NOT_FOUND_ERROR, +}; +#[cfg(test)] +use crate::dispatch::protocol::frame_context::FrameContext; +use crate::domains::rpc::protocol::RpcRequest; +#[cfg(not(test))] +use crate::domains::rpc::RpcWorkerRequestDelivery; + +struct RejectionSpec { + metric: &'static str, + error_code: u16, + message: &'static str, + reason: &'static str, +} + +const REJECTION_SPECS: [RejectionSpec; 4] = [ + RejectionSpec { + metric: "rpc_requests_rejected_duplicate_correlation_total", + error_code: crate::dispatch::protocol::error_codes::rpc::ERR_RPC_DUPLICATE_CORRELATION, + message: RPC_DUPLICATE_CORRELATION_ERROR, + reason: "duplicate live correlation", + }, + RejectionSpec { + metric: "rpc_requests_rejected_no_worker_total", + error_code: crate::dispatch::protocol::error_codes::rpc::ERR_ROUTE_NOT_REGISTERED, + message: RPC_NO_WORKERS_ERROR, + reason: "no matching worker registration", + }, + RejectionSpec { + metric: "rpc_requests_rejected_backpressure_total", + error_code: crate::dispatch::protocol::error_codes::rpc::ERR_RPC_BACKPRESSURE, + message: RPC_BACKPRESSURE_ERROR, + reason: "global pending capacity", + }, + RejectionSpec { + metric: "rpc_requests_rejected_backpressure_total", + error_code: crate::dispatch::protocol::error_codes::rpc::ERR_RPC_BACKPRESSURE, + message: RPC_BACKPRESSURE_ERROR, + reason: "route pending capacity", + }, +]; + +/// Aggregate name the admin `backpressure_rejects_total` field reads. +pub(in crate::domains::rpc::sink) const RPC_BACKPRESSURE_REJECTS_METRIC: &str = + "rpc_backpressure_rejects_total"; + +impl RpcDomainRuntime<'_> { + pub(super) fn handle_request_message( + &self, + envelope: &Envelope, + meta: &crate::runtime::ClientFrameMeta, + req: RpcRequest, + ) -> DeliveryOutcome { + self.expire_timed_out_requests_inline_if_due(); + self.counter_inc("rpc_requests_total"); + let caller_inbox_addr = envelope + .source() + .cloned() + .unwrap_or_else(|| session_inbox_address(meta.route_family, meta.session_id)); + + let metrics_enabled = self.metrics.is_some(); + let state_wait_start = metrics_enabled.then(Instant::now); + let mut state = self.state.lock(); + let state_wait_us = state_wait_start.map_or(0, Self::elapsed_micros_u64); + let state_hold_start = metrics_enabled.then(Instant::now); + let dispatch = RpcRequestState::dispatch_or_queue( + &mut *state, + req, + meta.session_id, + caller_inbox_addr, + self.request_timeout, + self.route_pending_capacity, + RPC_MAX_PENDING_REQUESTS, + self.enforce_global_pending_count + .then_some(self.global_pending_count.as_ref()), + ); + let state_hold_us = state_hold_start.map_or(0, Self::elapsed_micros_u64); + drop(state); + + self.observe_request_state_metrics(0, state_wait_us, state_hold_us, 0); + + match dispatch { + super::state_model::RpcRequestDispatch::Rejected { request, reason } => { + self.reject_with_spec(envelope, meta, &request, reason) + } + super::state_model::RpcRequestDispatch::Queued { + route, + correlation_id, + live_request_count, + } => self.accept_queued_request( + &route, + meta.route_family, + correlation_id, + live_request_count, + ), + super::state_model::RpcRequestDispatch::Immediate { + request, + registration, + live_request_count, + } => self.forward_immediate_request( + envelope, + meta, + request, + ®istration, + live_request_count, + ), + } + } + + fn observe_request_state_metrics( + &self, + route_registry_lookup_us: u64, + state_wait_us: u64, + state_hold_us: u64, + worker_selection_us: u64, + ) { + self.histogram_observe_us("rpc_route_registry_lookup_us", route_registry_lookup_us); + self.histogram_observe_us("rpc_dispatch_state_lock_us", state_wait_us); + self.histogram_observe_us("rpc_dispatch_state_wait_us", state_wait_us); + self.histogram_observe_us("rpc_dispatch_state_hold_us", state_hold_us); + self.histogram_observe_us("rpc_worker_selection_us", worker_selection_us); + } + + fn reject_with_spec( + &self, + envelope: &Envelope, + meta: &crate::runtime::ClientFrameMeta, + req: &RpcRequest, + reason: RpcRequestRejection, + ) -> DeliveryOutcome { + let spec = &REJECTION_SPECS[reason as usize]; + self.counter_inc(spec.metric); + // The admin surface reads one aggregate name. Without this, admission + // control rejections were invisible in `backpressure_rejects_total` + // even though they are exactly what it is meant to report. + if spec.error_code == crate::dispatch::protocol::error_codes::rpc::ERR_RPC_BACKPRESSURE { + self.counter_inc(RPC_BACKPRESSURE_REJECTS_METRIC); + } + tracing::warn!( + domain = "rpc", + correlation_id = %req.correlation_id, + route = req.route.as_str(), + reason = spec.reason, + "Rejected RPC request" + ); + self.reject_request_with_terminal_error(envelope, *meta, req, spec.error_code, spec.message) + } + + fn accept_queued_request( + &self, + route: &crate::runtime::routing::Route, + family: crate::runtime::routing::RouteFamily, + correlation_id: uuid::Uuid, + live_request_count: usize, + ) -> DeliveryOutcome { + self.histogram_observe_us("rpc_pending_track_us", 0); + self.histogram_observe_us("rpc_pending_route_index_us", 0); + self.gauge_set("rpc_pending_requests", live_request_count as u64); + self.schedule_admin_snapshot(false); + self.dispatch_queued_requests_for_family(family); + + tracing::debug!( + domain = "rpc", + correlation_id = %correlation_id, + route = route.as_str(), + live_request_count, + "Request queued on route-local RPC pending queue" + ); + + (None, None, false) + } + + fn forward_immediate_request( + &self, + envelope: &Envelope, + meta: &crate::runtime::ClientFrameMeta, + req: RpcRequest, + worker: &RpcWorkerDispatch, + live_request_count: usize, + ) -> DeliveryOutcome { + self.histogram_observe_us("rpc_pending_track_us", 0); + self.histogram_observe_us("rpc_pending_route_index_us", 0); + self.gauge_set("rpc_pending_requests", live_request_count as u64); + self.schedule_admin_snapshot(false); + + let metrics_enabled = self.metrics.is_some(); + let request_forward_start = metrics_enabled.then(Instant::now); + let forward_result = self.forward_request_to_worker(&req, worker); + if let Some(request_forward_start) = request_forward_start { + self.histogram_observe_elapsed_us("rpc_request_forward_us", request_forward_start); + } + + match forward_result { + Ok(()) => { + self.counter_inc("rpc_requests_dispatched_total"); + tracing::debug!( + domain = "rpc", + correlation_id = %req.correlation_id, + route = req.route.as_str(), + "Request forwarded to worker" + ); + (None, Some(false), false) + } + Err( + crate::runtime::RouteError::RouteNotFound(_) + | crate::runtime::RouteError::DeliveryFailed( + _, + DeliveryError::ActorStopped + | DeliveryError::Timeout + | DeliveryError::SinkPanicked + | DeliveryError::InvalidPayload { .. }, + ), + ) => self.handle_disconnected_worker_dispatch(envelope, meta, req, worker.session_id), + Err(crate::runtime::RouteError::DeliveryFailed( + _, + DeliveryError::MailboxFull { .. } | DeliveryError::HighLaneFull { .. }, + )) => self.handle_backpressured_worker_dispatch(envelope, meta, req), + } + } + + #[allow(clippy::needless_pass_by_value)] + fn handle_disconnected_worker_dispatch( + &self, + envelope: &Envelope, + meta: &crate::runtime::ClientFrameMeta, + req: RpcRequest, + worker_session_id: u64, + ) -> DeliveryOutcome { + self.counter_inc("rpc_request_forward_errors_total"); + let cleanup_result = self.apply_session_cleanup(worker_session_id); + let disconnect_deliveries = cleanup_result + .disconnect_deliveries + .into_iter() + .filter(|delivery| { + delivery.correlation_id != req.correlation_id + || *delivery.caller_inbox_addr.family() != meta.route_family + }) + .collect(); + self.forward_worker_disconnect_errors(disconnect_deliveries); + tracing::warn!( + domain = "rpc", + correlation_id = %req.correlation_id, + route = req.route.as_str(), + worker_session_id, + "Worker disconnected before request dispatch completed" + ); + self.reject_request_with_terminal_error( + envelope, + *meta, + &req, + crate::dispatch::protocol::error_codes::rpc::ERR_WORKER_NOT_FOUND, + RPC_WORKER_NOT_FOUND_ERROR, + ) + } + + #[allow(clippy::needless_pass_by_value)] + fn handle_backpressured_worker_dispatch( + &self, + envelope: &Envelope, + meta: &crate::runtime::ClientFrameMeta, + req: RpcRequest, + ) -> DeliveryOutcome { + self.counter_inc("rpc_request_forward_errors_total"); + // Inline dispatch backpressure counts toward the same aggregate as the + // deferred path; previously only the deferred path was visible. + self.counter_inc(RPC_BACKPRESSURE_REJECTS_METRIC); + let pending_len = self + .remove_pending_request_for_family(meta.route_family, &req.correlation_id) + .map(|(_, pending_len)| pending_len) + .unwrap_or_default(); + tracing::warn!( + domain = "rpc", + correlation_id = %req.correlation_id, + route = req.route.as_str(), + pending_len, + "Failed to forward request to worker due to backpressure" + ); + self.reject_request_with_terminal_error( + envelope, + *meta, + &req, + crate::dispatch::protocol::error_codes::rpc::ERR_RPC_BACKPRESSURE, + RPC_BACKPRESSURE_ERROR, + ) + } + + pub(super) fn reject_request_with_terminal_error( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + req: &RpcRequest, + code: u16, + message: &'static str, + ) -> DeliveryOutcome { + self.route_rpc_terminal_error_response(envelope, meta, req.correlation_id, code, message); + (None, None, true) + } + + pub(super) fn elapsed_micros_u64(start: Instant) -> u64 { + start.elapsed().as_micros().try_into().unwrap_or(u64::MAX) + } + + pub(super) fn forward_request_to_worker( + &self, + req: &crate::domains::rpc::protocol::RpcRequest, + worker: &RpcWorkerDispatch, + ) -> Result<(), crate::runtime::RouteError> { + #[cfg(test)] + let request_envelope = { + let mut payload_encoder = + crate::dispatch::protocol::payload_codec::PayloadEncoder::with_capacity(256); + let request_bytes = crate::dispatch::protocol::rpc_codec::encode_request_into( + req, + &mut payload_encoder, + ); + let request_ctx = FrameContext::new( + worker.session_id, + crate::dispatch::protocol::frame::ChannelId::Rpc, + crate::dispatch::protocol::tlv::MessageType::new(RPC_MSG_TYPE_REQUEST), + bytes::Bytes::from(request_bytes), + *worker.addr.family(), + ); + Envelope::new(worker.inbox_addr.clone(), request_ctx) + }; + + #[cfg(not(test))] + let request_envelope = Envelope::new( + worker.inbox_addr.clone(), + RpcWorkerRequestDelivery::new(worker.session_id, *worker.addr.family(), req.clone()), + ); + + self.router.route(request_envelope) + } + + pub(super) fn dispatch_queued_requests_for_family( + &self, + family: crate::runtime::routing::RouteFamily, + ) { + let mut snapshot_dirty = false; + loop { + let next_dispatch = self.state.lock().next_ready_dispatch_for_family(family); + let Some(dispatch) = next_dispatch else { + break; + }; + self.forward_queued_dispatch(&dispatch); + snapshot_dirty = true; + } + if snapshot_dirty { + self.schedule_admin_snapshot(false); + } + } + + pub(super) fn forward_queued_dispatch(&self, dispatch: &RpcQueuedDispatch) { + self.gauge_set("rpc_pending_requests", dispatch.live_request_count as u64); + + match self.forward_request_to_worker(&dispatch.request, &dispatch.registration) { + Ok(()) => { + self.counter_inc("rpc_requests_dispatched_total"); + } + Err( + crate::runtime::RouteError::RouteNotFound(_) + | crate::runtime::RouteError::DeliveryFailed( + _, + DeliveryError::ActorStopped + | DeliveryError::Timeout + | DeliveryError::SinkPanicked + | DeliveryError::InvalidPayload { .. }, + ), + ) => { + self.counter_inc("rpc_request_forward_errors_total"); + let cleanup_result = self.apply_session_cleanup(dispatch.registration.session_id); + self.forward_worker_disconnect_errors(cleanup_result.disconnect_deliveries); + } + Err(crate::runtime::RouteError::DeliveryFailed( + _, + DeliveryError::MailboxFull { .. } | DeliveryError::HighLaneFull { .. }, + )) => { + self.counter_inc("rpc_request_forward_errors_total"); + self.counter_inc(RPC_BACKPRESSURE_REJECTS_METRIC); + if let Some((pending, pending_len)) = self.remove_pending_request_for_family( + *dispatch.registration.addr.family(), + &dispatch.request.correlation_id, + ) { + if let Some(caller_inbox_addr) = pending.dispatch_info.caller_inbox_addr { + self.forward_pending_error_deliveries( + vec![RpcPendingErrorDelivery { + correlation_id: dispatch.request.correlation_id, + caller_session_id: pending.dispatch_info.caller_session_id, + caller_inbox_addr, + }], + crate::dispatch::protocol::error_codes::rpc::ERR_RPC_BACKPRESSURE, + RPC_BACKPRESSURE_ERROR, + "rpc_backpressure_errors_forwarded_total", + "rpc_backpressure_errors_dropped_total", + ); + } else { + self.counter_inc("rpc_backpressure_errors_dropped_total"); + self.counter_inc("rpc_responses_dropped_closed_caller_total"); + tracing::warn!( + domain = "rpc", + correlation_id = %dispatch.request.correlation_id, + "Dropped RPC backpressure error because caller session was already closed" + ); + } + self.gauge_set("rpc_pending_requests", pending_len as u64); + } + } + } + } + + pub(super) fn dispatch_all_queued_requests(&self) { + let mut families: Vec = { + let state = self.state.lock(); + state.routes.keys().map(|(family, _)| *family).collect() + }; + families.sort_by_key(crate::runtime::routing::RouteFamily::id); + families.dedup(); + for family in families { + self.dispatch_queued_requests_for_family(family); + } + } + + pub(super) fn remove_pending_request_for_family( + &self, + family: crate::runtime::routing::RouteFamily, + correlation_id: &uuid::Uuid, + ) -> Option<(RpcPendingRequest, usize)> { + let removed = { + let mut state = self.state.lock(); + state.remove_pending_request_for_family(family, correlation_id) + }; + + self.gauge_set( + "rpc_pending_requests", + removed.as_ref().map_or_else( + || self.pending_request_count() as u64, + |(_, pending_len)| *pending_len as u64, + ), + ); + if removed.is_some() { + self.release_global_pending(1); + } + removed + } +} diff --git a/src/domains/rpc/sink/domain_sink_impl.rs b/src/domains/rpc/sink/domain_sink_impl.rs deleted file mode 100644 index 3cffc974..00000000 --- a/src/domains/rpc/sink/domain_sink_impl.rs +++ /dev/null @@ -1,803 +0,0 @@ -use super::response_forwarder::RpcResponseForwarder; -use super::state_model::{ - rpc_admin_snapshot_due, rpc_timeout_sweep_interval, Arc, AtomicBool, DeliveryError, Duration, - Envelope, Instant, Ordering, Route, RouteAddress, RpcDomainActor, RpcDomainCommand, - RpcDomainCore, RpcDomainRuntime, RpcDomainSink, RpcLiveCounts, RpcPendingErrorDelivery, - RpcPendingRequest, RpcQueuedDispatch, RpcSessionCleanupResult, RpcWorkerCleanupResult, - RpcWorkerDispatch, RPC_BACKPRESSURE_ERROR, RPC_TIMEOUT_ERROR, RPC_WORKER_NOT_FOUND_ERROR, -}; -#[cfg(test)] -use super::state_model::{RpcQueuedRequest, RpcWorker, RPC_MSG_TYPE_REQUEST}; -#[cfg(test)] -use crate::dispatch::protocol::frame_context::FrameContext; -#[cfg(not(test))] -use crate::domains::rpc::RpcWorkerRequestDelivery; -use crate::runtime::routing::RouteFamily; - -impl RpcDomainActor { - pub(super) fn new(core: Arc, active: Arc) -> Self { - Self { core, active } - } - - pub(super) fn route_address() -> RouteAddress { - RouteAddress::new(RouteFamily::new(0), Route::new("internal://domain/rpc")) - } - - pub(super) fn runtime(&self) -> RpcDomainRuntime<'_> { - RpcDomainRuntime { - core: &self.core, - active: &self.active, - } - } -} - -impl RpcDomainSink { - pub(super) fn runtime(&self) -> RpcDomainRuntime<'_> { - RpcDomainRuntime { - core: &self.core, - active: &self.active, - } - } - - fn control_targets(&self) -> Vec> { - self.family_families.as_ref().map_or_else( - || vec![None], - |families| families.iter().copied().map(Some).collect(), - ) - } - - fn primary_control_target(&self) -> Option { - self.family_families - .as_ref() - .and_then(|families| families.first().copied()) - } - - fn try_send_control( - &self, - family: Option, - command: RpcDomainCommand, - ) -> Result<(), String> { - if let Some(runtime) = self.family_runtime.as_ref() { - let family = family.ok_or_else(|| "RPC family target is missing".to_string())?; - runtime - .try_enqueue(family, crate::runtime::FamilyActorLane::Control, command) - .map_err(|error| error.to_string()) - } else { - self.actor - .try_send_high_priority(command) - .map_err(|error| error.to_string()) - } - } - - pub fn stop(&self) { - self.active.store(false, Ordering::Relaxed); - if let Some(runtime) = self.family_runtime.as_ref() { - runtime.stop(); - } - self.actor.stop(); - } - - pub(crate) fn is_active(&self) -> bool { - self.active.load(Ordering::Relaxed) - } - - pub(crate) fn timeout_sweep_interval(&self) -> Duration { - self.runtime().timeout_sweep_interval() - } - - pub(crate) fn expire_timed_out_requests(&self) { - for family in self.control_targets() { - if let Err(error) = self.try_send_control( - family, - RpcDomainCommand::ExpireTimedOutRequestsAt(Instant::now(), None), - ) { - tracing::warn!( - domain = "rpc", - family = family.map(|target| target.id()), - error = %error, - "RPC timeout sweep enqueue failed" - ); - } - } - } - - #[cfg(test)] - pub(super) fn expire_timed_out_requests_at(&self, now: Instant) { - for family in self.control_targets() { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = self.try_send_control( - family, - RpcDomainCommand::ExpireTimedOutRequestsAt(now, Some(reply_tx)), - ) { - tracing::warn!( - domain = "rpc", - family = family.map(|target| target.id()), - error = %error, - "RPC timeout sweep enqueue failed" - ); - continue; - } - let _ = reply_rx.recv_timeout(Duration::from_secs(1)); - } - } - - #[cfg(test)] - pub(super) fn is_actor_running(&self) -> bool { - self.actor.is_running() - && self - .family_runtime - .as_ref() - .is_none_or(crate::runtime::FamilyActorPoolRuntime::is_running) - } - - pub(crate) fn actor_health_snapshot(&self) -> crate::runtime::ManagedActorHealthSnapshot { - self.family_runtime.as_ref().map_or_else( - || self.actor.health_snapshot(), - crate::runtime::FamilyActorPoolRuntime::managed_actor_health_snapshot, - ) - } - - #[cfg(test)] - pub(crate) fn panic_actor_for_tests(&self) { - let _ = self.try_send_control( - self.primary_control_target(), - RpcDomainCommand::PanicForTests, - ); - } - - #[cfg(test)] - pub(super) fn stop_actor_for_tests(&self) { - self.actor.stop(); - } - - #[cfg(test)] - pub(super) fn register_registration_for_tests(&self, registration: RpcWorker) { - self.core.state.lock().register_registration(registration); - } - - #[cfg(test)] - pub(super) fn track_pending_request_for_tests( - &self, - correlation_id: uuid::Uuid, - pending: RpcPendingRequest, - ) { - self.core.state.lock().pending.track_pending_for_family( - pending.dispatch_info.family, - correlation_id, - pending, - ); - } - - #[cfg(test)] - pub(super) fn queue_request_for_tests( - &self, - correlation_id: uuid::Uuid, - queued: RpcQueuedRequest, - ) { - self.core.state.lock().queue_request(correlation_id, queued); - } - - #[cfg(test)] - pub(super) fn live_request_count_for_tests(&self) -> usize { - self.core.state.lock().live_request_count() - } - - #[cfg(test)] - pub(super) fn pending_table_len_for_tests(&self) -> usize { - self.core.state.lock().pending.len() - } - - #[cfg(test)] - pub(super) fn queued_request_count_for_tests(&self) -> usize { - self.core.state.lock().queued.len() - } - - #[cfg(test)] - pub(super) fn route_queued_len_for_tests(&self, route: &Route) -> usize { - let mut state = self.core.state.lock(); - state - .route_state(route) - .map_or(0, |route_state| route_state.queued_len()) - } - - pub fn worker_count(&self) -> usize { - self.live_counts().workers - } - - pub fn pending_request_count(&self) -> usize { - self.live_counts().pending_requests - } - - fn live_counts(&self) -> RpcLiveCounts { - let mut total = RpcLiveCounts::default(); - for family in self.control_targets() { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = - self.try_send_control(family, RpcDomainCommand::ReadLiveCounts(reply_tx)) - { - tracing::warn!( - domain = "rpc", - family = family.map(|target| target.id()), - error = %error, - "RPC live-count query enqueue failed" - ); - continue; - } - if let Ok(counts) = reply_rx.recv_timeout(Duration::from_secs(1)) { - total.workers = total.workers.saturating_add(counts.workers); - total.pending_requests = total - .pending_requests - .saturating_add(counts.pending_requests); - } - } - total - } - - #[cfg(test)] - pub(super) fn sync_admin_snapshot(&self) { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = self.try_send_control( - self.primary_control_target(), - RpcDomainCommand::SyncAdminSnapshot(Some(reply_tx)), - ) { - tracing::warn!(domain = "rpc", error = %error, "RPC admin snapshot enqueue failed"); - return; - } - - let _ = reply_rx.recv_timeout(Duration::from_secs(1)); - } - - pub fn refresh_admin_snapshot_if_dirty(&self) { - if let Err(error) = self.try_send_control( - self.primary_control_target(), - RpcDomainCommand::RefreshAdminSnapshotIfDirty(None), - ) { - tracing::warn!(domain = "rpc", error = %error, "RPC admin snapshot refresh enqueue failed"); - } - } - - #[cfg(test)] - pub(super) fn apply_session_cleanup(&self, session_id: u64) -> RpcSessionCleanupResult { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = - self.actor - .try_send_high_priority(RpcDomainCommand::ApplySessionCleanupForTests( - session_id, reply_tx, - )) - { - tracing::warn!(domain = "rpc", error = %error, "RPC session cleanup enqueue failed"); - return RpcSessionCleanupResult::default(); - } - - reply_rx - .recv_timeout(Duration::from_secs(1)) - .unwrap_or_default() - } - - #[cfg(test)] - pub(super) fn apply_worker_unsubscribe( - &self, - worker_addr: &RouteAddress, - session_id: u64, - ) -> RpcWorkerCleanupResult { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = - self.actor - .try_send_high_priority(RpcDomainCommand::ApplyWorkerUnsubscribeForTests( - worker_addr.clone(), - session_id, - reply_tx, - )) - { - tracing::warn!(domain = "rpc", error = %error, "RPC worker unsubscribe enqueue failed"); - return RpcWorkerCleanupResult::default(); - } - - reply_rx - .recv_timeout(Duration::from_secs(1)) - .unwrap_or_default() - } -} - -impl RpcDomainRuntime<'_> { - fn u64_to_usize_saturating(value: u64) -> usize { - usize::try_from(value).unwrap_or(usize::MAX) - } - - fn elapsed_us_saturating(start: Instant) -> u64 { - start.elapsed().as_micros().try_into().unwrap_or(u64::MAX) - } - - pub(super) fn release_global_pending(&self, count: usize) { - if count == 0 { - return; - } - let _ = self.global_pending_count.fetch_update( - Ordering::AcqRel, - Ordering::Acquire, - |current| Some(current.saturating_sub(count)), - ); - } - - pub(crate) fn timeout_sweep_interval(&self) -> Duration { - rpc_timeout_sweep_interval(self.request_timeout) - } - - pub(super) fn live_counts(&self) -> RpcLiveCounts { - let state = self.state.lock(); - let workers = state.registration_count(); - RpcLiveCounts { - workers, - pending_requests: state.live_request_count(), - } - } - - pub(super) fn counter_inc(&self, name: &str) { - if let Some(ref metrics) = self.metrics { - metrics.counter_inc(name); - } - } - - pub(super) fn counter_add(&self, name: &str, amount: u64) { - if let Some(ref metrics) = self.metrics { - metrics.counter_add(name, amount); - } - } - - pub(super) fn gauge_set(&self, name: &str, value: u64) { - if let Some(ref metrics) = self.metrics { - metrics.gauge_set(name, value); - if name == "rpc_pending_requests" { - metrics.set_pending_request_count(Self::u64_to_usize_saturating(value)); - } - } - } - - pub(super) fn histogram_observe_us(&self, name: &str, value_us: u64) { - if let Some(ref metrics) = self.metrics { - metrics.histogram_observe_us(name, value_us); - } - } - - pub(super) fn histogram_observe_elapsed_us(&self, name: &str, start: Instant) { - self.histogram_observe_us(name, Self::elapsed_us_saturating(start)); - } - - pub(super) fn refresh_metrics_gauges(&self) { - if let Some(metrics) = &self.metrics { - let counts = self.core.aggregate_live_counts(); - metrics.set_worker_count(counts.workers); - metrics.set_pending_request_count(counts.pending_requests); - } - } - - pub(super) fn expire_timed_out_requests_inline_if_due(&self) { - let now_elapsed_us = Self::elapsed_us_saturating(self.snapshot_epoch); - let interval_us = self - .timeout_sweep_interval() - .as_micros() - .try_into() - .unwrap_or(u64::MAX); - let last_elapsed_us = self.last_inline_timeout_elapsed_us.load(Ordering::Relaxed); - - if now_elapsed_us.saturating_sub(last_elapsed_us) < interval_us { - return; - } - - if self - .last_inline_timeout_elapsed_us - .compare_exchange( - last_elapsed_us, - now_elapsed_us, - Ordering::AcqRel, - Ordering::Relaxed, - ) - .is_ok() - { - self.expire_timed_out_requests_at(Instant::now()); - } - } - - pub(super) fn expire_timed_out_requests_at(&self, now: Instant) { - let timeout_result = { - let mut state = self.state.lock(); - state.expire_timed_out(now) - }; - - if timeout_result.removed_pending == 0 { - return; - } - - self.release_global_pending(timeout_result.removed_pending); - let timeout_delivery_count = timeout_result.timeout_deliveries.len(); - self.gauge_set("rpc_pending_requests", timeout_result.pending_len as u64); - self.counter_add( - "rpc_request_timeouts_total", - timeout_result.removed_pending as u64, - ); - self.counter_add( - "rpc_cleanup_pending_removed_total", - timeout_result.removed_pending as u64, - ); - if timeout_result.closed_caller_drops > 0 { - self.counter_add( - "rpc_timeout_errors_dropped_total", - timeout_result.closed_caller_drops as u64, - ); - self.counter_add( - "rpc_responses_dropped_closed_caller_total", - timeout_result.closed_caller_drops as u64, - ); - } - self.schedule_admin_snapshot(false); - self.dispatch_all_queued_requests(); - - tracing::debug!( - domain = "rpc", - removed_pending = timeout_result.removed_pending, - delivered_timeouts = timeout_delivery_count, - closed_caller_drops = timeout_result.closed_caller_drops, - pending_len = timeout_result.pending_len, - "RPC request timeout sweep applied" - ); - - self.forward_pending_error_deliveries( - timeout_result.timeout_deliveries, - crate::dispatch::protocol::error_codes::rpc::ERR_RPC_TIMEOUT, - RPC_TIMEOUT_ERROR, - "rpc_timeout_errors_forwarded_total", - "rpc_timeout_errors_dropped_total", - ); - } - - pub(super) fn remove_pending_request_for_family( - &self, - family: crate::runtime::routing::RouteFamily, - correlation_id: &uuid::Uuid, - ) -> Option<(RpcPendingRequest, usize)> { - let removed = { - let mut state = self.state.lock(); - state.remove_pending_request_for_family(family, correlation_id) - }; - - self.gauge_set( - "rpc_pending_requests", - removed.as_ref().map_or_else( - || self.pending_request_count() as u64, - |(_, pending_len)| *pending_len as u64, - ), - ); - if removed.is_some() { - self.release_global_pending(1); - } - removed - } - - pub(super) fn apply_session_cleanup(&self, session_id: u64) -> RpcSessionCleanupResult { - let cleanup_result = { - let mut state = self.state.lock(); - state.cleanup_session(session_id) - }; - - self.gauge_set("rpc_pending_requests", cleanup_result.pending_len as u64); - self.release_global_pending(cleanup_result.removed_pending); - if cleanup_result.removed_registrations > 0 { - self.counter_add( - "rpc_cleanup_workers_removed_total", - cleanup_result.removed_registrations as u64, - ); - } - if cleanup_result.detached_callers > 0 { - self.counter_add( - "rpc_cleanup_callers_detached_total", - cleanup_result.detached_callers as u64, - ); - } - if cleanup_result.removed_pending > 0 { - self.counter_add( - "rpc_cleanup_pending_removed_total", - cleanup_result.removed_pending as u64, - ); - } - if cleanup_result.removed_registrations > 0 - || cleanup_result.detached_callers > 0 - || cleanup_result.removed_pending > 0 - { - self.schedule_admin_snapshot(false); - } - self.refresh_metrics_gauges(); - - tracing::debug!( - domain = "rpc", - session_id, - removed_workers = cleanup_result.removed_registrations, - detached_callers = cleanup_result.detached_callers, - removed_pending = cleanup_result.removed_pending, - pending_len = cleanup_result.pending_len, - "RPC session cleanup applied" - ); - - cleanup_result - } - - pub(super) fn apply_worker_unsubscribe( - &self, - worker_addr: &RouteAddress, - session_id: u64, - ) -> RpcWorkerCleanupResult { - let cleanup_result = { - let mut state = self.state.lock(); - state.unregister_registration(worker_addr, session_id) - }; - - self.gauge_set("rpc_pending_requests", cleanup_result.pending_len as u64); - self.release_global_pending(cleanup_result.removed_pending); - if cleanup_result.removed_registrations > 0 { - self.counter_add( - "rpc_cleanup_workers_removed_total", - cleanup_result.removed_registrations as u64, - ); - } - if cleanup_result.removed_pending > 0 { - self.counter_add( - "rpc_cleanup_pending_removed_total", - cleanup_result.removed_pending as u64, - ); - } - if cleanup_result.removed_registrations > 0 || cleanup_result.removed_pending > 0 { - self.schedule_admin_snapshot(false); - } - self.refresh_metrics_gauges(); - - tracing::debug!( - domain = "rpc", - worker = worker_addr.route().as_str(), - session_id, - removed_workers = cleanup_result.removed_registrations, - removed_pending = cleanup_result.removed_pending, - pending_len = cleanup_result.pending_len, - "RPC worker cleanup applied" - ); - - cleanup_result - } - - pub(super) fn forward_pending_error_deliveries( - &self, - error_deliveries: Vec, - error_code: u16, - error_message: &'static str, - forwarded_counter: &str, - dropped_counter: &str, - ) { - if error_deliveries.is_empty() { - return; - } - - for delivery in error_deliveries { - let correlation_id = delivery.correlation_id; - let response_envelope = - RpcResponseForwarder::terminal_error_envelope(delivery, error_code, error_message); - - if let Err(error) = self.router.route(response_envelope) { - self.counter_inc(dropped_counter); - tracing::warn!( - domain = "rpc", - correlation_id = %correlation_id, - error_code, - error = ?error, - "Failed to forward RPC terminal error to requester" - ); - } else { - self.counter_inc(forwarded_counter); - } - } - } - - pub(super) fn forward_worker_disconnect_errors( - &self, - disconnect_deliveries: Vec, - ) { - self.forward_pending_error_deliveries( - disconnect_deliveries, - crate::dispatch::protocol::error_codes::rpc::ERR_WORKER_NOT_FOUND, - RPC_WORKER_NOT_FOUND_ERROR, - "rpc_worker_disconnect_errors_forwarded_total", - "rpc_worker_disconnect_errors_dropped_total", - ); - } - - pub(super) fn pending_request_count(&self) -> usize { - self.live_counts().pending_requests - } - - pub(super) fn forward_request_to_worker( - &self, - req: &crate::domains::rpc::protocol::RpcRequest, - worker: &RpcWorkerDispatch, - ) -> Result<(), crate::runtime::RouteError> { - #[cfg(test)] - let request_envelope = { - let mut payload_encoder = - crate::dispatch::protocol::payload_codec::PayloadEncoder::with_capacity(256); - let request_bytes = crate::dispatch::protocol::rpc_codec::encode_request_into( - req, - &mut payload_encoder, - ); - let request_ctx = FrameContext::new( - worker.session_id, - crate::dispatch::protocol::frame::ChannelId::Rpc, - crate::dispatch::protocol::tlv::MessageType::new(RPC_MSG_TYPE_REQUEST), - bytes::Bytes::from(request_bytes), - *worker.addr.family(), - ); - Envelope::new(worker.inbox_addr.clone(), request_ctx) - }; - - #[cfg(not(test))] - let request_envelope = Envelope::new( - worker.inbox_addr.clone(), - RpcWorkerRequestDelivery::new(worker.session_id, *worker.addr.family(), req.clone()), - ); - - self.router.route(request_envelope) - } - - pub(super) fn dispatch_queued_requests_for_family( - &self, - family: crate::runtime::routing::RouteFamily, - ) { - let mut snapshot_dirty = false; - loop { - let next_dispatch = self.state.lock().next_ready_dispatch_for_family(family); - let Some(dispatch) = next_dispatch else { - break; - }; - self.forward_queued_dispatch(&dispatch); - snapshot_dirty = true; - } - if snapshot_dirty { - self.schedule_admin_snapshot(false); - } - } - - pub(super) fn forward_queued_dispatch(&self, dispatch: &RpcQueuedDispatch) { - self.gauge_set("rpc_pending_requests", dispatch.live_request_count as u64); - - match self.forward_request_to_worker(&dispatch.request, &dispatch.registration) { - Ok(()) => { - self.counter_inc("rpc_requests_dispatched_total"); - } - Err( - crate::runtime::RouteError::RouteNotFound(_) - | crate::runtime::RouteError::DeliveryFailed( - _, - DeliveryError::ActorStopped - | DeliveryError::Timeout - | DeliveryError::SinkPanicked, - ), - ) => { - self.counter_inc("rpc_request_forward_errors_total"); - let cleanup_result = self.apply_session_cleanup(dispatch.registration.session_id); - self.forward_worker_disconnect_errors(cleanup_result.disconnect_deliveries); - } - Err(crate::runtime::RouteError::DeliveryFailed( - _, - DeliveryError::MailboxFull { .. } | DeliveryError::HighLaneFull { .. }, - )) => { - self.counter_inc("rpc_request_forward_errors_total"); - self.counter_inc("rpc_backpressure_rejects_total"); - if let Some((pending, pending_len)) = self.remove_pending_request_for_family( - *dispatch.registration.addr.family(), - &dispatch.request.correlation_id, - ) { - if let Some(caller_inbox_addr) = pending.dispatch_info.caller_inbox_addr { - self.forward_pending_error_deliveries( - vec![RpcPendingErrorDelivery { - correlation_id: dispatch.request.correlation_id, - caller_session_id: pending.dispatch_info.caller_session_id, - caller_inbox_addr, - }], - crate::dispatch::protocol::error_codes::rpc::ERR_RPC_BACKPRESSURE, - RPC_BACKPRESSURE_ERROR, - "rpc_backpressure_errors_forwarded_total", - "rpc_backpressure_errors_dropped_total", - ); - } else { - self.counter_inc("rpc_backpressure_errors_dropped_total"); - self.counter_inc("rpc_responses_dropped_closed_caller_total"); - tracing::warn!( - domain = "rpc", - correlation_id = %dispatch.request.correlation_id, - "Dropped RPC backpressure error because caller session was already closed" - ); - } - self.gauge_set("rpc_pending_requests", pending_len as u64); - } - } - } - } - - pub(super) fn dispatch_all_queued_requests(&self) { - let mut families: Vec = { - let state = self.state.lock(); - state.routes.keys().map(|(family, _)| *family).collect() - }; - families.sort_by_key(crate::runtime::routing::RouteFamily::id); - families.dedup(); - for family in families { - self.dispatch_queued_requests_for_family(family); - } - } - - pub(super) fn refresh_admin_snapshot_if_dirty(&self) { - self.maybe_sync_admin_snapshot(false); - } - - /// Mark the admin snapshot dirty. Forced calls refresh immediately; regular - /// hot-path updates coalesce until an admin read or another forced refresh. - pub(super) fn schedule_admin_snapshot(&self, force: bool) { - if force { - self.snapshot_dirty.store(true, Ordering::Relaxed); - self.maybe_sync_admin_snapshot(true); - return; - } - - if self - .snapshot_dirty - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) - .is_err() - { - return; - } - - self.maybe_sync_admin_snapshot(false); - } - - /// Sync the admin snapshot when the snapshot interval elapses or a caller forces it. - /// - /// Even forced snapshots are still point-in-time copies of the sink's current - /// in-memory state, not linearizable reads of concurrent RPC activity. - pub(super) fn maybe_sync_admin_snapshot(&self, force: bool) { - #[cfg(feature = "bench-no-snapshot")] - if !force { - return; - } - - let now_elapsed_us = Self::elapsed_us_saturating(self.snapshot_epoch); - let last_snapshot_elapsed_us = self.last_snapshot_elapsed_us.load(Ordering::Relaxed); - let snapshot_dirty = self.snapshot_dirty.load(Ordering::Relaxed); - - if !rpc_admin_snapshot_due( - snapshot_dirty, - force, - now_elapsed_us, - last_snapshot_elapsed_us, - ) { - return; - } - - if self - .snapshot_syncing - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) - .is_err() - { - return; - } - - if !self.snapshot_dirty.swap(false, Ordering::AcqRel) { - self.snapshot_syncing.store(false, Ordering::Release); - return; - } - - let snapshot_start = Instant::now(); - self.sync_admin_snapshot(); - let snapshot_time_us = Self::elapsed_us_saturating(snapshot_start); - self.last_snapshot_elapsed_us.store( - Self::elapsed_us_saturating(self.snapshot_epoch), - Ordering::Relaxed, - ); - self.snapshot_syncing.store(false, Ordering::Release); - self.histogram_observe_us("rpc_admin_snapshot_us", snapshot_time_us); - } -} diff --git a/src/domains/rpc/sink/facade.rs b/src/domains/rpc/sink/facade.rs new file mode 100644 index 00000000..419f1c8a --- /dev/null +++ b/src/domains/rpc/sink/facade.rs @@ -0,0 +1,300 @@ +//! Public `RpcDomainSink` API and actor identity/lifecycle queries. + +use super::state_model::{ + Arc, AtomicBool, Duration, Instant, Ordering, Route, RouteAddress, RpcDomainActor, + RpcDomainCommand, RpcDomainCore, RpcDomainRuntime, RpcDomainSink, RpcLiveCounts, +}; +#[cfg(test)] +use super::state_model::{ + RpcPendingRequest, RpcQueuedRequest, RpcSessionCleanupResult, RpcWorker, RpcWorkerCleanupResult, +}; +use crate::runtime::routing::RouteFamily; + +impl RpcDomainActor { + pub(super) fn new(core: Arc, active: Arc) -> Self { + Self { core, active } + } + + pub(super) fn route_address() -> RouteAddress { + RouteAddress::new(RouteFamily::new(0), Route::new("internal://domain/rpc")) + } + + pub(super) fn runtime(&self) -> RpcDomainRuntime<'_> { + RpcDomainRuntime { + core: &self.core, + active: &self.active, + } + } +} + +impl RpcDomainSink { + pub(super) fn runtime(&self) -> RpcDomainRuntime<'_> { + RpcDomainRuntime { + core: &self.core, + active: &self.active, + } + } + + fn control_targets(&self) -> Vec> { + self.family_families.as_ref().map_or_else( + || vec![None], + |families| families.iter().copied().map(Some).collect(), + ) + } + + fn primary_control_target(&self) -> Option { + self.family_families + .as_ref() + .and_then(|families| families.first().copied()) + } + + fn try_send_control( + &self, + family: Option, + command: RpcDomainCommand, + ) -> Result<(), String> { + if let Some(runtime) = self.family_runtime.as_ref() { + let family = family.ok_or_else(|| "RPC family target is missing".to_string())?; + runtime + .try_enqueue(family, crate::runtime::FamilyActorLane::Control, command) + .map_err(|error| error.to_string()) + } else { + self.actor + .try_send_high_priority(command) + .map_err(|error| error.to_string()) + } + } + + pub fn stop(&self) { + self.active.store(false, Ordering::Relaxed); + if let Some(runtime) = self.family_runtime.as_ref() { + runtime.stop(); + } + self.actor.stop(); + } + + pub(crate) fn is_active(&self) -> bool { + self.active.load(Ordering::Relaxed) + } + + pub(crate) fn timeout_sweep_interval(&self) -> Duration { + self.runtime().timeout_sweep_interval() + } + + pub(crate) fn expire_timed_out_requests(&self) { + for family in self.control_targets() { + if let Err(error) = self.try_send_control( + family, + RpcDomainCommand::ExpireTimedOutRequestsAt(Instant::now(), None), + ) { + tracing::warn!( + domain = "rpc", + family = family.map(|target| target.id()), + error = %error, + "RPC timeout sweep enqueue failed" + ); + } + } + } + + #[cfg(test)] + pub(super) fn expire_timed_out_requests_at(&self, now: Instant) { + for family in self.control_targets() { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = self.try_send_control( + family, + RpcDomainCommand::ExpireTimedOutRequestsAt(now, Some(reply_tx)), + ) { + tracing::warn!( + domain = "rpc", + family = family.map(|target| target.id()), + error = %error, + "RPC timeout sweep enqueue failed" + ); + continue; + } + let _ = reply_rx.recv_timeout(Duration::from_secs(1)); + } + } + + #[cfg(test)] + pub(super) fn is_actor_running(&self) -> bool { + self.actor.is_running() + && self + .family_runtime + .as_ref() + .is_none_or(crate::runtime::FamilyActorPoolRuntime::is_running) + } + + pub(crate) fn actor_health_snapshot(&self) -> crate::runtime::ManagedActorHealthSnapshot { + self.family_runtime.as_ref().map_or_else( + || self.actor.health_snapshot(), + crate::runtime::FamilyActorPoolRuntime::managed_actor_health_snapshot, + ) + } + + /// Panic every provisioned family's handler (or the single actor in + /// non-sharded mode). Used by `panic_all_domain_actors_for_tests` to + /// drive the pool to full exhaustion; a single family's panic must never + /// be conflated with domain-wide health, so covering every family here + /// is required to actually observe pool-wide fail-closed behavior. + #[cfg(test)] + pub(crate) fn panic_actor_for_tests(&self) { + for family in self.control_targets() { + let _ = self.try_send_control(family, RpcDomainCommand::PanicForTests); + } + } + + #[cfg(test)] + pub(super) fn stop_actor_for_tests(&self) { + self.actor.stop(); + } + + #[cfg(test)] + pub(super) fn register_registration_for_tests(&self, registration: RpcWorker) { + self.core.state.lock().register_registration(registration); + } + + #[cfg(test)] + pub(super) fn track_pending_request_for_tests( + &self, + correlation_id: uuid::Uuid, + pending: RpcPendingRequest, + ) { + self.core.state.lock().pending.track_pending_for_family( + pending.dispatch_info.family, + correlation_id, + pending, + ); + } + + #[cfg(test)] + pub(super) fn queue_request_for_tests( + &self, + correlation_id: uuid::Uuid, + queued: RpcQueuedRequest, + ) { + self.core.state.lock().queue_request(correlation_id, queued); + } + + #[cfg(test)] + pub(super) fn live_request_count_for_tests(&self) -> usize { + self.core.state.lock().live_request_count() + } + + #[cfg(test)] + pub(super) fn pending_table_len_for_tests(&self) -> usize { + self.core.state.lock().pending.len() + } + + #[cfg(test)] + pub(super) fn queued_request_count_for_tests(&self) -> usize { + self.core.state.lock().queued.len() + } + + #[cfg(test)] + pub(super) fn route_queued_len_for_tests(&self, route: &Route) -> usize { + let mut state = self.core.state.lock(); + state + .route_state(route) + .map_or(0, |route_state| route_state.queued_len()) + } + + pub fn worker_count(&self) -> usize { + self.live_counts().workers + } + + pub fn pending_request_count(&self) -> usize { + self.live_counts().pending_requests + } + + fn live_counts(&self) -> RpcLiveCounts { + let mut total = RpcLiveCounts::default(); + for family in self.control_targets() { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = + self.try_send_control(family, RpcDomainCommand::ReadLiveCounts(reply_tx)) + { + tracing::warn!( + domain = "rpc", + family = family.map(|target| target.id()), + error = %error, + "RPC live-count query enqueue failed" + ); + continue; + } + if let Ok(counts) = reply_rx.recv_timeout(Duration::from_secs(1)) { + total.workers = total.workers.saturating_add(counts.workers); + total.pending_requests = total + .pending_requests + .saturating_add(counts.pending_requests); + } + } + total + } + + #[cfg(test)] + pub(super) fn sync_admin_snapshot(&self) { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = self.try_send_control( + self.primary_control_target(), + RpcDomainCommand::SyncAdminSnapshot(Some(reply_tx)), + ) { + tracing::warn!(domain = "rpc", error = %error, "RPC admin snapshot enqueue failed"); + return; + } + + let _ = reply_rx.recv_timeout(Duration::from_secs(1)); + } + + pub fn refresh_admin_snapshot_if_dirty(&self) { + if let Err(error) = self.try_send_control( + self.primary_control_target(), + RpcDomainCommand::RefreshAdminSnapshotIfDirty(None), + ) { + tracing::warn!(domain = "rpc", error = %error, "RPC admin snapshot refresh enqueue failed"); + } + } + + #[cfg(test)] + pub(super) fn apply_session_cleanup(&self, session_id: u64) -> RpcSessionCleanupResult { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = + self.actor + .try_send_high_priority(RpcDomainCommand::ApplySessionCleanupForTests( + session_id, reply_tx, + )) + { + tracing::warn!(domain = "rpc", error = %error, "RPC session cleanup enqueue failed"); + return RpcSessionCleanupResult::default(); + } + + reply_rx + .recv_timeout(Duration::from_secs(1)) + .unwrap_or_default() + } + + #[cfg(test)] + pub(super) fn apply_worker_unsubscribe( + &self, + worker_addr: &RouteAddress, + session_id: u64, + ) -> RpcWorkerCleanupResult { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = + self.actor + .try_send_high_priority(RpcDomainCommand::ApplyWorkerUnsubscribeForTests( + worker_addr.clone(), + session_id, + reply_tx, + )) + { + tracing::warn!(domain = "rpc", error = %error, "RPC worker unsubscribe enqueue failed"); + return RpcWorkerCleanupResult::default(); + } + + reply_rx + .recv_timeout(Duration::from_secs(1)) + .unwrap_or_default() + } +} diff --git a/src/domains/rpc/sink/family_runtime.rs b/src/domains/rpc/sink/family_runtime.rs index 59ac3d8f..658dbec1 100644 --- a/src/domains/rpc/sink/family_runtime.rs +++ b/src/domains/rpc/sink/family_runtime.rs @@ -32,6 +32,9 @@ impl RpcDomainSink { ) -> Self { let core = Arc::new(RpcDomainCore { state: Mutex::new(RpcState::new()), + cleaned_up_sessions: Mutex::new(crate::runtime::CleanedUpSessions::new( + crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, + )), router, admin_read_model, request_timeout: RPC_DEFAULT_REQUEST_TIMEOUT, @@ -83,71 +86,77 @@ impl RpcDomainSink { let pool = crate::runtime::FamilyActorPool::new(families) .map_err(|error| format!("create RPC family actor pool: {error}"))?; let core_for_factory = core.clone(); - Ok(crate::runtime::FamilyActorPoolRuntime::spawn( - pool, - active.clone(), - move |family| Self::family_core_for(&core_for_factory, family), - move |core, family, _lane, command| { - let runtime = RpcDomainRuntime { - core, - active: active.as_ref(), - }; - match command { - RpcDomainCommand::Deliver(envelope, reply) => { - let result = if *envelope.destination().family() == family { - runtime.deliver_envelope(&envelope) - } else { - Err(DeliveryError::ActorStopped) - }; - let _ = reply.send(result); - } - RpcDomainCommand::ExpireTimedOutRequestsAt(now, reply) => { - runtime.expire_timed_out_requests_at(now); - if let Some(reply) = reply { - let _ = reply.send(()); + Ok( + crate::runtime::FamilyActorPoolRuntime::spawn_with_family_failed_metric( + pool, + active.clone(), + move |family| Self::family_core_for(&core_for_factory, family), + move |core, family, _lane, command| { + let runtime = RpcDomainRuntime { + core, + active: active.as_ref(), + }; + match command { + RpcDomainCommand::Deliver(envelope, reply) => { + let result = if *envelope.destination().family() == family { + runtime.deliver_envelope(&envelope) + } else { + Err(DeliveryError::ActorStopped) + }; + let _ = reply.send(result); } - } - RpcDomainCommand::ReadLiveCounts(reply) => { - let _ = reply.send(runtime.live_counts()); - } - #[cfg(test)] - RpcDomainCommand::SyncAdminSnapshot(reply) => { - runtime.sync_admin_snapshot(); - if let Some(reply) = reply { - let _ = reply.send(()); + RpcDomainCommand::ExpireTimedOutRequestsAt(now, reply) => { + runtime.expire_timed_out_requests_at(now); + if let Some(reply) = reply { + let _ = reply.send(()); + } } - } - RpcDomainCommand::RefreshAdminSnapshotIfDirty(reply) => { - runtime.refresh_admin_snapshot_if_dirty(); - if let Some(reply) = reply { - let _ = reply.send(()); + RpcDomainCommand::ReadLiveCounts(reply) => { + let _ = reply.send(runtime.live_counts()); + } + #[cfg(test)] + RpcDomainCommand::SyncAdminSnapshot(reply) => { + runtime.sync_admin_snapshot(); + if let Some(reply) = reply { + let _ = reply.send(()); + } + } + RpcDomainCommand::RefreshAdminSnapshotIfDirty(reply) => { + runtime.refresh_admin_snapshot_if_dirty(); + if let Some(reply) = reply { + let _ = reply.send(()); + } + } + #[cfg(test)] + RpcDomainCommand::ApplySessionCleanupForTests(session_id, reply) => { + let _ = reply.send(runtime.apply_session_cleanup(session_id)); + } + #[cfg(test)] + RpcDomainCommand::ApplyWorkerUnsubscribeForTests( + worker_addr, + session_id, + reply, + ) => { + let _ = reply + .send(runtime.apply_worker_unsubscribe(&worker_addr, session_id)); + } + #[cfg(test)] + RpcDomainCommand::PanicForTests => { + panic!("test RPC family actor panic"); } } - #[cfg(test)] - RpcDomainCommand::ApplySessionCleanupForTests(session_id, reply) => { - let _ = reply.send(runtime.apply_session_cleanup(session_id)); - } - #[cfg(test)] - RpcDomainCommand::ApplyWorkerUnsubscribeForTests( - worker_addr, - session_id, - reply, - ) => { - let _ = - reply.send(runtime.apply_worker_unsubscribe(&worker_addr, session_id)); - } - #[cfg(test)] - RpcDomainCommand::PanicForTests => { - panic!("test RPC family actor panic"); - } - } - }, - )) + }, + crate::domains::rpc::metrics::METRIC_FAMILY_FAILED_CLOSED_TOTAL, + ), + ) } fn family_core_for(shared: &Arc, family: RouteFamily) -> Arc { let family_core = Arc::new(RpcDomainCore { state: Mutex::new(RpcState::new()), + cleaned_up_sessions: Mutex::new(crate::runtime::CleanedUpSessions::new( + crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, + )), router: shared.router.clone(), admin_read_model: shared.admin_read_model.clone(), request_timeout: shared.request_timeout, diff --git a/src/domains/rpc/sink/ingress.rs b/src/domains/rpc/sink/ingress.rs new file mode 100644 index 00000000..9c7cae77 --- /dev/null +++ b/src/domains/rpc/sink/ingress.rs @@ -0,0 +1,260 @@ +//! Envelope ingress: validate an inbound envelope, parse it into an RPC +//! message, and dispatch to the registration/delivery/response layers. + +use super::state_model::{ + DeliveryError, Envelope, Instant, RpcClientRequest, RpcClientResponseBody, + RpcDeliveryOutcome as DeliveryOutcome, RpcDomainRuntime, RPC_MSG_TYPE_REQUEST, +}; +use crate::domains::rpc::protocol::RpcMessage; + +impl RpcDomainRuntime<'_> { + pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { + if self.handle_cleanup_envelope(envelope) { + return Ok(()); + } + self.ensure_active()?; + Self::log_delivery(envelope); + + let request = Self::extract_request(envelope)?; + let meta = request.meta; + let request_started = self.record_request_start(); + + if !Self::valid_request_envelope(envelope, meta) { + let response_meta = Self::response_meta_for_source(envelope, meta); + self.route_rpc_client_response( + envelope, + response_meta, + &RpcClientResponseBody::Error("route family mismatch".to_string()), + ); + return Ok(()); + } + + // This request was already queued (on the normal lane) before this + // session's disconnect cleanup ran (on the high-priority lane) and + // jumped ahead of it. Reject rather than silently recreating a worker + // registration or pending request for a session that is already gone + // and will never be cleaned up again. + if self.is_cleaned_up_session(meta.session_id) { + let response_meta = Self::response_meta_for_source(envelope, meta); + self.route_rpc_client_response( + envelope, + response_meta, + &RpcClientResponseBody::Error("session already closed".to_string()), + ); + return Ok(()); + } + + Self::log_parse_start(meta); + + let Some(rpc_msg) = self.parse_request_message( + envelope, + meta, + request.message, + &request.raw_payload, + request_started, + ) else { + return Ok(()); + }; + + if !Self::valid_rpc_message(meta, &rpc_msg) { + let response_meta = Self::response_meta_for_source(envelope, meta); + self.route_rpc_client_response( + envelope, + response_meta, + &RpcClientResponseBody::Error("route family mismatch".to_string()), + ); + return Ok(()); + } + + let (response, snapshot_policy, request_failed) = + self.handle_rpc_message(envelope, &meta, rpc_msg); + + self.complete_request( + envelope, + meta, + response, + snapshot_policy, + request_failed, + request_started, + ); + + Ok(()) + } + + fn handle_rpc_message( + &self, + envelope: &Envelope, + meta: &crate::runtime::ClientFrameMeta, + rpc_msg: RpcMessage, + ) -> DeliveryOutcome { + match rpc_msg { + RpcMessage::RegisterWorker { + worker_addr, + max_concurrent, + } => self.handle_register_worker_message(envelope, meta, worker_addr, max_concurrent), + RpcMessage::UnregisterWorker { worker_addr } => { + self.handle_unregister_worker_message(meta, worker_addr) + } + RpcMessage::Request(req) => self.handle_request_message(envelope, meta, req), + RpcMessage::Response(resp) => self.handle_response_message(envelope, meta, &resp), + } + } + + fn ensure_active(&self) -> Result<(), DeliveryError> { + crate::runtime::ingress_support::ensure_actor_active(self.active) + } + + fn log_delivery(envelope: &Envelope) { + crate::runtime::ingress_support::log_envelope_received( + "rpc", + "RPC domain sink: received envelope", + envelope, + ); + } + + fn extract_request(envelope: &Envelope) -> Result { + Self::request_from_envelope(envelope).ok_or_else(|| { + tracing::warn!(domain = "rpc", "Envelope payload was not RpcClientRequest"); + DeliveryError::ActorStopped + }) + } + + fn record_request_start(&self) -> Option { + self.metrics + .as_ref() + .map(crate::domains::rpc::RpcMetrics::record_request_start) + } + + fn log_parse_start(meta: crate::runtime::ClientFrameMeta) { + tracing::debug!( + domain = "rpc", + session = meta.session_id, + msg_type = meta.message_type, + "RPC: parsing request" + ); + } + + fn parse_request_message( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + message: Result< + crate::domains::rpc::protocol::RpcMessage, + crate::domains::rpc::protocol::RpcDecodeError, + >, + raw_payload: &[u8], + request_started: Option, + ) -> Option { + match message { + Ok(msg) => Some(msg), + Err(e) => { + if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) + { + metrics.record_failure(started_at); + } + tracing::warn!(domain = "rpc", error = %e, "Failed to parse RPC message"); + let (error_code, error_message) = match &e { + crate::domains::rpc::protocol::RpcDecodeError::InvalidCallRoute(_) => ( + crate::dispatch::protocol::error_codes::rpc::ERR_INVALID_ROUTE, + "Invalid RPC call route", + ), + crate::domains::rpc::protocol::RpcDecodeError::InvalidRegistrationPattern( + _, + ) => ( + crate::dispatch::protocol::error_codes::rpc::ERR_INVALID_SUBSCRIPTION_PATTERN, + "Invalid RPC registration pattern", + ), + crate::domains::rpc::protocol::RpcDecodeError::StructurallyUndecodable(_) => ( + crate::dispatch::protocol::error_codes::rpc::ERR_BACKEND_ERROR, + "RPC message parse failed", + ), + }; + if meta.message_type == RPC_MSG_TYPE_REQUEST { + if let Ok(correlation_id) = + crate::dispatch::protocol::rpc_codec::extract_request_correlation_id( + raw_payload, + ) + { + self.route_rpc_terminal_error_response( + envelope, + Self::response_meta_for_source(envelope, meta), + correlation_id, + error_code, + error_message, + ); + return None; + } + } + self.route_rpc_client_response( + envelope, + Self::response_meta_for_source(envelope, meta), + &RpcClientResponseBody::CodeError { + code: error_code, + message: error_message.to_string(), + }, + ); + None + } + } + } + + fn valid_request_envelope(envelope: &Envelope, meta: crate::runtime::ClientFrameMeta) -> bool { + meta.route_family == *envelope.destination().family() + && envelope + .source() + .is_none_or(|source| *source.family() == meta.route_family) + } + + pub(super) fn response_meta_for_source( + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + ) -> crate::runtime::ClientFrameMeta { + envelope.source().map_or(meta, |source| { + let mut response_meta = meta; + response_meta.route_family = *source.family(); + response_meta + }) + } + + fn valid_rpc_message( + meta: crate::runtime::ClientFrameMeta, + message: &crate::domains::rpc::protocol::RpcMessage, + ) -> bool { + use crate::domains::rpc::protocol::RpcMessage; + + match message { + RpcMessage::RegisterWorker { worker_addr, .. } + | RpcMessage::UnregisterWorker { worker_addr } => { + *worker_addr.family() == meta.route_family + } + RpcMessage::Request(request) => request.family_id == meta.route_family, + RpcMessage::Response(_) => true, + } + } + + fn complete_request( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + response: Option, + snapshot_policy: Option, + request_failed: bool, + request_started: Option, + ) { + if let Some(force_snapshot) = snapshot_policy { + self.schedule_admin_snapshot(force_snapshot); + } + + if let Some(response) = response { + self.route_rpc_client_response(envelope, meta, &response); + } + + if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { + if request_failed { + metrics.record_failure(started_at); + } else { + metrics.record_success(started_at); + } + } + } +} diff --git a/src/domains/rpc/sink/mailbox.rs b/src/domains/rpc/sink/mailbox.rs new file mode 100644 index 00000000..0a71ac98 --- /dev/null +++ b/src/domains/rpc/sink/mailbox.rs @@ -0,0 +1,151 @@ +//! Mailbox-lane routing and the domain actor's message loop. + +use super::state_model::{ + DeliveryError, Envelope, MailboxSink, RpcDomainActor, RpcDomainCommand, RpcDomainSink, +}; +use crate::runtime::{Actor, Context}; + +impl MailboxSink for RpcDomainSink { + fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.deliver_with_priority(envelope, false) + } + + fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.deliver_with_priority(envelope, true) + } +} + +impl RpcDomainSink { + fn deliver_with_priority( + &self, + envelope: Envelope, + high_priority: bool, + ) -> Result<(), DeliveryError> { + // Family liveness is gated per-family inside `try_enqueue` below + // (`FamilyActorPoolRuntime::is_family_running`) -- a panic scoped to + // one route family must not reject delivery to every other family + // sharing this pool. + if !self.actor.is_running() { + return Err(DeliveryError::ActorStopped); + } + if self.family_runtime.is_some() { + self.deliver_to_family(envelope, high_priority) + } else { + self.deliver_to_actor(envelope, high_priority) + } + } +} + +impl Actor for RpcDomainActor { + type Message = RpcDomainCommand; + + fn receive(&mut self, msg: Self::Message, _ctx: &mut Context) { + let runtime = self.runtime(); + match msg { + RpcDomainCommand::Deliver(envelope, reply) => { + let _ = reply.send(runtime.deliver_envelope(&envelope)); + } + RpcDomainCommand::ExpireTimedOutRequestsAt(now, reply) => { + runtime.expire_timed_out_requests_at(now); + if let Some(reply) = reply { + let _ = reply.send(()); + } + } + RpcDomainCommand::ReadLiveCounts(reply) => { + let _ = reply.send(runtime.live_counts()); + } + #[cfg(test)] + RpcDomainCommand::SyncAdminSnapshot(reply) => { + runtime.sync_admin_snapshot(); + if let Some(reply) = reply { + let _ = reply.send(()); + } + } + RpcDomainCommand::RefreshAdminSnapshotIfDirty(reply) => { + runtime.refresh_admin_snapshot_if_dirty(); + if let Some(reply) = reply { + let _ = reply.send(()); + } + } + #[cfg(test)] + RpcDomainCommand::ApplySessionCleanupForTests(session_id, reply) => { + let _ = reply.send(runtime.apply_session_cleanup(session_id)); + } + #[cfg(test)] + RpcDomainCommand::ApplyWorkerUnsubscribeForTests(worker_addr, session_id, reply) => { + let _ = reply.send(runtime.apply_worker_unsubscribe(&worker_addr, session_id)); + } + #[cfg(test)] + RpcDomainCommand::PanicForTests => { + panic!("test RPC domain actor panic"); + } + } + } +} + +impl RpcDomainSink { + fn deliver_to_family( + &self, + envelope: Envelope, + high_priority: bool, + ) -> Result<(), DeliveryError> { + let Some(runtime) = self.family_runtime.as_ref() else { + return Err(DeliveryError::ActorStopped); + }; + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + let family = *envelope.destination().family(); + let command = RpcDomainCommand::Deliver(envelope, reply_tx); + let lane = if high_priority { + crate::runtime::FamilyActorLane::Control + } else { + crate::runtime::FamilyActorLane::Normal + }; + runtime + .try_enqueue(family, lane, command) + .map_err(Self::family_enqueue_error)?; + + // Family delivery is called synchronously by the async transport edge. + // Client responses are routed by the actor itself; waiting here would + // block a Tokio worker while the synchronous domain actor runs. + drop(reply_rx); + Ok(()) + } + + fn family_enqueue_error(error: crate::runtime::FamilyActorEnqueueError) -> DeliveryError { + match error { + crate::runtime::FamilyActorEnqueueError::NormalLaneFull => DeliveryError::MailboxFull { + capacity: crate::runtime::FAMILY_ACTOR_NORMAL_LANE_CAPACITY, + current_len: crate::runtime::FAMILY_ACTOR_NORMAL_LANE_CAPACITY, + }, + crate::runtime::FamilyActorEnqueueError::ControlLaneFull => { + DeliveryError::HighLaneFull { + capacity: crate::runtime::FAMILY_ACTOR_CONTROL_LANE_CAPACITY, + current_len: crate::runtime::FAMILY_ACTOR_CONTROL_LANE_CAPACITY, + } + } + crate::runtime::FamilyActorEnqueueError::UnknownFamily + | crate::runtime::FamilyActorEnqueueError::ActorStopped => DeliveryError::ActorStopped, + } + } + + fn deliver_to_actor( + &self, + envelope: Envelope, + high_priority: bool, + ) -> Result<(), DeliveryError> { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + let command = RpcDomainCommand::Deliver(envelope, reply_tx); + let enqueue_result = if high_priority { + self.actor.try_send_high_priority(command) + } else { + self.actor.try_send(command) + }; + enqueue_result?; + + // Reporting a busy actor as a stopped one costs the caller its session: + // ingress treats `ActorStopped` as fatal but `Timeout` as retryable. + reply_rx + .recv_timeout(super::state_model::RPC_ACTOR_REPLY_TIMEOUT) + .unwrap_or_else(|error| Err(crate::runtime::reply_wait::map_reply_wait_error(error))) + } +} diff --git a/src/domains/rpc/sink/mailbox_adapter.rs b/src/domains/rpc/sink/mailbox_adapter.rs index 5cdc93d6..3f19efcf 100644 --- a/src/domains/rpc/sink/mailbox_adapter.rs +++ b/src/domains/rpc/sink/mailbox_adapter.rs @@ -57,7 +57,7 @@ impl RpcDomainRuntime<'_> { ); FrameContext::new( meta.session_id, - test_protocol_channel_from_client(meta.channel), + crate::protocol::test_support::channel_id_from_client(meta.channel), crate::dispatch::protocol::tlv::MessageType::new(meta.message_type), bytes::Bytes::from(response_bytes), meta.route_family, @@ -104,7 +104,7 @@ impl RpcDomainRuntime<'_> { ); FrameContext::new( meta.session_id, - test_protocol_channel_from_client(meta.channel), + crate::protocol::test_support::channel_id_from_client(meta.channel), crate::dispatch::protocol::tlv::MessageType::new(RPC_MSG_TYPE_RESPONSE), bytes::Bytes::from(response_bytes), meta.route_family, @@ -170,21 +170,3 @@ fn test_client_channel_from_protocol( } } } - -#[cfg(test)] -pub(super) fn test_protocol_channel_from_client( - channel: crate::runtime::ClientChannel, -) -> crate::dispatch::protocol::frame::ChannelId { - match channel { - crate::runtime::ClientChannel::Control => { - crate::dispatch::protocol::frame::ChannelId::Control - } - crate::runtime::ClientChannel::Pub => crate::dispatch::protocol::frame::ChannelId::Pub, - crate::runtime::ClientChannel::Sub => crate::dispatch::protocol::frame::ChannelId::Sub, - crate::runtime::ClientChannel::Rpc => crate::dispatch::protocol::frame::ChannelId::Rpc, - crate::runtime::ClientChannel::Lease => crate::dispatch::protocol::frame::ChannelId::Lease, - crate::runtime::ClientChannel::Internal => { - crate::dispatch::protocol::frame::ChannelId::Internal - } - } -} diff --git a/src/domains/rpc/sink/mailbox_sink_impl.rs b/src/domains/rpc/sink/mailbox_sink_impl.rs deleted file mode 100644 index cd1fe9ef..00000000 --- a/src/domains/rpc/sink/mailbox_sink_impl.rs +++ /dev/null @@ -1,758 +0,0 @@ -use super::state_model::{ - session_inbox_address, DeliveryError, Envelope, Instant, MailboxSink, Ordering, - RpcClientRequest, RpcClientResponseBody, RpcDeliveryOutcome as DeliveryOutcome, RpcDomainActor, - RpcDomainCommand, RpcDomainRuntime, RpcDomainSink, RpcRequestRejection, RpcRequestState, - RpcWorker, RPC_BACKPRESSURE_ERROR, RPC_DUPLICATE_CORRELATION_ERROR, RPC_MAX_PENDING_REQUESTS, - RPC_MSG_TYPE_REQUEST, RPC_NO_WORKERS_ERROR, RPC_WORKER_NOT_FOUND_ERROR, -}; -use crate::domains::rpc::protocol::{RpcMessage, RpcRequest}; -use crate::runtime::{Actor, Context}; - -struct RejectionSpec { - metric: &'static str, - error_code: u16, - message: &'static str, - reason: &'static str, -} - -const REJECTION_SPECS: [RejectionSpec; 4] = [ - RejectionSpec { - metric: "rpc_requests_rejected_duplicate_correlation_total", - error_code: crate::dispatch::protocol::error_codes::rpc::ERR_RPC_DUPLICATE_CORRELATION, - message: RPC_DUPLICATE_CORRELATION_ERROR, - reason: "duplicate live correlation", - }, - RejectionSpec { - metric: "rpc_requests_rejected_no_worker_total", - error_code: crate::dispatch::protocol::error_codes::rpc::ERR_ROUTE_NOT_REGISTERED, - message: RPC_NO_WORKERS_ERROR, - reason: "no matching worker registration", - }, - RejectionSpec { - metric: "rpc_requests_rejected_backpressure_total", - error_code: crate::dispatch::protocol::error_codes::rpc::ERR_RPC_BACKPRESSURE, - message: RPC_BACKPRESSURE_ERROR, - reason: "global pending capacity", - }, - RejectionSpec { - metric: "rpc_requests_rejected_backpressure_total", - error_code: crate::dispatch::protocol::error_codes::rpc::ERR_RPC_BACKPRESSURE, - message: RPC_BACKPRESSURE_ERROR, - reason: "route pending capacity", - }, -]; - -impl MailboxSink for RpcDomainSink { - fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { - self.deliver_with_priority(envelope, false) - } - - fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { - self.deliver_with_priority(envelope, true) - } -} - -impl RpcDomainSink { - fn deliver_with_priority( - &self, - envelope: Envelope, - high_priority: bool, - ) -> Result<(), DeliveryError> { - if !self.actor.is_running() - || self - .family_runtime - .as_ref() - .is_some_and(|runtime| !runtime.is_running()) - { - return Err(DeliveryError::ActorStopped); - } - if self.family_runtime.is_some() { - self.deliver_to_family(envelope, high_priority) - } else { - self.deliver_to_actor(envelope, high_priority) - } - } -} - -impl Actor for RpcDomainActor { - type Message = RpcDomainCommand; - - fn receive(&mut self, msg: Self::Message, _ctx: &mut Context) { - let runtime = self.runtime(); - match msg { - RpcDomainCommand::Deliver(envelope, reply) => { - let _ = reply.send(runtime.deliver_envelope(&envelope)); - } - RpcDomainCommand::ExpireTimedOutRequestsAt(now, reply) => { - runtime.expire_timed_out_requests_at(now); - if let Some(reply) = reply { - let _ = reply.send(()); - } - } - RpcDomainCommand::ReadLiveCounts(reply) => { - let _ = reply.send(runtime.live_counts()); - } - #[cfg(test)] - RpcDomainCommand::SyncAdminSnapshot(reply) => { - runtime.sync_admin_snapshot(); - if let Some(reply) = reply { - let _ = reply.send(()); - } - } - RpcDomainCommand::RefreshAdminSnapshotIfDirty(reply) => { - runtime.refresh_admin_snapshot_if_dirty(); - if let Some(reply) = reply { - let _ = reply.send(()); - } - } - #[cfg(test)] - RpcDomainCommand::ApplySessionCleanupForTests(session_id, reply) => { - let _ = reply.send(runtime.apply_session_cleanup(session_id)); - } - #[cfg(test)] - RpcDomainCommand::ApplyWorkerUnsubscribeForTests(worker_addr, session_id, reply) => { - let _ = reply.send(runtime.apply_worker_unsubscribe(&worker_addr, session_id)); - } - #[cfg(test)] - RpcDomainCommand::PanicForTests => { - panic!("test RPC domain actor panic"); - } - } - } -} - -impl RpcDomainSink { - fn deliver_to_family( - &self, - envelope: Envelope, - high_priority: bool, - ) -> Result<(), DeliveryError> { - let Some(runtime) = self.family_runtime.as_ref() else { - return Err(DeliveryError::ActorStopped); - }; - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - let family = *envelope.destination().family(); - let command = RpcDomainCommand::Deliver(envelope, reply_tx); - let lane = if high_priority { - crate::runtime::FamilyActorLane::Control - } else { - crate::runtime::FamilyActorLane::Normal - }; - runtime - .try_enqueue(family, lane, command) - .map_err(Self::family_enqueue_error)?; - - // Family delivery is called synchronously by the async transport edge. - // Client responses are routed by the actor itself; waiting here would - // block a Tokio worker while the synchronous domain actor runs. - drop(reply_rx); - Ok(()) - } - - fn family_enqueue_error(error: crate::runtime::FamilyActorEnqueueError) -> DeliveryError { - match error { - crate::runtime::FamilyActorEnqueueError::NormalLaneFull => DeliveryError::MailboxFull { - capacity: crate::runtime::FAMILY_ACTOR_NORMAL_LANE_CAPACITY, - current_len: crate::runtime::FAMILY_ACTOR_NORMAL_LANE_CAPACITY, - }, - crate::runtime::FamilyActorEnqueueError::ControlLaneFull => { - DeliveryError::HighLaneFull { - capacity: crate::runtime::FAMILY_ACTOR_CONTROL_LANE_CAPACITY, - current_len: crate::runtime::FAMILY_ACTOR_CONTROL_LANE_CAPACITY, - } - } - crate::runtime::FamilyActorEnqueueError::UnknownFamily - | crate::runtime::FamilyActorEnqueueError::ActorStopped => DeliveryError::ActorStopped, - } - } - - fn deliver_to_actor( - &self, - envelope: Envelope, - high_priority: bool, - ) -> Result<(), DeliveryError> { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - let command = RpcDomainCommand::Deliver(envelope, reply_tx); - let enqueue_result = if high_priority { - self.actor.try_send_high_priority(command) - } else { - self.actor.try_send(command) - }; - enqueue_result?; - - reply_rx - .recv_timeout(std::time::Duration::from_secs(1)) - .unwrap_or(Err(DeliveryError::ActorStopped)) - } -} - -impl RpcDomainRuntime<'_> { - pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { - if self.handle_cleanup_envelope(envelope) { - return Ok(()); - } - self.ensure_active()?; - Self::log_delivery(envelope); - - let request = Self::extract_request(envelope)?; - let meta = request.meta; - let request_started = self.record_request_start(); - - if !Self::valid_request_envelope(envelope, meta) { - let response_meta = Self::response_meta_for_source(envelope, meta); - self.route_rpc_client_response( - envelope, - response_meta, - &RpcClientResponseBody::Error("route family mismatch".to_string()), - ); - return Ok(()); - } - - Self::log_parse_start(meta); - - let Some(rpc_msg) = self.parse_request_message( - envelope, - meta, - request.message, - &request.raw_payload, - request_started, - ) else { - return Ok(()); - }; - - if !Self::valid_rpc_message(meta, &rpc_msg) { - let response_meta = Self::response_meta_for_source(envelope, meta); - self.route_rpc_client_response( - envelope, - response_meta, - &RpcClientResponseBody::Error("route family mismatch".to_string()), - ); - return Ok(()); - } - - let (response, snapshot_policy, request_failed) = - self.handle_rpc_message(envelope, &meta, rpc_msg); - - self.complete_request( - envelope, - meta, - response, - snapshot_policy, - request_failed, - request_started, - ); - - Ok(()) - } - - fn handle_rpc_message( - &self, - envelope: &Envelope, - meta: &crate::runtime::ClientFrameMeta, - rpc_msg: RpcMessage, - ) -> DeliveryOutcome { - match rpc_msg { - RpcMessage::RegisterWorker { - worker_addr, - max_concurrent, - } => self.handle_register_worker_message(envelope, meta, worker_addr, max_concurrent), - RpcMessage::UnregisterWorker { worker_addr } => { - self.handle_unregister_worker_message(meta, worker_addr) - } - RpcMessage::Request(req) => self.handle_request_message(envelope, meta, req), - RpcMessage::Response(resp) => self.handle_response_message(envelope, meta, &resp), - } - } - - #[allow(clippy::needless_pass_by_value)] - fn handle_register_worker_message( - &self, - envelope: &Envelope, - meta: &crate::runtime::ClientFrameMeta, - worker_addr: crate::runtime::routing::RouteAddress, - max_concurrent: usize, - ) -> DeliveryOutcome { - let worker_inbox_addr = envelope.source().cloned().unwrap_or_else(|| { - session_inbox_address(*envelope.destination().family(), meta.session_id) - }); - { - let mut state = self.state.lock(); - if matches!( - RpcRequestState::register( - &mut *state, - RpcWorker::new( - worker_addr.clone(), - worker_inbox_addr, - meta.session_id, - max_concurrent, - ) - ), - super::state_model::RpcWorkerRegistration::WildcardLimit - ) { - return ( - Some(RpcClientResponseBody::CodeError { - code: crate::dispatch::protocol::error_codes::rpc::ERR_SUBSCRIPTION_LIMIT, - message: "wildcard subscription limit exceeded (128 per session)" - .to_string(), - }), - Some(false), - false, - ); - } - } - self.dispatch_queued_requests_for_family(*worker_addr.family()); - tracing::debug!( - domain = "rpc", - worker = worker_addr.route().as_str(), - session = meta.session_id, - "Worker registered" - ); - self.refresh_metrics_gauges(); - ( - Some(RpcClientResponseBody::Ok { data: vec![] }), - Some(true), - false, - ) - } - - #[allow(clippy::needless_pass_by_value)] - fn handle_unregister_worker_message( - &self, - meta: &crate::runtime::ClientFrameMeta, - worker_addr: crate::runtime::routing::RouteAddress, - ) -> DeliveryOutcome { - let cleanup_result = self.apply_worker_unsubscribe(&worker_addr, meta.session_id); - self.forward_worker_disconnect_errors(cleanup_result.disconnect_deliveries); - tracing::debug!( - domain = "rpc", - worker = worker_addr.route().as_str(), - session = meta.session_id, - removed_workers = cleanup_result.removed_registrations, - removed_pending = cleanup_result.removed_pending, - "Worker unregistered" - ); - ( - Some(RpcClientResponseBody::Ok { data: vec![] }), - Some(true), - false, - ) - } - - fn handle_request_message( - &self, - envelope: &Envelope, - meta: &crate::runtime::ClientFrameMeta, - req: RpcRequest, - ) -> DeliveryOutcome { - self.expire_timed_out_requests_inline_if_due(); - self.counter_inc("rpc_requests_total"); - let caller_inbox_addr = envelope - .source() - .cloned() - .unwrap_or_else(|| session_inbox_address(meta.route_family, meta.session_id)); - - let metrics_enabled = self.metrics.is_some(); - let state_wait_start = metrics_enabled.then(Instant::now); - let mut state = self.state.lock(); - let state_wait_us = state_wait_start.map_or(0, Self::elapsed_micros_u64); - let state_hold_start = metrics_enabled.then(Instant::now); - let dispatch = RpcRequestState::dispatch_or_queue( - &mut *state, - req, - meta.session_id, - caller_inbox_addr, - self.request_timeout, - self.route_pending_capacity, - RPC_MAX_PENDING_REQUESTS, - self.enforce_global_pending_count - .then_some(self.global_pending_count.as_ref()), - ); - let state_hold_us = state_hold_start.map_or(0, Self::elapsed_micros_u64); - drop(state); - - self.observe_request_state_metrics(0, state_wait_us, state_hold_us, 0); - - match dispatch { - super::state_model::RpcRequestDispatch::Rejected { request, reason } => { - self.reject_with_spec(envelope, meta, &request, reason) - } - super::state_model::RpcRequestDispatch::Queued { - route, - correlation_id, - live_request_count, - } => self.accept_queued_request( - &route, - meta.route_family, - correlation_id, - live_request_count, - ), - super::state_model::RpcRequestDispatch::Immediate { - request, - registration, - live_request_count, - } => self.forward_immediate_request( - envelope, - meta, - request, - ®istration, - live_request_count, - ), - } - } - - fn observe_request_state_metrics( - &self, - route_registry_lookup_us: u64, - state_wait_us: u64, - state_hold_us: u64, - worker_selection_us: u64, - ) { - self.histogram_observe_us("rpc_route_registry_lookup_us", route_registry_lookup_us); - self.histogram_observe_us("rpc_dispatch_state_lock_us", state_wait_us); - self.histogram_observe_us("rpc_dispatch_state_wait_us", state_wait_us); - self.histogram_observe_us("rpc_dispatch_state_hold_us", state_hold_us); - self.histogram_observe_us("rpc_worker_selection_us", worker_selection_us); - } - - fn reject_with_spec( - &self, - envelope: &Envelope, - meta: &crate::runtime::ClientFrameMeta, - req: &RpcRequest, - reason: RpcRequestRejection, - ) -> DeliveryOutcome { - let spec = &REJECTION_SPECS[reason as usize]; - self.counter_inc(spec.metric); - tracing::warn!( - domain = "rpc", - correlation_id = %req.correlation_id, - route = req.route.as_str(), - reason = spec.reason, - "Rejected RPC request" - ); - self.reject_request_with_terminal_error(envelope, *meta, req, spec.error_code, spec.message) - } - - fn accept_queued_request( - &self, - route: &crate::runtime::routing::Route, - family: crate::runtime::routing::RouteFamily, - correlation_id: uuid::Uuid, - live_request_count: usize, - ) -> DeliveryOutcome { - self.histogram_observe_us("rpc_pending_track_us", 0); - self.histogram_observe_us("rpc_pending_route_index_us", 0); - self.gauge_set("rpc_pending_requests", live_request_count as u64); - self.schedule_admin_snapshot(false); - self.dispatch_queued_requests_for_family(family); - - tracing::debug!( - domain = "rpc", - correlation_id = %correlation_id, - route = route.as_str(), - live_request_count, - "Request queued on route-local RPC pending queue" - ); - - (None, None, false) - } - - fn forward_immediate_request( - &self, - envelope: &Envelope, - meta: &crate::runtime::ClientFrameMeta, - req: RpcRequest, - worker: &super::state_model::RpcWorkerDispatch, - live_request_count: usize, - ) -> DeliveryOutcome { - self.histogram_observe_us("rpc_pending_track_us", 0); - self.histogram_observe_us("rpc_pending_route_index_us", 0); - self.gauge_set("rpc_pending_requests", live_request_count as u64); - self.schedule_admin_snapshot(false); - - let metrics_enabled = self.metrics.is_some(); - let request_forward_start = metrics_enabled.then(Instant::now); - let forward_result = self.forward_request_to_worker(&req, worker); - if let Some(request_forward_start) = request_forward_start { - self.histogram_observe_elapsed_us("rpc_request_forward_us", request_forward_start); - } - - match forward_result { - Ok(()) => { - self.counter_inc("rpc_requests_dispatched_total"); - tracing::debug!( - domain = "rpc", - correlation_id = %req.correlation_id, - route = req.route.as_str(), - "Request forwarded to worker" - ); - (None, Some(false), false) - } - Err( - crate::runtime::RouteError::RouteNotFound(_) - | crate::runtime::RouteError::DeliveryFailed( - _, - DeliveryError::ActorStopped - | DeliveryError::Timeout - | DeliveryError::SinkPanicked, - ), - ) => self.handle_disconnected_worker_dispatch(envelope, meta, req, worker.session_id), - Err(crate::runtime::RouteError::DeliveryFailed( - _, - DeliveryError::MailboxFull { .. } | DeliveryError::HighLaneFull { .. }, - )) => self.handle_backpressured_worker_dispatch(envelope, meta, req), - } - } - - #[allow(clippy::needless_pass_by_value)] - fn handle_disconnected_worker_dispatch( - &self, - envelope: &Envelope, - meta: &crate::runtime::ClientFrameMeta, - req: RpcRequest, - worker_session_id: u64, - ) -> DeliveryOutcome { - self.counter_inc("rpc_request_forward_errors_total"); - let cleanup_result = self.apply_session_cleanup(worker_session_id); - let disconnect_deliveries = cleanup_result - .disconnect_deliveries - .into_iter() - .filter(|delivery| { - delivery.correlation_id != req.correlation_id - || *delivery.caller_inbox_addr.family() != meta.route_family - }) - .collect(); - self.forward_worker_disconnect_errors(disconnect_deliveries); - tracing::warn!( - domain = "rpc", - correlation_id = %req.correlation_id, - route = req.route.as_str(), - worker_session_id, - "Worker disconnected before request dispatch completed" - ); - self.reject_request_with_terminal_error( - envelope, - *meta, - &req, - crate::dispatch::protocol::error_codes::rpc::ERR_WORKER_NOT_FOUND, - RPC_WORKER_NOT_FOUND_ERROR, - ) - } - - #[allow(clippy::needless_pass_by_value)] - fn handle_backpressured_worker_dispatch( - &self, - envelope: &Envelope, - meta: &crate::runtime::ClientFrameMeta, - req: RpcRequest, - ) -> DeliveryOutcome { - self.counter_inc("rpc_request_forward_errors_total"); - let pending_len = self - .remove_pending_request_for_family(meta.route_family, &req.correlation_id) - .map(|(_, pending_len)| pending_len) - .unwrap_or_default(); - tracing::warn!( - domain = "rpc", - correlation_id = %req.correlation_id, - route = req.route.as_str(), - pending_len, - "Failed to forward request to worker due to backpressure" - ); - self.reject_request_with_terminal_error( - envelope, - *meta, - &req, - crate::dispatch::protocol::error_codes::rpc::ERR_RPC_BACKPRESSURE, - RPC_BACKPRESSURE_ERROR, - ) - } - - fn reject_request_with_terminal_error( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - req: &RpcRequest, - code: u16, - message: &'static str, - ) -> DeliveryOutcome { - self.route_rpc_terminal_error_response(envelope, meta, req.correlation_id, code, message); - (None, None, true) - } - - fn elapsed_micros_u64(start: Instant) -> u64 { - start.elapsed().as_micros().try_into().unwrap_or(u64::MAX) - } - - fn handle_cleanup_envelope(&self, envelope: &Envelope) -> bool { - if let Some(cleanup) = envelope.payload::() { - let cleanup_result = self.apply_session_cleanup(cleanup.session_id); - self.forward_worker_disconnect_errors(cleanup_result.disconnect_deliveries); - return true; - } - - false - } - - fn ensure_active(&self) -> Result<(), DeliveryError> { - if !self.active.load(Ordering::Relaxed) { - return Err(DeliveryError::ActorStopped); - } - - Ok(()) - } - - fn log_delivery(envelope: &Envelope) { - tracing::debug!( - domain = "rpc", - destination = %envelope.destination(), - source = ?envelope.source(), - "RPC domain sink: received envelope" - ); - } - - fn extract_request(envelope: &Envelope) -> Result { - Self::request_from_envelope(envelope).ok_or_else(|| { - tracing::warn!(domain = "rpc", "Envelope payload was not RpcClientRequest"); - DeliveryError::ActorStopped - }) - } - - fn record_request_start(&self) -> Option { - self.metrics - .as_ref() - .map(crate::domains::rpc::RpcMetrics::record_request_start) - } - - fn log_parse_start(meta: crate::runtime::ClientFrameMeta) { - tracing::debug!( - domain = "rpc", - session = meta.session_id, - msg_type = meta.message_type, - "RPC: parsing request" - ); - } - - fn parse_request_message( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - message: Result< - crate::domains::rpc::protocol::RpcMessage, - crate::domains::rpc::protocol::RpcDecodeError, - >, - raw_payload: &[u8], - request_started: Option, - ) -> Option { - match message { - Ok(msg) => Some(msg), - Err(e) => { - if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) - { - metrics.record_failure(started_at); - } - tracing::warn!(domain = "rpc", error = %e, "Failed to parse RPC message"); - let (error_code, error_message) = match &e { - crate::domains::rpc::protocol::RpcDecodeError::InvalidCallRoute(_) => ( - crate::dispatch::protocol::error_codes::rpc::ERR_INVALID_ROUTE, - "Invalid RPC call route", - ), - crate::domains::rpc::protocol::RpcDecodeError::InvalidRegistrationPattern( - _, - ) => ( - crate::dispatch::protocol::error_codes::rpc::ERR_INVALID_SUBSCRIPTION_PATTERN, - "Invalid RPC registration pattern", - ), - crate::domains::rpc::protocol::RpcDecodeError::StructurallyUndecodable(_) => ( - crate::dispatch::protocol::error_codes::rpc::ERR_BACKEND_ERROR, - "RPC message parse failed", - ), - }; - if meta.message_type == RPC_MSG_TYPE_REQUEST { - if let Ok(correlation_id) = - crate::dispatch::protocol::rpc_codec::extract_request_correlation_id( - raw_payload, - ) - { - self.route_rpc_terminal_error_response( - envelope, - Self::response_meta_for_source(envelope, meta), - correlation_id, - error_code, - error_message, - ); - return None; - } - } - self.route_rpc_client_response( - envelope, - Self::response_meta_for_source(envelope, meta), - &RpcClientResponseBody::CodeError { - code: error_code, - message: error_message.to_string(), - }, - ); - None - } - } - } - - fn valid_request_envelope(envelope: &Envelope, meta: crate::runtime::ClientFrameMeta) -> bool { - meta.route_family == *envelope.destination().family() - && envelope - .source() - .is_none_or(|source| *source.family() == meta.route_family) - } - - fn response_meta_for_source( - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - ) -> crate::runtime::ClientFrameMeta { - envelope.source().map_or(meta, |source| { - let mut response_meta = meta; - response_meta.route_family = *source.family(); - response_meta - }) - } - - fn valid_rpc_message( - meta: crate::runtime::ClientFrameMeta, - message: &crate::domains::rpc::protocol::RpcMessage, - ) -> bool { - use crate::domains::rpc::protocol::RpcMessage; - - match message { - RpcMessage::RegisterWorker { worker_addr, .. } - | RpcMessage::UnregisterWorker { worker_addr } => { - *worker_addr.family() == meta.route_family - } - RpcMessage::Request(request) => request.family_id == meta.route_family, - RpcMessage::Response(_) => true, - } - } - - fn complete_request( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - response: Option, - snapshot_policy: Option, - request_failed: bool, - request_started: Option, - ) { - if let Some(force_snapshot) = snapshot_policy { - self.schedule_admin_snapshot(force_snapshot); - } - - if let Some(response) = response { - self.route_rpc_client_response(envelope, meta, &response); - } - - if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { - if request_failed { - metrics.record_failure(started_at); - } else { - metrics.record_success(started_at); - } - } - } -} diff --git a/src/domains/rpc/sink/mod.rs b/src/domains/rpc/sink/mod.rs index 77517be9..15098b43 100644 --- a/src/domains/rpc/sink/mod.rs +++ b/src/domains/rpc/sink/mod.rs @@ -1,10 +1,14 @@ -mod domain_sink_impl; +mod cleanup; +mod delivery; +mod facade; mod family_runtime; +mod ingress; +mod mailbox; mod mailbox_adapter; -mod mailbox_sink_impl; mod observability; +mod registration; mod response_forwarder; -mod response_sink_impl; +mod responses; mod state_model; pub use state_model::RpcDomainSink; diff --git a/src/domains/rpc/sink/observability.rs b/src/domains/rpc/sink/observability.rs index b5d5da0d..e19c1c44 100644 --- a/src/domains/rpc/sink/observability.rs +++ b/src/domains/rpc/sink/observability.rs @@ -1,5 +1,6 @@ use super::state_model::{ - Arc, Instant, Mutex, RpcDomainCore, RpcDomainRuntime, RpcLiveCounts, RpcState, + rpc_admin_snapshot_due, rpc_timeout_sweep_interval, Arc, Duration, Instant, Mutex, Ordering, + RpcDomainCore, RpcDomainRuntime, RpcLiveCounts, RpcState, RPC_TIMEOUT_ERROR, }; impl RpcDomainCore { @@ -41,6 +42,231 @@ impl RpcDomainCore { } impl RpcDomainRuntime<'_> { + fn u64_to_usize_saturating(value: u64) -> usize { + usize::try_from(value).unwrap_or(usize::MAX) + } + + pub(super) fn elapsed_us_saturating(start: Instant) -> u64 { + start.elapsed().as_micros().try_into().unwrap_or(u64::MAX) + } + + pub(super) fn release_global_pending(&self, count: usize) { + if count == 0 { + return; + } + let _ = self.global_pending_count.fetch_update( + Ordering::AcqRel, + Ordering::Acquire, + |current| Some(current.saturating_sub(count)), + ); + } + + pub(crate) fn timeout_sweep_interval(&self) -> Duration { + rpc_timeout_sweep_interval(self.request_timeout) + } + + pub(super) fn live_counts(&self) -> RpcLiveCounts { + let state = self.state.lock(); + let workers = state.registration_count(); + RpcLiveCounts { + workers, + pending_requests: state.live_request_count(), + } + } + + pub(super) fn counter_inc(&self, name: &str) { + if let Some(ref metrics) = self.metrics { + metrics.counter_inc(name); + } + } + + pub(super) fn counter_add(&self, name: &str, amount: u64) { + if let Some(ref metrics) = self.metrics { + metrics.counter_add(name, amount); + } + } + + pub(super) fn gauge_set(&self, name: &str, value: u64) { + if let Some(ref metrics) = self.metrics { + metrics.gauge_set(name, value); + if name == "rpc_pending_requests" { + metrics.set_pending_request_count(Self::u64_to_usize_saturating(value)); + } + } + } + + pub(super) fn histogram_observe_us(&self, name: &str, value_us: u64) { + if let Some(ref metrics) = self.metrics { + metrics.histogram_observe_us(name, value_us); + } + } + + pub(super) fn histogram_observe_elapsed_us(&self, name: &str, start: Instant) { + self.histogram_observe_us(name, Self::elapsed_us_saturating(start)); + } + + pub(super) fn refresh_metrics_gauges(&self) { + if let Some(metrics) = &self.metrics { + let counts = self.core.aggregate_live_counts(); + metrics.set_worker_count(counts.workers); + metrics.set_pending_request_count(counts.pending_requests); + } + } + + pub(super) fn expire_timed_out_requests_inline_if_due(&self) { + let now_elapsed_us = Self::elapsed_us_saturating(self.snapshot_epoch); + let interval_us = self + .timeout_sweep_interval() + .as_micros() + .try_into() + .unwrap_or(u64::MAX); + let last_elapsed_us = self.last_inline_timeout_elapsed_us.load(Ordering::Relaxed); + + if now_elapsed_us.saturating_sub(last_elapsed_us) < interval_us { + return; + } + + if self + .last_inline_timeout_elapsed_us + .compare_exchange( + last_elapsed_us, + now_elapsed_us, + Ordering::AcqRel, + Ordering::Relaxed, + ) + .is_ok() + { + self.expire_timed_out_requests_at(Instant::now()); + } + } + + pub(super) fn expire_timed_out_requests_at(&self, now: Instant) { + let timeout_result = { + let mut state = self.state.lock(); + state.expire_timed_out(now) + }; + + if timeout_result.removed_pending == 0 { + return; + } + + self.release_global_pending(timeout_result.removed_pending); + let timeout_delivery_count = timeout_result.timeout_deliveries.len(); + self.gauge_set("rpc_pending_requests", timeout_result.pending_len as u64); + self.counter_add( + "rpc_request_timeouts_total", + timeout_result.removed_pending as u64, + ); + self.counter_add( + "rpc_cleanup_pending_removed_total", + timeout_result.removed_pending as u64, + ); + if timeout_result.closed_caller_drops > 0 { + self.counter_add( + "rpc_timeout_errors_dropped_total", + timeout_result.closed_caller_drops as u64, + ); + self.counter_add( + "rpc_responses_dropped_closed_caller_total", + timeout_result.closed_caller_drops as u64, + ); + } + self.schedule_admin_snapshot(false); + self.dispatch_all_queued_requests(); + + tracing::debug!( + domain = "rpc", + removed_pending = timeout_result.removed_pending, + delivered_timeouts = timeout_delivery_count, + closed_caller_drops = timeout_result.closed_caller_drops, + pending_len = timeout_result.pending_len, + "RPC request timeout sweep applied" + ); + + self.forward_pending_error_deliveries( + timeout_result.timeout_deliveries, + crate::dispatch::protocol::error_codes::rpc::ERR_RPC_TIMEOUT, + RPC_TIMEOUT_ERROR, + "rpc_timeout_errors_forwarded_total", + "rpc_timeout_errors_dropped_total", + ); + } + + pub(super) fn pending_request_count(&self) -> usize { + self.live_counts().pending_requests + } + + pub(super) fn refresh_admin_snapshot_if_dirty(&self) { + self.maybe_sync_admin_snapshot(false); + } + + /// Mark the admin snapshot dirty. Forced calls refresh immediately; regular + /// hot-path updates coalesce until an admin read or another forced refresh. + pub(super) fn schedule_admin_snapshot(&self, force: bool) { + if force { + self.snapshot_dirty.store(true, Ordering::Relaxed); + self.maybe_sync_admin_snapshot(true); + return; + } + + if self + .snapshot_dirty + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return; + } + + self.maybe_sync_admin_snapshot(false); + } + + /// Sync the admin snapshot when the snapshot interval elapses or a caller forces it. + /// + /// Even forced snapshots are still point-in-time copies of the sink's current + /// in-memory state, not linearizable reads of concurrent RPC activity. + pub(super) fn maybe_sync_admin_snapshot(&self, force: bool) { + #[cfg(feature = "bench-no-snapshot")] + if !force { + return; + } + + let now_elapsed_us = Self::elapsed_us_saturating(self.snapshot_epoch); + let last_snapshot_elapsed_us = self.last_snapshot_elapsed_us.load(Ordering::Relaxed); + let snapshot_dirty = self.snapshot_dirty.load(Ordering::Relaxed); + + if !rpc_admin_snapshot_due( + snapshot_dirty, + force, + now_elapsed_us, + last_snapshot_elapsed_us, + ) { + return; + } + + if self + .snapshot_syncing + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return; + } + + if !self.snapshot_dirty.swap(false, Ordering::AcqRel) { + self.snapshot_syncing.store(false, Ordering::Release); + return; + } + + let snapshot_start = Instant::now(); + self.sync_admin_snapshot(); + let snapshot_time_us = Self::elapsed_us_saturating(snapshot_start); + self.last_snapshot_elapsed_us.store( + Self::elapsed_us_saturating(self.snapshot_epoch), + Ordering::Relaxed, + ); + self.snapshot_syncing.store(false, Ordering::Release); + self.histogram_observe_us("rpc_admin_snapshot_us", snapshot_time_us); + } + /// Copy a point-in-time view of live in-memory RPC state into the admin read /// model for the current broker process only. /// diff --git a/src/domains/rpc/sink/registration.rs b/src/domains/rpc/sink/registration.rs new file mode 100644 index 00000000..5287c41e --- /dev/null +++ b/src/domains/rpc/sink/registration.rs @@ -0,0 +1,82 @@ +//! Worker registration and unregistration message handling. + +use super::state_model::{ + session_inbox_address, Envelope, RpcClientResponseBody, RpcDeliveryOutcome as DeliveryOutcome, + RpcDomainRuntime, RpcRequestState, RpcWorker, RpcWorkerRegistration, +}; + +impl RpcDomainRuntime<'_> { + #[allow(clippy::needless_pass_by_value)] + pub(super) fn handle_register_worker_message( + &self, + envelope: &Envelope, + meta: &crate::runtime::ClientFrameMeta, + worker_addr: crate::runtime::routing::RouteAddress, + max_concurrent: usize, + ) -> DeliveryOutcome { + let worker_inbox_addr = envelope.source().cloned().unwrap_or_else(|| { + session_inbox_address(*envelope.destination().family(), meta.session_id) + }); + { + let mut state = self.state.lock(); + if matches!( + RpcRequestState::register( + &mut *state, + RpcWorker::new( + worker_addr.clone(), + worker_inbox_addr, + meta.session_id, + max_concurrent, + ) + ), + RpcWorkerRegistration::WildcardLimit + ) { + return ( + Some(RpcClientResponseBody::CodeError { + code: crate::dispatch::protocol::error_codes::rpc::ERR_SUBSCRIPTION_LIMIT, + message: "wildcard subscription limit exceeded (128 per session)" + .to_string(), + }), + Some(false), + false, + ); + } + } + self.dispatch_queued_requests_for_family(*worker_addr.family()); + tracing::debug!( + domain = "rpc", + worker = worker_addr.route().as_str(), + session = meta.session_id, + "Worker registered" + ); + self.refresh_metrics_gauges(); + ( + Some(RpcClientResponseBody::Ok { data: vec![] }), + Some(true), + false, + ) + } + + #[allow(clippy::needless_pass_by_value)] + pub(super) fn handle_unregister_worker_message( + &self, + meta: &crate::runtime::ClientFrameMeta, + worker_addr: crate::runtime::routing::RouteAddress, + ) -> DeliveryOutcome { + let cleanup_result = self.apply_worker_unsubscribe(&worker_addr, meta.session_id); + self.forward_worker_disconnect_errors(cleanup_result.disconnect_deliveries); + tracing::debug!( + domain = "rpc", + worker = worker_addr.route().as_str(), + session = meta.session_id, + removed_workers = cleanup_result.removed_registrations, + removed_pending = cleanup_result.removed_pending, + "Worker unregistered" + ); + ( + Some(RpcClientResponseBody::Ok { data: vec![] }), + Some(true), + false, + ) + } +} diff --git a/src/domains/rpc/sink/response_forwarder.rs b/src/domains/rpc/sink/response_forwarder.rs index 373b7c98..d273ca2a 100644 --- a/src/domains/rpc/sink/response_forwarder.rs +++ b/src/domains/rpc/sink/response_forwarder.rs @@ -27,7 +27,7 @@ impl RpcResponseForwarder { ); let context = FrameContext::new( caller.caller_session_id, - super::mailbox_adapter::test_protocol_channel_from_client(meta.channel), + crate::protocol::test_support::channel_id_from_client(meta.channel), crate::dispatch::protocol::tlv::MessageType::new(RPC_MSG_TYPE_RESPONSE), bytes::Bytes::from(response_bytes), *caller_inbox_addr.family(), diff --git a/src/domains/rpc/sink/response_sink_impl.rs b/src/domains/rpc/sink/responses.rs similarity index 62% rename from src/domains/rpc/sink/response_sink_impl.rs rename to src/domains/rpc/sink/responses.rs index 36d11393..57b369f9 100644 --- a/src/domains/rpc/sink/response_sink_impl.rs +++ b/src/domains/rpc/sink/responses.rs @@ -3,7 +3,7 @@ use super::state_model::{ session_inbox_address, Envelope, Instant, RpcDeliveryOutcome as DeliveryOutcome, RpcDomainRuntime, RpcPendingDispatchInfo, RpcPendingErrorDelivery, RpcPendingResponseDisposition, RpcResponseState, RpcState, RPC_CORRELATION_NOT_FOUND_ERROR, - RPC_INVALID_SEQUENCE_ERROR, RPC_WRONG_WORKER_ERROR, + RPC_INVALID_SEQUENCE_ERROR, RPC_RESPONSE_UNDELIVERABLE_ERROR, RPC_WRONG_WORKER_ERROR, }; use crate::domains::rpc::protocol::RpcResponse; @@ -24,6 +24,19 @@ fn elapsed_micros_optional(start: Option) -> u64 { start.map_or(0, elapsed_micros_u64) } +/// Delivery attempts for one response chunk before the RPC is ended. +/// +/// RPC RESPONSE has no acknowledgement from the broker back to the worker, so +/// a worker can never learn that a chunk failed to reach the caller and +/// cannot resend it - every supported SDK simply advances to the next chunk, +/// or closes after the terminal one. A retry budget greater than one attempt +/// therefore waits for a resend that will never come: a nonterminal chunk's +/// "retry" silently becomes an invalid-sequence error on the NEXT chunk the +/// worker sends, and a terminal chunk's "retry" just sits pending until the +/// caller's own timeout. The broker must end the RPC on the first failed +/// forward instead. +pub(in crate::domains::rpc::sink) const MAX_RESPONSE_DELIVERY_ATTEMPTS: u32 = 1; + impl RpcDomainRuntime<'_> { pub(super) fn handle_response_message( &self, @@ -63,8 +76,8 @@ impl RpcDomainRuntime<'_> { } => self.handle_wrong_response_worker(context, resp, owner_worker_session_id), RpcPendingResponseDisposition::Forward { pending: caller_info, - removed_pending, - } => self.handle_forwarded_response(context, resp, &caller_info, removed_pending), + stream_end, + } => self.handle_forwarded_response(context, resp, &caller_info, stream_end), RpcPendingResponseDisposition::InvalidSequence { pending: caller_info, expected_seq, @@ -129,19 +142,39 @@ impl RpcDomainRuntime<'_> { context: ResponseStateContext<'_>, resp: &RpcResponse, caller_info: &RpcPendingDispatchInfo, - removed_pending: bool, + stream_end: bool, ) -> DeliveryOutcome { let ResponseStateContext { - envelope: _, + envelope, meta, - mut state, + state, state_wait_us, state_hold_start, pending_route_lookup_us, } = context; + let state_hold_us = elapsed_micros_optional(state_hold_start); + drop(state); + + self.histogram_observe_us("rpc_pending_route_lookup_us", pending_route_lookup_us); + self.histogram_observe_us("rpc_response_state_wait_us", state_wait_us); + self.histogram_observe_us("rpc_response_state_hold_us", state_hold_us); + + // Forward before touching the request's cursor. A stream whose cursor + // moved past a chunk the caller never received would present every + // later chunk as contiguous. + if !self.forward_response_to_requester(meta, resp, caller_info) { + return self.handle_undeliverable_response(envelope, meta, resp, caller_info); + } + + let mut state = self.core.state.lock(); + let completed = RpcResponseState::commit_response_delivery( + &mut *state, + meta.route_family, + &resp.correlation_id, + stream_end, + ); let mut state_changed = false; - if removed_pending { - self.release_global_pending(1); + if stream_end && completed { let completion_latency_us = elapsed_micros_u64(caller_info.submitted_at_instant); RpcResponseState::release_dispatch( &mut *state, @@ -149,25 +182,20 @@ impl RpcDomainRuntime<'_> { Some(completion_latency_us), ); let live_request_count = RpcResponseState::live_count(&*state); + drop(state); + self.release_global_pending(1); self.histogram_observe_us("rpc_pending_route_remove_us", pending_route_lookup_us); self.histogram_observe_us("rpc_pending_untrack_us", pending_route_lookup_us); self.gauge_set("rpc_pending_requests", live_request_count as u64); state_changed = true; + } else { + drop(state); } - let state_hold_us = elapsed_micros_optional(state_hold_start); - drop(state); - - self.histogram_observe_us("rpc_pending_route_lookup_us", pending_route_lookup_us); - self.histogram_observe_us("rpc_response_state_wait_us", state_wait_us); - self.histogram_observe_us("rpc_response_state_hold_us", state_hold_us); - - self.forward_response_to_requester(meta, resp, caller_info); - tracing::debug!( domain = "rpc", correlation_id = %resp.correlation_id, - stream_end = resp.stream_end, + stream_end, "Response forwarded to requester" ); @@ -184,15 +212,61 @@ impl RpcDomainRuntime<'_> { (None, state_changed.then_some(false), false) } + /// Handle a chunk the caller could not receive. + /// + /// Ends the RPC immediately: RPC RESPONSE has no ACK, so the worker that + /// sent this chunk has no way to learn delivery failed and will never + /// resend it. Waiting would only delay a failure the caller is going to + /// see either way, while leaving the worker producing into a stream that + /// no longer has a live listener. + fn handle_undeliverable_response( + &self, + envelope: &Envelope, + meta: &crate::runtime::ClientFrameMeta, + resp: &RpcResponse, + caller_info: &RpcPendingDispatchInfo, + ) -> DeliveryOutcome { + let failures = { + let mut state = self.core.state.lock(); + RpcResponseState::record_delivery_failure( + &mut *state, + meta.route_family, + &resp.correlation_id, + ) + }; + self.counter_inc("rpc_response_delivery_retries_total"); + if failures < MAX_RESPONSE_DELIVERY_ATTEMPTS { + tracing::debug!( + domain = "rpc", + correlation_id = %resp.correlation_id, + seq = resp.seq, + failures, + "Caller outbound saturated; chunk stays retryable at its sequence" + ); + return (None, None, false); + } + + let worker_inbox_addr = envelope.source().cloned().unwrap_or_else(|| { + session_inbox_address(*envelope.destination().family(), meta.session_id) + }); + self.terminate_undeliverable_stream(meta, resp, caller_info, worker_inbox_addr); + (None, Some(false), true) + } + + /// Forward one response chunk, reporting whether the caller received it. + /// + /// The boolean matters: a chunk the caller never got leaves a hole in the + /// stream, and every later chunk would arrive looking contiguous. Callers + /// of this method must end the RPC rather than continue past a `false`. fn forward_response_to_requester( &self, meta: &crate::runtime::ClientFrameMeta, resp: &RpcResponse, caller_info: &RpcPendingDispatchInfo, - ) { + ) -> bool { let metrics_enabled = self.metrics.is_some(); let response_forward_start = metrics_enabled.then(Instant::now); - if let Some(forward_envelope) = + let delivered = if let Some(forward_envelope) = RpcResponseForwarder::response_envelope(meta, resp, caller_info) { if let Err(error) = self.router.route(forward_envelope) { @@ -200,16 +274,107 @@ impl RpcDomainRuntime<'_> { tracing::warn!( domain = "rpc", correlation_id = %resp.correlation_id, + seq = resp.seq, error = ?error, "Failed to forward response to requester" ); + false + } else { + true } } else { self.counter_inc("rpc_responses_dropped_closed_caller_total"); - } + false + }; if let Some(response_forward_start) = response_forward_start { self.histogram_observe_elapsed_us("rpc_response_forward_us", response_forward_start); } + delivered + } + + /// End an RPC whose response chunk could not reach the caller. + /// + /// Backpressure may reject or terminate an RPC, but it must never drop a + /// chunk and keep forwarding later ones. Both sides are told: the caller + /// so it sees a terminated stream instead of a sequence gap, and the + /// worker so it stops producing into a stream that no longer exists. + fn terminate_undeliverable_stream( + &self, + meta: &crate::runtime::ClientFrameMeta, + resp: &RpcResponse, + caller_info: &RpcPendingDispatchInfo, + worker_inbox_addr: crate::runtime::routing::RouteAddress, + ) { + { + let mut state = self.core.state.lock(); + if RpcResponseState::abandon_pending( + &mut *state, + meta.route_family, + &resp.correlation_id, + ) + .is_none() + { + // Another path already ended this request. + return; + } + RpcResponseState::release_dispatch(&mut *state, caller_info, None); + let live_request_count = RpcResponseState::live_count(&*state); + self.gauge_set("rpc_pending_requests", live_request_count as u64); + } + self.release_global_pending(1); + self.counter_inc("rpc_streams_terminated_undeliverable_total"); + self.counter_inc("rpc_cleanup_pending_removed_total"); + self.schedule_admin_snapshot(false); + self.dispatch_queued_requests_for_family( + caller_info + .caller_inbox_addr + .as_ref() + .map_or(caller_info.family, |addr| *addr.family()), + ); + + // This path is only reached after the worker has already produced at + // least one response chunk, so the call may have partially executed. + // `ERR_RPC_BACKPRESSURE` is documented and spec-classified (REQ-PROTO-012) + // as safe to retry - it means "never accepted" - which is not true + // here. Use the domain's indeterminate/backend-error code instead, + // matching the `indeterminate_error_code` convention used elsewhere + // for "outcome unknown, do not blindly retry" so a client cannot + // safely re-invoke a non-idempotent call whose side effects may have + // already run. + if let Some(caller_inbox_addr) = caller_info.caller_inbox_addr.clone() { + self.forward_pending_error_deliveries( + vec![RpcPendingErrorDelivery { + correlation_id: resp.correlation_id, + caller_session_id: caller_info.caller_session_id, + caller_inbox_addr, + }], + crate::dispatch::protocol::error_codes::rpc::ERR_BACKEND_ERROR, + RPC_RESPONSE_UNDELIVERABLE_ERROR, + "rpc_undeliverable_stream_errors_forwarded_total", + "rpc_undeliverable_stream_errors_dropped_total", + ); + } else { + self.counter_inc("rpc_undeliverable_stream_errors_dropped_total"); + } + + self.forward_pending_error_deliveries( + vec![RpcPendingErrorDelivery { + correlation_id: resp.correlation_id, + caller_session_id: meta.session_id, + caller_inbox_addr: worker_inbox_addr, + }], + crate::dispatch::protocol::error_codes::rpc::ERR_BACKEND_ERROR, + RPC_RESPONSE_UNDELIVERABLE_ERROR, + "rpc_worker_stream_cancels_forwarded_total", + "rpc_worker_stream_cancels_dropped_total", + ); + + tracing::warn!( + domain = "rpc", + correlation_id = %resp.correlation_id, + seq = resp.seq, + "Terminated RPC stream: response chunk could not be delivered to the caller" + ); } fn handle_invalid_response_sequence( diff --git a/src/domains/rpc/sink/state_model/constants.rs b/src/domains/rpc/sink/state_model/constants.rs index decddbb9..e7861a01 100644 --- a/src/domains/rpc/sink/state_model/constants.rs +++ b/src/domains/rpc/sink/state_model/constants.rs @@ -1,3 +1,12 @@ +/// How long a client-path RPC delivery waits for the actor's reply. +/// +/// Named here rather than inlined so the queue/stream/rpc budgets are visible +/// together; see `QUEUE_ACTOR_REPLY_TIMEOUT`. Note this sits far below midge's +/// own runtime response deadline, so it will always fire first - which is safe +/// only because a timeout is now a retryable error, not a session close. +pub(in crate::domains::rpc::sink) const RPC_ACTOR_REPLY_TIMEOUT: std::time::Duration = + std::time::Duration::from_secs(1); + use super::Duration; pub(in crate::domains::rpc::sink) const RPC_MSG_TYPE_REQUEST: u16 = 302; @@ -18,6 +27,8 @@ pub(in crate::domains::rpc::sink) const RPC_WRONG_WORKER_ERROR: &str = "Worker does not own this correlation ID"; pub(in crate::domains::rpc::sink) const RPC_INVALID_SEQUENCE_ERROR: &str = "RPC response sequence must start at seq=0 and advance contiguously"; +pub(in crate::domains::rpc::sink) const RPC_RESPONSE_UNDELIVERABLE_ERROR: &str = + "RPC stream terminated: a response chunk could not be delivered to the caller"; pub(in crate::domains::rpc::sink) const RPC_TIMEOUT_ERROR: &str = "Worker did not reply within timeout period"; // RPC remains broker-local and in-memory. Worker registrations and pending diff --git a/src/domains/rpc/sink/state_model/mod.rs b/src/domains/rpc/sink/state_model/mod.rs index 0e2ce5a9..02d2a5b3 100644 --- a/src/domains/rpc/sink/state_model/mod.rs +++ b/src/domains/rpc/sink/state_model/mod.rs @@ -38,11 +38,13 @@ mod worker; #[cfg(test)] pub(super) use constants::RPC_MSG_TYPE_RESPONSE; pub(super) use constants::{ - RPC_ADMIN_SNAPSHOT_INTERVAL_US, RPC_BACKPRESSURE_ERROR, RPC_CORRELATION_NOT_FOUND_ERROR, - RPC_DEFAULT_REQUEST_TIMEOUT, RPC_DEFAULT_ROUTE_PENDING_CAPACITY, - RPC_DUPLICATE_CORRELATION_ERROR, RPC_INVALID_SEQUENCE_ERROR, RPC_MAX_PENDING_REQUESTS, - RPC_MAX_TIMEOUT_SWEEP_INTERVAL, RPC_MIN_TIMEOUT_SWEEP_INTERVAL, RPC_MSG_TYPE_REQUEST, - RPC_NO_WORKERS_ERROR, RPC_TIMEOUT_ERROR, RPC_WORKER_NOT_FOUND_ERROR, RPC_WRONG_WORKER_ERROR, + RPC_ACTOR_REPLY_TIMEOUT, RPC_ADMIN_SNAPSHOT_INTERVAL_US, RPC_BACKPRESSURE_ERROR, + RPC_CORRELATION_NOT_FOUND_ERROR, RPC_DEFAULT_REQUEST_TIMEOUT, + RPC_DEFAULT_ROUTE_PENDING_CAPACITY, RPC_DUPLICATE_CORRELATION_ERROR, + RPC_INVALID_SEQUENCE_ERROR, RPC_MAX_PENDING_REQUESTS, RPC_MAX_TIMEOUT_SWEEP_INTERVAL, + RPC_MIN_TIMEOUT_SWEEP_INTERVAL, RPC_MSG_TYPE_REQUEST, RPC_NO_WORKERS_ERROR, + RPC_RESPONSE_UNDELIVERABLE_ERROR, RPC_TIMEOUT_ERROR, RPC_WORKER_NOT_FOUND_ERROR, + RPC_WRONG_WORKER_ERROR, }; pub(super) use expiration::{rpc_timeout_sweep_interval, ExpiringPendingRequest}; pub(super) use pending_table::{RpcPendingResponseDisposition, RpcPendingTable}; diff --git a/src/domains/rpc/sink/state_model/pending_table.rs b/src/domains/rpc/sink/state_model/pending_table.rs index 67adff94..44947857 100644 --- a/src/domains/rpc/sink/state_model/pending_table.rs +++ b/src/domains/rpc/sink/state_model/pending_table.rs @@ -18,7 +18,7 @@ pub(in crate::domains::rpc::sink) enum RpcPendingResponseDisposition { }, Forward { pending: RpcPendingDispatchInfo, - removed_pending: bool, + stream_end: bool, }, InvalidSequence { pending: RpcPendingDispatchInfo, @@ -115,19 +115,20 @@ impl RpcPendingTable { }; } + // Deliberately does not advance the sequence or drop the request here. + // Delivery can still fail, and a stream whose cursor moved past a + // chunk the caller never received presents later chunks as + // contiguous. `commit_response_delivery` runs once the chunk is + // actually handed over. if stream_end { - let pending = self - .remove(&key) - .expect("RPC pending request checked above") - .into_dispatch_info(); return RpcPendingResponseDisposition::Forward { - pending, - removed_pending: true, + pending: pending.dispatch_info(), + stream_end: true, }; } let tracked = pending.dispatch_info(); - let Some(next_expected_seq) = pending.next_expected_seq.checked_add(1) else { + if pending.next_expected_seq.checked_add(1).is_none() { let pending = self .remove(&key) .expect("RPC pending request checked above") @@ -136,12 +137,56 @@ impl RpcPendingTable { pending, expected_seq, }; - }; - pending.next_expected_seq = next_expected_seq; + } RpcPendingResponseDisposition::Forward { pending: tracked, - removed_pending: false, + stream_end: false, + } + } + + /// Advance past a chunk the caller has actually received. + /// + /// Returns `false` when the request is already gone. A `stream_end` chunk + /// completes the request and drops it. + pub(in crate::domains::rpc::sink) fn commit_response_delivery( + &mut self, + family: RouteFamily, + correlation_id: &uuid::Uuid, + stream_end: bool, + ) -> bool { + let key = RpcCorrelationKey { + family, + correlation_id: *correlation_id, + }; + if stream_end { + return self.remove(&key).is_some(); } + let Some(pending) = self.pending.get_mut(&key) else { + return false; + }; + pending.next_expected_seq = pending.next_expected_seq.saturating_add(1); + pending.delivery_retries = 0; + true + } + + /// Record that a chunk could not be handed to the caller, returning how + /// many consecutive delivery failures this request has now seen. + /// + /// The sequence stays put, so the worker may resend the same chunk. + pub(in crate::domains::rpc::sink) fn record_delivery_failure( + &mut self, + family: RouteFamily, + correlation_id: &uuid::Uuid, + ) -> u32 { + let key = RpcCorrelationKey { + family, + correlation_id: *correlation_id, + }; + let Some(pending) = self.pending.get_mut(&key) else { + return u32::MAX; + }; + pending.delivery_retries = pending.delivery_retries.saturating_add(1); + pending.delivery_retries } pub(in crate::domains::rpc::sink) fn contains_correlation_in_family( diff --git a/src/domains/rpc/sink/state_model/requests.rs b/src/domains/rpc/sink/state_model/requests.rs index 3623902b..c2c418b9 100644 --- a/src/domains/rpc/sink/state_model/requests.rs +++ b/src/domains/rpc/sink/state_model/requests.rs @@ -7,6 +7,8 @@ pub(in crate::domains::rpc::sink) struct RpcPendingRequest { pub(in crate::domains::rpc::sink) worker_addr: RouteAddress, pub(in crate::domains::rpc::sink) worker_session_id: u64, pub(in crate::domains::rpc::sink) next_expected_seq: u64, + /// Consecutive failures delivering the current chunk to the caller. + pub(in crate::domains::rpc::sink) delivery_retries: u32, pub(in crate::domains::rpc::sink) submitted_at: DateTime, pub(in crate::domains::rpc::sink) expires_at: Instant, } @@ -60,6 +62,7 @@ impl RpcPendingRequest { worker_addr: registration_addr, worker_session_id: registration_session_id, next_expected_seq: 0, + delivery_retries: 0, submitted_at, expires_at, } diff --git a/src/domains/rpc/sink/state_model/sink.rs b/src/domains/rpc/sink/state_model/sink.rs index 728605fb..59f9e044 100644 --- a/src/domains/rpc/sink/state_model/sink.rs +++ b/src/domains/rpc/sink/state_model/sink.rs @@ -4,9 +4,15 @@ use super::{ }; #[cfg(test)] use super::{RouteAddress, RpcSessionCleanupResult, RpcWorkerCleanupResult}; +use crate::runtime::CleanedUpSessions; pub(in crate::domains::rpc::sink) struct RpcDomainCore { pub(in crate::domains::rpc::sink) state: Mutex, + /// Sessions disconnect cleanup has already run for in this family core; + /// guards against a stale queued request recreating state. See + /// `sink/cleanup.rs`. Local per family core, like `state` - a session + /// cleaned up in one family does not need to be rejected in another. + pub(in crate::domains::rpc::sink) cleaned_up_sessions: Mutex, pub(in crate::domains::rpc::sink) router: Arc, #[cfg_attr(feature = "bench-no-snapshot", allow(dead_code))] pub(in crate::domains::rpc::sink) admin_read_model: diff --git a/src/domains/rpc/sink/state_model/state.rs b/src/domains/rpc/sink/state_model/state.rs index fa0f3adc..ea4f0c4d 100644 --- a/src/domains/rpc/sink/state_model/state.rs +++ b/src/domains/rpc/sink/state_model/state.rs @@ -66,6 +66,29 @@ pub(in crate::domains::rpc::sink) trait RpcResponseState { fn live_count(&self) -> usize; fn release_dispatch(&mut self, pending: &RpcPendingDispatchInfo, latency_us: Option); + + /// Advance past a chunk the caller has actually received; a `stream_end` + /// chunk completes and drops the request. + fn commit_response_delivery( + &mut self, + family: RouteFamily, + correlation_id: &uuid::Uuid, + stream_end: bool, + ) -> bool; + + /// Record a failed delivery attempt, returning the consecutive count. + fn record_delivery_failure(&mut self, family: RouteFamily, correlation_id: &uuid::Uuid) -> u32; + + /// Drop a live pending request without delivering anything further on it. + /// + /// Used when a response chunk could not be handed to the caller: the + /// stream has a hole, so it must end rather than continue with later + /// chunks that would silently present as contiguous. + fn abandon_pending( + &mut self, + family: RouteFamily, + correlation_id: &uuid::Uuid, + ) -> Option; } impl RpcDispatchState for RpcState { @@ -130,6 +153,33 @@ impl RpcResponseState for RpcState { fn release_dispatch(&mut self, pending: &RpcPendingDispatchInfo, latency_us: Option) { self.release_registration_for_dispatch_info(pending, latency_us); } + + fn commit_response_delivery( + &mut self, + family: RouteFamily, + correlation_id: &uuid::Uuid, + stream_end: bool, + ) -> bool { + self.pending + .commit_response_delivery(family, correlation_id, stream_end) + } + + fn record_delivery_failure(&mut self, family: RouteFamily, correlation_id: &uuid::Uuid) -> u32 { + self.pending.record_delivery_failure(family, correlation_id) + } + + fn abandon_pending( + &mut self, + family: RouteFamily, + correlation_id: &uuid::Uuid, + ) -> Option { + self.pending + .remove(&super::RpcCorrelationKey { + family, + correlation_id: *correlation_id, + }) + .map(super::RpcPendingRequest::into_dispatch_info) + } } impl RpcState { diff --git a/src/domains/rpc/sink/tests/cleanup_and_worker_errors.rs b/src/domains/rpc/sink/tests/cleanup_and_worker_errors.rs index 8cb65982..9ba0acac 100644 --- a/src/domains/rpc/sink/tests/cleanup_and_worker_errors.rs +++ b/src/domains/rpc/sink/tests/cleanup_and_worker_errors.rs @@ -705,13 +705,17 @@ fn should_remove_pending_request_on_stream_end_given_rpc_pending_table() { match result { RpcPendingResponseDisposition::Forward { pending: tracked, - removed_pending, + stream_end, } => { assert_eq!(tracked.caller_session_id, 42); assert_eq!(tracked.caller_inbox_addr, Some(caller_inbox_addr)); assert_eq!(&tracked.route, worker_addr.route()); assert_eq!(tracked.registration_id, 0); - assert!(removed_pending); + assert!(stream_end); + // The request is only dropped once the terminal chunk has + // actually reached the caller. + assert_eq!(pending.len(), 1); + assert!(pending.commit_response_delivery(RouteFamily::new(1), &correlation_id, true)); assert_eq!(pending.len(), 0); } other => panic!("expected terminal response handling, found {other:?}"), diff --git a/src/domains/rpc/sink/tests/correctness.rs b/src/domains/rpc/sink/tests/correctness.rs index 8fd770cd..8978a222 100644 --- a/src/domains/rpc/sink/tests/correctness.rs +++ b/src/domains/rpc/sink/tests/correctness.rs @@ -92,3 +92,53 @@ fn should_reject_rpc_worker_registration_when_worker_family_differs_from_request assert_eq!(message, "route family mismatch"); assert_eq!(sink.worker_count(), 0); } + +#[test] +fn should_reject_stale_worker_registration_after_disconnect_cleanup_marks_session() { + // Arrange + let family = RouteFamily::new(1); + let session_id = 9; + let source = session_inbox_address(family, session_id); + let destination = RouteAddress::new(family, Route::new("rpc://inbound")); + let mailbox = Arc::new(Mailbox::new(8)); + let router = Arc::new(Router::new()); + router.register(source.clone(), mailbox.clone()); + let sink = new_correctness_rpc_sink(router); + let worker_addr = RouteAddress::new(family, Route::new("rpc://acme/system/resource/operation")); + let register_request = crate::domains::rpc::RpcClientRequest::new( + crate::runtime::ClientFrameMeta::new( + session_id, + crate::runtime::ClientChannel::Rpc, + 300, + family, + ), + Ok(crate::domains::rpc::RpcMessage::RegisterWorker { + worker_addr: worker_addr.clone(), + max_concurrent: 1, + }), + ); + + // Act: run disconnect cleanup for this session before the stale + // RegisterWorker below is processed, equivalent to what the + // high-priority mailbox lane guarantees a real disconnect races against + // a queued normal-lane request. + sink.deliver(Envelope::new( + RouteAddress::new(family, Route::new("rpc://cleanup")), + crate::runtime::SessionCleanup { session_id }, + )) + .expect("deliver session cleanup"); + assert_eq!(sink.worker_count(), 0); + + sink.deliver(Envelope::from_route(source, destination, register_request)) + .expect("deliver stale worker registration"); + + // Assert: the stale registration from the now-cleaned-up session is + // rejected instead of resurrecting a worker registration for it. + let (code, message) = receive_rpc_error(&mailbox, "stale registration rejection response"); + assert_eq!( + code, + crate::dispatch::protocol::error_codes::rpc::ERR_BACKEND_ERROR + ); + assert_eq!(message, "session already closed"); + assert_eq!(sink.worker_count(), 0); +} diff --git a/src/domains/rpc/sink/tests/response_sequence.rs b/src/domains/rpc/sink/tests/response_sequence.rs index 0a3f3dcb..c1e532d0 100644 --- a/src/domains/rpc/sink/tests/response_sequence.rs +++ b/src/domains/rpc/sink/tests/response_sequence.rs @@ -33,22 +33,24 @@ fn should_retain_pending_request_before_stream_end_given_rpc_pending_table() { match result { RpcPendingResponseDisposition::Forward { pending: tracked, - removed_pending, + stream_end, } => { assert_eq!(tracked.caller_session_id, 84); assert_eq!(tracked.caller_inbox_addr, Some(caller_inbox_addr)); assert_eq!(&tracked.route, worker_addr.route()); assert_eq!(tracked.registration_id, 0); - assert!(!removed_pending); + assert!(!stream_end); assert_eq!(pending.len(), 1); - assert_eq!( - pending.pending[&RpcCorrelationKey { - family: RouteFamily::new(1), - correlation_id, - }] - .next_expected_seq, - 1 - ); + let seq_key = RpcCorrelationKey { + family: RouteFamily::new(1), + correlation_id, + }; + // The lookup does not move the cursor: delivery can still fail, + // and advancing first is what lets a dropped chunk pass as + // contiguous. + assert_eq!(pending.pending[&seq_key].next_expected_seq, 0); + assert!(pending.commit_response_delivery(RouteFamily::new(1), &correlation_id, false)); + assert_eq!(pending.pending[&seq_key].next_expected_seq, 1); } other => panic!("expected non-terminal response handling, found {other:?}"), } @@ -321,3 +323,274 @@ fn should_reject_duplicate_worker_response_chunk_given_rpc_sink() { RPC_INVALID_SEQUENCE_ERROR, ); } + +/// Fails the first N deliveries with backpressure, then captures the rest. +struct BackpressuredThenCapturingSink { + failures_remaining: parking_lot::Mutex, + frames: Arc>>, +} + +impl MailboxSink for BackpressuredThenCapturingSink { + fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { + { + let mut remaining = self.failures_remaining.lock(); + if *remaining > 0 { + *remaining -= 1; + return Err(DeliveryError::MailboxFull { + capacity: 1_000, + current_len: 1_000, + }); + } + } + let frame = envelope + .payload::() + .expect("rpc frame payload") + .clone(); + self.frames.lock().push(frame); + Ok(()) + } + + fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.deliver(envelope) + } +} + +/// Deliver one worker response chunk for `request`. +fn deliver_test_rpc_chunk( + sink: &Arc, + request: &crate::domains::rpc::protocol::RpcRequest, + family: RouteFamily, + worker_source: &RouteAddress, + seq: u64, + stream_end: bool, +) -> Result<(), DeliveryError> { + let payload = crate::dispatch::protocol::rpc_codec::encode_response_message( + &crate::domains::rpc::protocol::RpcResponse::chunk( + request.correlation_id, + seq, + bytes::Bytes::from(format!("chunk-{seq}")), + stream_end, + ), + ); + sink.deliver(Envelope::from_route( + worker_source.clone(), + RouteAddress::new(family, request.route.clone()), + FrameContext::new( + 42, + crate::dispatch::protocol::frame::ChannelId::Rpc, + crate::dispatch::protocol::tlv::MessageType::new(303), + bytes::Bytes::from(payload), + family, + ), + )) +} + +/// Build a well-formed RPC request, deliver it, and hand back the parsed form. +fn deliver_test_rpc_request( + sink: &Arc, + family: RouteFamily, + request_route: &Route, + request_source: RouteAddress, +) -> crate::domains::rpc::protocol::RpcRequest { + let request_frame = crate::benchkit::build_rpc_request(request_route.as_str(), b"ping"); + let (request_msg_type, request_payload) = + crate::benchkit::extract_single_tlv_field(&request_frame); + let request_ctx = FrameContext::new( + 1, + crate::dispatch::protocol::frame::ChannelId::Rpc, + crate::dispatch::protocol::tlv::MessageType::new(request_msg_type), + request_payload.clone(), + family, + ); + let request = match crate::dispatch::protocol::rpc_codec::parse_request( + &request_ctx, + &request_payload, + family, + ) + .expect("parse rpc request") + { + crate::domains::rpc::protocol::RpcMessage::Request(request) => request, + other => panic!("expected rpc request, found {other:?}"), + }; + sink.deliver(Envelope::from_route( + request_source, + RouteAddress::new(family, request_route.clone()), + request_ctx, + )) + .expect("deliver request"); + request +} + +#[test] +fn should_terminate_stream_when_a_response_chunk_cannot_be_delivered() { + // Arrange + // A full outbound channel makes the caller's inbox reject one chunk. The + // broker may reject or terminate the RPC under backpressure, but it must + // never drop a chunk and keep forwarding later ones - that hands the + // caller a silently corrupted stream with a sequence gap. + let router = Arc::new(Router::new()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let metrics = crate::observability::metrics::MetricsCollector::new(); + let sink = Arc::new( + RpcDomainSink::new(router.clone(), admin_read_model).with_metrics(metrics.clone()), + ); + let family = RouteFamily::new(1); + let request_route = Route::new("rpc://bench/system/resource/stream"); + let request_addr = RouteAddress::new(family, request_route.clone()); + let request_source = session_inbox_address(family, 1); + let worker_source = session_inbox_address(family, 42); + let reply_frames = Arc::new(parking_lot::Mutex::new(Vec::::new())); + router.register( + request_source.clone(), + Arc::new(BackpressuredThenCapturingSink { + failures_remaining: parking_lot::Mutex::new(1), + frames: reply_frames.clone(), + }) as Arc, + ); + let worker_frames = Arc::new(parking_lot::Mutex::new(Vec::::new())); + router.register( + worker_source.clone(), + Arc::new(CaptureRpcFrameSink { + frames: worker_frames.clone(), + }) as Arc, + ); + sink.register_registration_for_tests(test_rpc_worker(family, &request_route, 42)); + let request = deliver_test_rpc_request(&sink, family, &request_route, request_source); + + let deliver_chunk = |seq: u64, body: &'static [u8], stream_end: bool| { + let payload = crate::dispatch::protocol::rpc_codec::encode_response_message( + &crate::domains::rpc::protocol::RpcResponse::chunk( + request.correlation_id, + seq, + bytes::Bytes::from_static(body), + stream_end, + ), + ); + sink.deliver(Envelope::from_route( + worker_source.clone(), + request_addr.clone(), + FrameContext::new( + 42, + crate::dispatch::protocol::frame::ChannelId::Rpc, + crate::dispatch::protocol::tlv::MessageType::new(303), + bytes::Bytes::from(payload), + family, + ), + )) + }; + + // Act + // RPC RESPONSE has no ACK: a real worker never learns a chunk failed to + // reach the caller, so it cannot resend it - it just moves on to the next + // chunk (or closes, for a single-chunk response). The broker must + // therefore terminate on the FIRST undeliverable chunk rather than wait + // for a resend that will never come. + deliver_chunk(0, b"chunk-zero", false).expect("deliver chunk 0"); + deliver_chunk(1, b"chunk-one", false).expect("deliver chunk 1"); + + // Assert + assert_eq!( + sink.pending_request_count(), + 0, + "an undeliverable chunk must terminate the RPC, not leave the stream live" + ); + let frames = reply_frames.lock(); + let forwarded = frames + .iter() + .map(parse_forwarded_rpc_response) + .collect::>(); + assert!( + !forwarded + .iter() + .any(|response| response.seq == 1 && !response.stream_end), + "chunk 1 must not be forwarded as a normal chunk after chunk 0 was dropped: {forwarded:?}" + ); + assert!( + forwarded.iter().any(|response| response.stream_end), + "the caller must be told the stream terminated, got {forwarded:?}" + ); + + // The worker must also be cancelled, or it keeps producing into a stream + // that no longer exists - the amplification that turned two corrupted + // streams into tens of thousands of late dropped responses. + let worker_frames = worker_frames.lock(); + let worker_cancels = worker_frames + .iter() + .filter(|frame| frame.msg_type.as_u16() == 303) + .map(parse_forwarded_rpc_response) + .filter(|response| response.correlation_id == request.correlation_id && response.stream_end) + .count(); + assert!( + worker_cancels >= 1, + "the worker must be told to stop producing, got {worker_frames:?}" + ); + + // Health must not read green through a corrupted stream. + assert!( + metrics.counter_get(crate::domains::rpc::metrics::METRIC_FAILURE_TOTAL) >= 1, + "a terminated stream must be counted as a failure, not silent success" + ); + assert!( + metrics.counter_get("rpc_response_delivery_retries_total") >= 1, + "the failed delivery attempt must be observable" + ); +} + +#[test] +fn should_terminate_immediately_on_the_first_undeliverable_terminal_chunk() { + // Arrange + // RPC RESPONSE has no ACK. A terminal (stream_end) chunk that cannot be + // forwarded must not wait around for a resend that never arrives - every + // supported SDK considers the call finished once it has sent the last + // chunk, so a request stuck waiting for a nonexistent retry would sit + // pending until the caller-side timeout instead of failing promptly. + let router = Arc::new(Router::new()); + let admin_read_model = crate::control::admin::read_model::AdminReadModel::new(); + let sink = Arc::new(RpcDomainSink::new(router.clone(), admin_read_model)); + let family = RouteFamily::new(1); + let request_route = Route::new("rpc://bench/system/resource/terminal-undeliverable"); + let request_source = session_inbox_address(family, 1); + let worker_source = session_inbox_address(family, 42); + let reply_frames = Arc::new(parking_lot::Mutex::new(Vec::::new())); + router.register( + request_source.clone(), + Arc::new(BackpressuredThenCapturingSink { + // Never drains: nothing ever resends, so this must never succeed. + failures_remaining: parking_lot::Mutex::new(usize::MAX), + frames: reply_frames.clone(), + }) as Arc, + ); + let worker_frames = Arc::new(parking_lot::Mutex::new(Vec::::new())); + router.register( + worker_source.clone(), + Arc::new(CaptureRpcFrameSink { + frames: worker_frames.clone(), + }) as Arc, + ); + sink.register_registration_for_tests(test_rpc_worker(family, &request_route, 42)); + let request = deliver_test_rpc_request(&sink, family, &request_route, request_source); + + // Act + // A single terminal chunk, exactly as a real worker sends one and then + // considers the call done - no resend follows. + deliver_test_rpc_chunk(&sink, &request, family, &worker_source, 0, true) + .expect("deliver terminal chunk"); + + // Assert + assert_eq!( + sink.pending_request_count(), + 0, + "an undeliverable terminal chunk must not stay pending waiting for a resend" + ); + let worker_frames = worker_frames.lock(); + let worker_cancels = worker_frames + .iter() + .filter(|frame| frame.msg_type.as_u16() == 303) + .map(parse_forwarded_rpc_response) + .filter(|response| response.correlation_id == request.correlation_id && response.stream_end) + .count(); + assert!( + worker_cancels >= 1, + "the worker must be told the request ended, got {worker_frames:?}" + ); +} diff --git a/src/domains/rpc/sink/tests/state_metrics_and_timeouts.rs b/src/domains/rpc/sink/tests/state_metrics_and_timeouts.rs index b07beff3..3b8b16f4 100644 --- a/src/domains/rpc/sink/tests/state_metrics_and_timeouts.rs +++ b/src/domains/rpc/sink/tests/state_metrics_and_timeouts.rs @@ -375,18 +375,35 @@ pub(super) fn should_route_rpc_worker_unsubscribe_helper_through_managed_actor() } #[test] -pub(super) fn should_keep_rpc_mailbox_sink_impl_below_file_size_limit() { - // Arrange - let line_count = include_str!("../mailbox_sink_impl.rs").lines().count(); - - // Act - let within_limit = line_count < 1_000; - - // Assert - assert!( - within_limit, - "rpc mailbox sink impl has {line_count} lines; split before adding behavior" - ); +pub(super) fn should_keep_rpc_sink_files_below_size_limit() { + // Arrange: one entry per top-level sink/*.rs file. New files must be + // added here explicitly rather than falling back to a directory scan, so + // an oversized new file fails loudly instead of silently passing. + let files: &[(&str, &str)] = &[ + ("cleanup.rs", include_str!("../cleanup.rs")), + ("delivery.rs", include_str!("../delivery.rs")), + ("facade.rs", include_str!("../facade.rs")), + ("family_runtime.rs", include_str!("../family_runtime.rs")), + ("ingress.rs", include_str!("../ingress.rs")), + ("mailbox.rs", include_str!("../mailbox.rs")), + ("mailbox_adapter.rs", include_str!("../mailbox_adapter.rs")), + ("observability.rs", include_str!("../observability.rs")), + ("registration.rs", include_str!("../registration.rs")), + ( + "response_forwarder.rs", + include_str!("../response_forwarder.rs"), + ), + ("responses.rs", include_str!("../responses.rs")), + ]; + + // Act / Assert + for (name, contents) in files { + let line_count = contents.lines().count(); + assert!( + line_count < 1_000, + "rpc sink/{name} has {line_count} lines; split before adding behavior" + ); + } } #[test] diff --git a/src/domains/schedule/actor/claim_and_ack.rs b/src/domains/schedule/actor/claim_and_ack.rs index 6ae47d31..b82ac6cf 100644 --- a/src/domains/schedule/actor/claim_and_ack.rs +++ b/src/domains/schedule/actor/claim_and_ack.rs @@ -22,12 +22,82 @@ fn acknowledge_claim_batch( persistence.acknowledge_claims(family_id, claims, write_options) } +/// One page of schedule definitions: entries, whether more remain, and the +/// continuation cursor. +pub(crate) type ScheduleListPage = (Arc>>, bool, Option); + +/// One page of schedule definitions plus the total definition count. +pub(crate) type ScheduleListDefs = (Arc>>, u64); + impl ScheduleActor { + /// Longest run of entries starting at `start` that still fits one wire + /// frame, always at least one so a page makes forward progress. + /// + /// Every entry may be individually small and legal while the aggregate is + /// unencodable; without this the response is built, handed to the outbound + /// sink, and fails the TLV length assertion. + /// # Errors + /// + /// Returns the offending route when a single entry exceeds the ceiling on + /// its own. Creation now refuses such definitions, so this only covers + /// entries stored before that check existed. Forcing one into a page would + /// produce a response that cannot be framed and is silently dropped, so it + /// surfaces as an explicit, classifiable error naming the route instead. + /// + /// `continuation_reserve` charges each entry as if it might end up being + /// the last one on the page, so a caller whose page format re-encodes + /// that boundary entry's route a second time (e.g. `list_entries_v2`'s + /// continuation cursor, which duplicates `family_prefix + route`) does + /// not silently exceed the ceiling it already budgeted against. Callers + /// with no such duplication pass a reserve of zero. + fn bounded_page_len( + entries: &[Arc], + start: usize, + take: usize, + continuation_reserve: impl Fn(&ScheduleListEntry) -> usize, + ) -> Result { + let ceiling = + crate::domains::schedule::list_wire_budget::schedule_list_response_byte_ceiling(); + let mut used = 0usize; + let mut fitted = 0usize; + for entry in entries.iter().skip(start).take(take) { + let cost = + crate::domains::schedule::list_wire_budget::schedule_list_entry_wire_bytes(entry) + .saturating_add(continuation_reserve(entry)); + if cost > ceiling { + if fitted > 0 { + // End the page here; the next page starts at the offending + // entry and reports it. + break; + } + return Err(format!( + "schedule {} is {cost} wire bytes, exceeding the {ceiling}-byte limit a \ + list response can return", + entry.route + )); + } + let next = used.saturating_add(cost); + if next > ceiling && fitted > 0 { + break; + } + used = next; + fitted = fitted.saturating_add(1); + if used > ceiling { + break; + } + } + Ok(fitted.max(1).min(take)) + } + + /// # Errors + /// + /// Returns the offending route when a stored definition is too large to + /// appear in any list response. pub fn list_entries_v2( &mut self, cursor: Option<&str>, limit: u64, - ) -> (Arc>>, bool, Option) { + ) -> Result { const MAX_LIMIT: usize = 1_000; let take = usize::try_from(limit) .unwrap_or(MAX_LIMIT) @@ -39,48 +109,67 @@ impl ScheduleActor { .and_then(|value| value.strip_prefix(&family_prefix)) .and_then(|value| ordered.iter().position(|entry| entry.route == value)) .map_or(0, |index| index.saturating_add(1)); - let end = start.saturating_add(take).min(ordered.len()); - let entries = Arc::new(ordered[start.min(ordered.len())..end].to_vec()); + let start = start.min(ordered.len()); + let requested = start.saturating_add(take).min(ordered.len()) - start; + // Reserve room for the continuation field, which re-encodes + // `family_prefix + route` for whichever entry ends up last on the + // page - on top of that route already being counted once inside the + // entry itself. + let family_prefix_len = family_prefix.len(); + let fitted = if requested == 0 { + 0 + } else { + Self::bounded_page_len(&ordered, start, requested, |entry| { + family_prefix_len.saturating_add(entry.route.len()) + })? + }; + let end = start.saturating_add(fitted); + let entries = Arc::new(ordered[start..end].to_vec()); let has_more = end < ordered.len(); let continuation = has_more.then(|| format!("{family_prefix}{}", ordered[end - 1].route)); - (entries, has_more, continuation) + Ok((entries, has_more, continuation)) } fn u64_to_usize_saturating(value: u64) -> usize { usize::try_from(value).unwrap_or(usize::MAX) } - pub fn list_entries( - &mut self, - offset: u64, - limit: u64, - ) -> (Arc>>, u64) { + /// # Errors + /// + /// Returns the offending route when a stored definition is too large to + /// appear in any list response. + pub fn list_entries(&mut self, offset: u64, limit: u64) -> Result { let total_count = self.schedules.len() as u64; let start = Self::u64_to_usize_saturating(offset); if start >= self.list_entries.len() { - return (Arc::new(Vec::new()), total_count); + return Ok((Arc::new(Vec::new()), total_count)); } let remaining = self.list_entries.len() - start; - let take = if limit == 0 { + let requested = if limit == 0 { remaining } else { remaining.min(Self::u64_to_usize_saturating(limit)) }; + // `limit = 0` means "all remaining", so the caller's count cannot bound + // the response - only the wire budget can. Clients detect the short + // page by comparing entry count against `total_count` and continue + // from `offset`, which the V1 contract already supports. + let take = Self::bounded_page_len(&self.list_entries, start, requested, |_| 0)?; if start == 0 && take == self.list_entries.len() { if let Some(cache) = &self.list_cache { - return (cache.clone(), total_count); + return Ok((cache.clone(), total_count)); } let cache = Arc::new(self.list_entries.clone()); self.list_cache = Some(cache.clone()); - return (cache, total_count); + return Ok((cache, total_count)); } - ( + Ok(( Arc::new(self.list_entries[start..start + take].to_vec()), total_count, - ) + )) } fn store_claims_for<'a>( diff --git a/src/domains/schedule/actor/definitions_and_listing.rs b/src/domains/schedule/actor/definitions_and_listing.rs index 9f1829a3..dfb56f51 100644 --- a/src/domains/schedule/actor/definitions_and_listing.rs +++ b/src/domains/schedule/actor/definitions_and_listing.rs @@ -33,6 +33,11 @@ impl ScheduleActor { now: Instant, ) -> Result { let route_parts = parse_concrete_schedule_route(&route)?; + crate::domains::schedule::list_wire_budget::validate_listable_definition( + &route, + &cron, + payload.len(), + )?; let ( previous_next_fire_ms, previous_list_index, @@ -183,6 +188,11 @@ impl ScheduleActor { entry.route )); } + crate::domains::schedule::list_wire_budget::validate_listable_definition( + &entry.route, + &entry.cron, + entry.payload.len(), + )?; let ( previous_fire_ms, diff --git a/src/domains/schedule/actor/scan_helpers_and_handle.rs b/src/domains/schedule/actor/scan_helpers_and_handle.rs index 75ff6176..bcaae4cc 100644 --- a/src/domains/schedule/actor/scan_helpers_and_handle.rs +++ b/src/domains/schedule/actor/scan_helpers_and_handle.rs @@ -140,20 +140,27 @@ impl ScheduleActor { Err(e) => ScheduleResponse::Error(ScheduleFailure::parse(e)), } } - ScheduleMessage::List { offset, limit } => { - let (entries, total_count) = self.list_entries(offset, limit); - ScheduleResponse::ListDefs { + ScheduleMessage::List { offset, limit } => match self.list_entries(offset, limit) { + Ok((entries, total_count)) => ScheduleResponse::ListDefs { entries, total_count, - } - } + }, + Err(error) => ScheduleResponse::Error(ScheduleFailure::new( + ScheduleFailureCategory::InvalidTarget, + error, + )), + }, ScheduleMessage::ListV2 { cursor, limit } => { - let (entries, has_more, continuation) = - self.list_entries_v2(cursor.as_deref(), limit); - ScheduleResponse::ListPage { - entries, - has_more, - continuation, + match self.list_entries_v2(cursor.as_deref(), limit) { + Ok((entries, has_more, continuation)) => ScheduleResponse::ListPage { + entries, + has_more, + continuation, + }, + Err(error) => ScheduleResponse::Error(ScheduleFailure::new( + ScheduleFailureCategory::InvalidTarget, + error, + )), } } ScheduleMessage::Subscribe { .. } diff --git a/src/domains/schedule/actor/tests.rs b/src/domains/schedule/actor/tests.rs index 880bf26d..b6a79f96 100644 --- a/src/domains/schedule/actor/tests.rs +++ b/src/domains/schedule/actor/tests.rs @@ -454,11 +454,11 @@ fn should_invalidate_shared_full_list_cache_given_schedule_delete() { Bytes::from_static(b"second"), ) .expect("create second schedule"); - let (cached, _) = actor.list_entries(0, 0); + let (cached, _) = actor.list_entries(0, 0).expect("list entries"); // Act actor.delete_schedule(first_route).expect("delete schedule"); - let (refreshed, total_count) = actor.list_entries(0, 0); + let (refreshed, total_count) = actor.list_entries(0, 0).expect("list entries"); // Assert assert_eq!(cached.len(), 2); @@ -939,3 +939,206 @@ fn should_not_remove_schedule_given_cancel_persistence_failure() { "list index must still contain the route on cancel persist failure" ); } + +#[test] +fn should_page_a_byte_bounded_list_to_completion_via_offset() { + // Arrange + // `limit=0` means "all remaining", but the response is still bounded by + // the wire frame and can therefore return fewer entries than exist. The + // wire format carries no `has_more` flag for canonical LIST, only + // `total_count` - so a client must detect truncation by comparing the + // returned entry count to `total_count` and continue with + // `offset += entries.len()`. This proves that procedure actually + // recovers every entry rather than stopping at the first partial page. + let mut actor = make_actor(); + let payload = Bytes::from(vec![b'p'; 1024]); + let total = 300; + for index in 0..total { + actor + .create_schedule( + format!("schedule://acme/jobs/list-page-{index:04}/run"), + "* * * * *".to_string(), + payload.clone(), + ) + .expect("create schedule"); + } + + // Act + let mut seen = std::collections::HashSet::new(); + let mut offset = 0u64; + let mut pages = 0; + loop { + let (entries, total_count) = actor.list_entries(offset, 0).expect("list entries"); + pages += 1; + assert!(pages < 50, "pagination must converge"); + assert!(!entries.is_empty(), "each page must make forward progress"); + for entry in entries.iter() { + seen.insert(entry.route.clone()); + } + offset += u64::try_from(entries.len()).unwrap(); + if offset >= total_count { + break; + } + } + + // Assert + assert!(pages > 1, "1 KiB payloads must not fit in a single page"); + assert_eq!( + seen.len(), + total, + "offset continuation must recover every entry" + ); +} + +#[test] +fn should_bound_list_response_to_one_wire_frame() { + // Arrange + // A schedule LIST response is encoded into a single TLV value, whose + // length prefix is a u16. `limit = 0` means "all remaining", so a few + // hundred ordinary schedules with 1 KiB payloads produce a response far + // past 65_535 bytes - which panicked the outbound sink rather than + // paginating. Every entry here is individually small and legal; only the + // aggregate is oversized. + let mut actor = make_actor(); + let payload = Bytes::from(vec![b'p'; 1024]); + for index in 0..250 { + actor + .create_schedule( + format!("schedule://acme/jobs/bulk-{index:04}/run"), + "* * * * *".to_string(), + payload.clone(), + ) + .expect("create schedule"); + } + + // Act + let (entries, total_count) = actor.list_entries(0, 0).expect("list entries"); + let payload_bytes = crate::dispatch::protocol::schedule_codec::encode_response( + 701, + &crate::domains::schedule::ScheduleResponse::ListDefs { + entries: entries.clone(), + total_count, + }, + ); + + // Assert + assert_eq!(total_count, 250, "every schedule must still be counted"); + assert!( + u16::try_from(payload_bytes.len()).is_ok(), + "list response is {} bytes, past the {}-byte TLV value limit", + payload_bytes.len(), + u16::MAX + ); + assert!( + !entries.is_empty(), + "the page must still make forward progress" + ); + assert!( + entries.len() < 250, + "an oversized listing must be truncated into a page, not returned whole" + ); +} + +#[test] +fn should_keep_list_v2_continuation_inside_one_wire_frame() { + // Arrange + // ListPage's continuation cursor duplicates the last returned route as a + // separate wire field (`family_prefix + route`), on top of that same + // route already being encoded once inside the entry itself. A page filled + // right up to the byte ceiling using only the entry's own cost therefore + // produces a response the continuation field pushes past u16::MAX - a + // legal definition can silently become unencodable purely because of + // where the page boundary happened to land. + let mut actor = make_actor(); + // A payload sized so entries divide the ceiling with a small remainder, + // guaranteeing the last admitted entry sits close enough to the edge that + // only the (missing) continuation reserve decides whether it still fits. + // A long resource name so the duplicated continuation route (family + // prefix + this same route) meaningfully exceeds the fixed envelope + // margin - a short route would fit inside that margin's slack and the + // bug would not reproduce. + let long_resource = "x".repeat(400); + let route_for = |index: usize| format!("schedule://acme/jobs/{long_resource}-{index:04}/run"); + let listable_ceiling = + crate::domains::schedule::list_wire_budget::schedule_list_response_byte_ceiling(); + let probe_route = route_for(0); + let entry_fixed = crate::domains::schedule::list_wire_budget::schedule_list_entry_wire_bytes( + &crate::domains::schedule::ScheduleListEntry { + route: probe_route.clone(), + cron: "* * * * *".to_string(), + delivery_mode: crate::domains::schedule::ScheduleDeliveryMode::Broadcast, + payload: Bytes::new(), + }, + ); + let entries_per_page = 4; + let payload_len = (listable_ceiling / entries_per_page).saturating_sub(entry_fixed); + let payload = Bytes::from(vec![b'p'; payload_len]); + for index in 0..(entries_per_page * 2) { + actor + .create_schedule(route_for(index), "* * * * *".to_string(), payload.clone()) + .expect("create schedule"); + } + + // Act + let (entries, has_more, continuation) = actor + .list_entries_v2(None, u64::try_from(entries_per_page * 2).unwrap()) + .expect("list entries v2"); + + // Assert + let response_bytes = crate::dispatch::protocol::schedule_codec::encode_response( + 720, + &crate::domains::schedule::ScheduleResponse::ListPage { + entries: entries.clone(), + has_more, + continuation, + }, + ); + assert!( + u16::try_from(response_bytes.len()).is_ok(), + "list_v2 page is {} bytes, past the {}-byte TLV value limit", + response_bytes.len(), + u16::MAX + ); +} + +#[test] +fn should_reject_schedule_whose_definition_could_never_be_listed() { + // Arrange + // A CREATE arrives as one TLV value, so a payload can be ~140 bytes larger + // than a LIST entry for the same route can encode. Accepting one leaves a + // definition that is stored but permanently unlistable: the page cannot be + // framed, so the response is dropped rather than answered. + let mut actor = make_actor(); + let route = "schedule://acme/jobs/unlistable/run"; + let cron = "* * * * *"; + let listable_ceiling = + crate::domains::schedule::list_wire_budget::schedule_list_response_byte_ceiling(); + let probe = crate::domains::schedule::ScheduleListEntry { + route: route.to_string(), + cron: cron.to_string(), + delivery_mode: crate::domains::schedule::ScheduleDeliveryMode::Broadcast, + payload: Bytes::new(), + }; + let entry_overhead = + crate::domains::schedule::list_wire_budget::schedule_list_entry_wire_bytes(&probe); + let largest_listable = listable_ceiling - entry_overhead; + + // Act + let accepted = actor.create_schedule( + route.to_string(), + cron.to_string(), + Bytes::from(vec![b'p'; largest_listable]), + ); + let rejected = actor.create_schedule( + "schedule://acme/jobs/unlistable-two/run".to_string(), + cron.to_string(), + Bytes::from(vec![b'p'; largest_listable + 1]), + ); + + // Assert + assert!(accepted.is_ok(), "a listable definition must be accepted"); + assert!( + rejected.is_err(), + "a definition that could never be listed must be refused at create" + ); +} diff --git a/src/domains/schedule/list_wire_budget.rs b/src/domains/schedule/list_wire_budget.rs new file mode 100644 index 00000000..d966c12a --- /dev/null +++ b/src/domains/schedule/list_wire_budget.rs @@ -0,0 +1,67 @@ +//! Wire-size budget for schedule LIST responses. +//! +//! A LIST response is carried as a single TLV value with a `u16` length, so +//! the page must be bounded by bytes and not only by entry count. Every entry +//! can be individually small and legal while the aggregate is unencodable. + +use super::protocol::ScheduleListEntry; + +/// A schedule response is carried as one TLV value, whose length prefix is a +/// `u16`. Anything past this can never be framed. +pub(crate) const MAX_SCHEDULE_RESPONSE_PAYLOAD_BYTES: usize = u16::MAX as usize; + +/// Generous allowance for a list response's non-entry bytes: the success flag, +/// the total count or the version, `has_more` and continuation fields, and the end +/// sentinel. Deliberately over-counted so the ceiling stays safe if the +/// encoder's envelope grows. +const SCHEDULE_LIST_ENVELOPE_OVERHEAD_BYTES: usize = 128; + +/// Generous allowance for one encoded list entry's fixed parts: the has-entry +/// marker, the delivery mode, and the length prefixes on route, cron and +/// payload. +const SCHEDULE_LIST_ENTRY_FIXED_OVERHEAD_BYTES: usize = 32; + +/// Largest entry payload a list response can carry. +#[must_use] +pub(crate) fn schedule_list_response_byte_ceiling() -> usize { + MAX_SCHEDULE_RESPONSE_PAYLOAD_BYTES.saturating_sub(SCHEDULE_LIST_ENVELOPE_OVERHEAD_BYTES) +} + +/// Reject a definition that could never appear in a LIST response. +/// +/// A CREATE arrives as a single TLV value, so its payload may be larger than +/// the same definition costs as a list entry. Storing one leaves a schedule +/// that fires normally but can never be listed: the page cannot be framed, so +/// the response is dropped rather than answered. Refusing it at write time is +/// the only point where the client can still do something about it. +/// +/// # Errors +/// +/// Returns the reason when the encoded entry would exceed the list ceiling. +pub(crate) fn validate_listable_definition( + route: &str, + cron: &str, + payload_len: usize, +) -> Result<(), String> { + let entry_bytes = SCHEDULE_LIST_ENTRY_FIXED_OVERHEAD_BYTES + .saturating_add(route.len()) + .saturating_add(cron.len()) + .saturating_add(payload_len); + let ceiling = schedule_list_response_byte_ceiling(); + if entry_bytes > ceiling { + return Err(format!( + "schedule definition is {entry_bytes} wire bytes, exceeding the {ceiling}-byte \ + limit a list response can return" + )); + } + Ok(()) +} + +/// Conservative wire cost of one encoded `ScheduleListEntry`. +#[must_use] +pub(crate) fn schedule_list_entry_wire_bytes(entry: &ScheduleListEntry) -> usize { + SCHEDULE_LIST_ENTRY_FIXED_OVERHEAD_BYTES + .saturating_add(entry.route.len()) + .saturating_add(entry.cron.len()) + .saturating_add(entry.payload.len()) +} diff --git a/src/domains/schedule/mod.rs b/src/domains/schedule/mod.rs index 1b0ea1eb..52e91186 100644 --- a/src/domains/schedule/mod.rs +++ b/src/domains/schedule/mod.rs @@ -18,6 +18,7 @@ pub mod actor; mod definition_validation; +pub(crate) mod list_wire_budget; pub mod metrics; pub mod protocol; pub mod session; diff --git a/src/domains/schedule/sink/cleanup.rs b/src/domains/schedule/sink/cleanup.rs new file mode 100644 index 00000000..0c0da1f5 --- /dev/null +++ b/src/domains/schedule/sink/cleanup.rs @@ -0,0 +1,61 @@ +//! Disconnect cleanup and stale queued-request rejection state. +//! +//! `SessionCleanup` is delivered on the high-priority mailbox lane, so it can +//! pass an older, already-queued normal-lane request from the same session. +//! Remembering the cleaned-up session lets that stale request fail instead of +//! silently recreating a subscription for a session that is already gone and +//! will never be cleaned up again. + +use super::model::{Envelope, ScheduleDomainRuntime, ScheduleDomainSink}; + +impl ScheduleDomainSink { + /// Remove every Schedule subscription owned by one disconnected session. + /// + /// This crosses the mailbox (high-priority lane); the work itself happens + /// in `ScheduleDomainRuntime::unsubscribe_all`. + pub fn unsubscribe_all(&self, session_id: u64) { + if let Err(error) = + self.actor + .try_send_high_priority(super::model::ScheduleDomainCommand::CleanupSession( + session_id, + )) + { + tracing::warn!(domain = "schedule", error = %error, "Schedule cleanup enqueue failed"); + } + } +} + +impl ScheduleDomainRuntime<'_> { + pub(super) fn is_cleaned_up_session(&self, session_id: u64) -> bool { + self.core.cleaned_up_sessions.lock().contains(session_id) + } + + pub(super) fn handle_cleanup_envelope(&self, envelope: &Envelope) -> bool { + if let Some(cleanup) = envelope.payload::() { + // Mark first so an older normal-lane request that cleanup jumped + // over cannot recreate a subscription for this session below. + self.core + .cleaned_up_sessions + .lock() + .mark(cleanup.session_id); + self.unsubscribe_all(cleanup.session_id); + return true; + } + + false + } + + /// Remove every Schedule subscription owned by one session. + pub(super) fn unsubscribe_all(&self, session_id: u64) { + let mut families = self.core.sub_families.lock(); + for (family, state) in families.iter_mut() { + state.remove_session(*family, session_id); + } + families.retain(|_, state| !state.is_empty()); + tracing::debug!( + domain = "schedule", + session = session_id, + "All schedule subscriptions removed for session" + ); + } +} diff --git a/src/domains/schedule/sink/definitions.rs b/src/domains/schedule/sink/definitions.rs new file mode 100644 index 00000000..c568b7b8 --- /dev/null +++ b/src/domains/schedule/sink/definitions.rs @@ -0,0 +1,259 @@ +//! Schedule definition CRUD: dispatching Create/CreateBatch/Cancel/List +//! messages to the per-family `ScheduleActor`, and hydrating those actors +//! from persisted storage at startup. + +use super::model::{ + duration_millis, now_epoch_ms, Entry, HashMap, ScheduleDomainRuntime, EXECUTIONS_WINDOW_MS, +}; + +impl ScheduleDomainRuntime<'_> { + pub(super) fn apply_schedule_message( + &self, + actor: &mut crate::domains::schedule::ScheduleActor, + schedule_msg: crate::domains::schedule::ScheduleMessage, + ) -> (crate::domains::schedule::ScheduleResponse, bool) { + use crate::domains::schedule::{ScheduleMessage, ScheduleResponse}; + + match schedule_msg { + ScheduleMessage::Create { + route, + cron, + delivery_mode, + payload, + } => Self::apply_create_message(actor, route, cron, delivery_mode, payload), + ScheduleMessage::CreateBatch { entries } => { + Self::apply_create_batch_message(actor, entries) + } + ScheduleMessage::Cancel { route } => Self::apply_cancel_message(actor, &route), + ScheduleMessage::List { offset, limit } => { + let response = match actor.list_entries(offset, limit) { + Ok((entries, total_count)) => ScheduleResponse::ListDefs { + entries, + total_count, + }, + Err(error) => { + ScheduleResponse::Error(crate::domains::schedule::ScheduleFailure::new( + crate::domains::schedule::ScheduleFailureCategory::InvalidTarget, + error, + )) + } + }; + (response, false) + } + ScheduleMessage::ListV2 { cursor, limit } => { + let response = match actor.list_entries_v2(cursor.as_deref(), limit) { + Ok((entries, has_more, continuation)) => ScheduleResponse::ListPage { + entries, + has_more, + continuation, + }, + Err(error) => { + ScheduleResponse::Error(crate::domains::schedule::ScheduleFailure::new( + crate::domains::schedule::ScheduleFailureCategory::InvalidTarget, + error, + )) + } + }; + (response, false) + } + ScheduleMessage::Subscribe { + family_id, + route, + session_id, + subscriber, + } => ( + self.apply_subscribe_message(family_id, &route, session_id, subscriber), + false, + ), + ScheduleMessage::Unsubscribe { + family_id, + route, + session_id, + .. + } => ( + self.apply_unsubscribe_message(family_id, &route, session_id), + false, + ), + ScheduleMessage::UnsubscribeAll { session_id, .. } => { + self.unsubscribe_all(session_id); + (ScheduleResponse::Ok, false) + } + } + } + + fn apply_create_message( + actor: &mut crate::domains::schedule::ScheduleActor, + route: String, + cron: String, + delivery_mode: crate::domains::schedule::ScheduleDeliveryMode, + payload: bytes::Bytes, + ) -> (crate::domains::schedule::ScheduleResponse, bool) { + use crate::domains::schedule::{ScheduleFailure, ScheduleResponse}; + + if let Some(failure) = + crate::domains::schedule::definition_validation::schedule_definition_failure( + &route, &cron, + ) + { + return (ScheduleResponse::Error(failure), false); + } + + match actor.create_schedule_with_mode(route, cron, delivery_mode, payload) { + Ok(changed) => (ScheduleResponse::Ok, changed), + Err(error) => ( + ScheduleResponse::Error(ScheduleFailure::parse(error)), + false, + ), + } + } + + fn apply_create_batch_message( + actor: &mut crate::domains::schedule::ScheduleActor, + entries: Vec, + ) -> (crate::domains::schedule::ScheduleResponse, bool) { + use crate::domains::schedule::{ScheduleFailure, ScheduleResponse}; + + if let Some(failure) = entries.iter().find_map(|entry| { + crate::domains::schedule::definition_validation::schedule_definition_failure( + &entry.route, + &entry.cron, + ) + }) { + return (ScheduleResponse::Error(failure), false); + } + + match actor.create_schedules(entries) { + Ok(changed) => (ScheduleResponse::Ok, changed > 0), + Err(error) => ( + ScheduleResponse::Error(ScheduleFailure::parse(error)), + false, + ), + } + } + + fn apply_cancel_message( + actor: &mut crate::domains::schedule::ScheduleActor, + route: &str, + ) -> (crate::domains::schedule::ScheduleResponse, bool) { + use crate::domains::schedule::{ + ScheduleFailure, ScheduleFailureCategory, ScheduleResponse, + }; + + if let Err(error) = + crate::domains::schedule::protocol::validate_concrete_schedule_route(route) + { + return ( + ScheduleResponse::Error(ScheduleFailure::new( + ScheduleFailureCategory::InvalidTarget, + error, + )), + false, + ); + } + + match actor.delete_schedule(route) { + Ok(removed) => (ScheduleResponse::Ok, removed), + Err(error) => ( + ScheduleResponse::Error(ScheduleFailure::parse(error)), + false, + ), + } + } + + pub(super) fn get_or_create_actor<'a>( + &'a self, + actors: &'a mut HashMap< + crate::runtime::routing::RouteFamily, + crate::domains::schedule::ScheduleActor, + >, + route_family: crate::runtime::routing::RouteFamily, + ) -> Result<&'a mut crate::domains::schedule::ScheduleActor, String> { + match actors.entry(route_family) { + Entry::Occupied(entry) => Ok(entry.into_mut()), + Entry::Vacant(entry) => { + let actor = crate::domains::schedule::ScheduleActor::try_new_with_storage( + route_family, + self.core.store.clone(), + self.core.write_options, + )?; + Ok(entry.insert(actor)) + } + } + } + + /// # Errors + /// + /// Returns an error when listing column families or preloading a persisted + /// schedule actor fails. + pub(super) fn preload_persisted_families(&self) -> Result<(), String> { + let started_at = std::time::Instant::now(); + let column_families = self + .core + .store + .list_column_families() + .map_err(|e| format!("list schedule column families failed: {e}"))?; + let persisted_family_count = column_families + .iter() + .filter(|column_family| column_family.id() != 0) + .count(); + tracing::info!( + domain = "schedule", + persisted_family_count, + "Schedule preload discovered persisted families" + ); + + let mut actors = self.core.actors.lock(); + let mut preloaded_family_count = 0_usize; + for column_family in column_families { + if column_family.id() == 0 { + continue; + } + + let family = crate::runtime::routing::RouteFamily::new(column_family.id()); + if actors.contains_key(&family) { + continue; + } + + let actor = crate::domains::schedule::ScheduleActor::try_new_with_storage( + family, + self.core.store.clone(), + self.core.write_options, + )?; + actors.insert(family, actor); + preloaded_family_count = preloaded_family_count.saturating_add(1); + tracing::debug!( + domain = "schedule", + route_family = family.id(), + preloaded_family_count, + persisted_family_count, + "Schedule persisted family preloaded" + ); + } + + // Seed the rolling-window acknowledgement counter from persisted + // last_fire_ms values so executions-per-minute survives restarts for + // occurrences already acknowledged within the last 60 seconds. + let now_ms = now_epoch_ms(); + let cutoff_ms = now_ms.saturating_sub(EXECUTIONS_WINDOW_MS); + let mut deque = self.core.recent_acknowledgement_ms.lock(); + for actor in actors.values() { + for ts in actor.last_fire_timestamps_since(cutoff_ms) { + deque.push_back(ts); + } + } + deque.make_contiguous().sort_unstable(); + drop(deque); + + drop(actors); + + self.schedule_admin_snapshot(true); + tracing::info!( + domain = "schedule", + preloaded_family_count, + persisted_family_count, + elapsed_ms = duration_millis(started_at.elapsed()), + "Schedule actor projection preload completed" + ); + Ok(()) + } +} diff --git a/src/domains/schedule/sink/delivery.rs b/src/domains/schedule/sink/delivery.rs new file mode 100644 index 00000000..d575fd38 --- /dev/null +++ b/src/domains/schedule/sink/delivery.rs @@ -0,0 +1,357 @@ +//! Due-schedule fan-out: claiming due fires, delivering them to live +//! subscribers or the durable pending-fire path, and acknowledging delivery. + +use super::delivery_strategy::DeliveryStrategy; +use super::model::{ + Envelope, HashMap, HashSet, Ordering, PendingFireKey, PendingFireState, PendingFireStates, + ScheduleDomainRuntime, EXECUTIONS_WINDOW_MS, +}; +#[cfg(test)] +use crate::dispatch::protocol::frame_context::FrameContext; + +type PendingAckRetryMap = HashMap>; + +type LivePublishCandidate = ( + crate::runtime::routing::RouteFamily, + u64, + String, + crate::domains::schedule::ScheduleDeliveryMode, + bytes::Bytes, +); + +struct DueScanPlan { + live_publish_candidates: Vec, + ack_retry_candidates: PendingAckRetryMap, + snapshot_dirty: bool, +} + +impl ScheduleDomainRuntime<'_> { + pub(super) fn scan_due_schedules(&self) { + let DueScanPlan { + live_publish_candidates, + mut ack_retry_candidates, + snapshot_dirty, + } = self.claim_due(); + let had_live_handoffs = + self.deliver_claims(live_publish_candidates, &mut ack_retry_candidates); + let acknowledged_handoffs = self.acknowledge_delivered(ack_retry_candidates); + + if snapshot_dirty || had_live_handoffs || acknowledged_handoffs { + self.schedule_admin_snapshot(false); + } + + self.refresh_metrics_gauges(); + } + + pub(super) fn force_due_scan_for_tests(&self, ready_count: usize) { + { + let mut actors = self.core.actors.lock(); + for actor in actors.values_mut() { + actor.bench_prepare_scan(ready_count); + } + } + + self.scan_due_schedules(); + self.schedule_admin_snapshot(true); + } + + fn claim_due(&self) -> DueScanPlan { + let mut live_publish_candidates = Vec::new(); + let mut ack_retry_candidates = PendingAckRetryMap::new(); + let mut snapshot_dirty = false; + let mut actors = self.core.actors.lock(); + let mut pending_ack_retries = self.core.pending_ack_retries.lock(); + + for (family, actor) in actors.iter_mut() { + if !actor.claim_due_fires().is_empty() { + snapshot_dirty = true; + } + + Self::collect_family_pending_fires( + *family, + actor, + &mut pending_ack_retries, + &mut live_publish_candidates, + &mut ack_retry_candidates, + ); + } + + DueScanPlan { + live_publish_candidates, + ack_retry_candidates, + snapshot_dirty, + } + } + + fn collect_family_pending_fires( + family: crate::runtime::routing::RouteFamily, + actor: &crate::domains::schedule::ScheduleActor, + pending_ack_retries: &mut HashMap, + live_publish_candidates: &mut Vec, + ack_retry_candidates: &mut PendingAckRetryMap, + ) { + let family_id = family.as_u64(); + let pending_fires = actor.pending_claimed_occurrences_for_publish(); + let mut pending_keys = HashSet::with_capacity(pending_fires.len()); + let remove_retry_entry = { + let tracked_retries = pending_ack_retries.entry(family_id).or_default(); + for pending_fire in pending_fires { + let pending_key = (pending_fire.fire_ms, pending_fire.route.clone()); + pending_keys.insert(pending_key.clone()); + + match tracked_retries + .entry(pending_key.clone()) + .or_insert(PendingFireState::Claimed) + { + PendingFireState::HandedOff => { + ack_retry_candidates + .entry(family) + .or_default() + .push(pending_key); + continue; + } + PendingFireState::Acknowledged => continue, + PendingFireState::Claimed => {} + } + + live_publish_candidates.push(( + family, + pending_fire.fire_ms, + pending_fire.route, + pending_fire.delivery_mode, + pending_fire.payload, + )); + } + + tracked_retries.retain(|pending_key, _| pending_keys.contains(pending_key)); + tracked_retries.is_empty() + }; + + if remove_retry_entry { + pending_ack_retries.remove(&family_id); + } + } + + fn deliver_claims( + &self, + live_publish_candidates: Vec, + ack_retry_candidates: &mut PendingAckRetryMap, + ) -> bool { + let mut had_live_handoffs = false; + + for (family, fire_ms, route, delivery_mode, payload) in live_publish_candidates { + let accepted = self.handle_schedule_publish(family, &route, delivery_mode, &payload); + had_live_handoffs |= accepted; + if !accepted { + self.core + .live_publish_failures + .fetch_add(1, Ordering::Relaxed); + } + ack_retry_candidates + .entry(family) + .or_default() + .push((fire_ms, route)); + } + + had_live_handoffs + } + + fn acknowledge_delivered(&self, ack_retry_candidates: PendingAckRetryMap) -> bool { + let mut acknowledged_handoffs = false; + + if ack_retry_candidates.is_empty() { + return false; + } + + let mut actors = self.core.actors.lock(); + let mut pending_ack_retries = self.core.pending_ack_retries.lock(); + for (family, ack_candidates) in ack_retry_candidates { + if let Some(actor) = actors.get_mut(&family) { + acknowledged_handoffs |= self.acknowledge_family_pending_fire_claims( + family, + actor, + &ack_candidates, + &mut pending_ack_retries, + ); + } + } + + acknowledged_handoffs + } + + fn acknowledge_family_pending_fire_claims( + &self, + family: crate::runtime::routing::RouteFamily, + actor: &mut crate::domains::schedule::ScheduleActor, + ack_candidates: &[PendingFireKey], + pending_ack_retries: &mut HashMap, + ) -> bool { + let family_id = family.as_u64(); + let tracked = pending_ack_retries.entry(family_id).or_default(); + for pending_key in ack_candidates { + tracked.insert(pending_key.clone(), PendingFireState::HandedOff); + } + match actor.ack_pending_fire_claims(ack_candidates) { + Ok((acked, acknowledged_at_ms)) if acked > 0 => { + for pending_key in ack_candidates { + tracked.insert(pending_key.clone(), PendingFireState::Acknowledged); + } + Self::clear_ack_retry_candidates(family_id, ack_candidates, pending_ack_retries); + self.record_recent_acknowledgements(acked, acknowledged_at_ms); + true + } + Ok(_) => { + Self::clear_ack_retry_candidates(family_id, ack_candidates, pending_ack_retries); + false + } + Err(error) => { + self.core.ack_failures.fetch_add(1, Ordering::Relaxed); + tracing::warn!( + route_family = family.as_u64(), + error = %error, + "Failed to acknowledge pending schedule fires" + ); + false + } + } + } + + fn clear_ack_retry_candidates( + family_id: u64, + ack_candidates: &[PendingFireKey], + pending_ack_retries: &mut HashMap, + ) { + let remove_retry_entry = + if let Some(tracked_retries) = pending_ack_retries.get_mut(&family_id) { + for pending_key in ack_candidates { + tracked_retries.remove(pending_key); + } + tracked_retries.is_empty() + } else { + false + }; + if remove_retry_entry { + pending_ack_retries.remove(&family_id); + } + } + + fn record_recent_acknowledgements(&self, acked: usize, acknowledged_at_ms: u64) { + let mut deque = self.core.recent_acknowledgement_ms.lock(); + let cutoff = acknowledged_at_ms.saturating_sub(EXECUTIONS_WINDOW_MS); + while deque.front().copied().is_some_and(|t| t < cutoff) { + deque.pop_front(); + } + for _ in 0..acked { + deque.push_back(acknowledged_at_ms); + } + } + + pub(super) fn route_live_notify( + &self, + session_id: u64, + subscription_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + route: &str, + payload: &bytes::Bytes, + ) -> bool { + #[cfg(test)] + let notify_payload = crate::dispatch::protocol::schedule_codec::encode_notify( + subscription_id, + route, + payload.as_ref(), + ); + + #[cfg(test)] + let notify_ctx = FrameContext::new( + session_id, + crate::dispatch::protocol::frame::ChannelId::Sub, + crate::dispatch::protocol::tlv::MessageType::new(705), + bytes::Bytes::from(notify_payload), + *subscriber.family(), + ); + + #[cfg(test)] + let notify_envelope = Envelope::new(subscriber.clone(), notify_ctx); + + #[cfg(not(test))] + let notify_envelope = Envelope::new( + subscriber.clone(), + crate::domains::schedule::ScheduleClientNotification::new( + session_id, + *subscriber.family(), + subscription_id, + route.to_string(), + payload.clone(), + ), + ); + + // Subscriber notify routing is best-effort and must not redefine the + // schedule domain's durable acknowledgement boundary. + self.core.router.route(notify_envelope).is_ok() + } + + pub(super) fn handle_schedule_publish( + &self, + family: crate::runtime::routing::RouteFamily, + route: &str, + delivery_mode: crate::domains::schedule::ScheduleDeliveryMode, + payload: &bytes::Bytes, + ) -> bool { + let mut families = self.core.sub_families.lock(); + let Some(state) = families.get_mut(&family) else { + return false; + }; + let mut subscription_ids = state.matching_ids(family, route); + subscription_ids.sort_unstable(); + if subscription_ids.is_empty() { + return false; + } + + let cursor = state + .round_robin_cursors + .get(route) + .copied() + .unwrap_or_else(|| super::delivery_strategy::initial_round_robin_cursor(route)); + let strategy = + DeliveryStrategy::select_recipients(delivery_mode, &subscription_ids, cursor); + let mut any_accepted = false; + for subscription_id in strategy.recipients() { + let Some(subscription) = state.subscriptions.get(*subscription_id) else { + continue; + }; + let accepted = self.route_live_notify( + subscription.session_id, + subscription.subscription_id, + &subscription.subscriber, + route, + payload, + ); + any_accepted |= accepted; + if accepted && strategy.stops_after_success() { + let index = subscription_ids + .iter() + .position(|candidate| candidate == subscription_id) + .unwrap_or(cursor); + state + .round_robin_cursors + .insert(route.to_string(), (index + 1) % subscription_ids.len()); + return true; + } + } + if strategy.stops_after_success() { + state + .round_robin_cursors + .insert(route.to_string(), (cursor + 1) % subscription_ids.len()); + } + any_accepted + } + + pub(super) fn handle_domain_publish(&self, event: &crate::runtime::DomainPublishEvent) { + self.handle_schedule_publish( + event.family_id, + event.route.as_str(), + crate::domains::schedule::ScheduleDeliveryMode::Broadcast, + &event.payload, + ); + } +} diff --git a/src/domains/schedule/sink/delivery_strategy.rs b/src/domains/schedule/sink/delivery_strategy.rs index b7758c28..36c4e57e 100644 --- a/src/domains/schedule/sink/delivery_strategy.rs +++ b/src/domains/schedule/sink/delivery_strategy.rs @@ -40,10 +40,76 @@ impl DeliveryStrategy { } } +/// Starting cursor for a route that has not delivered before. +/// +/// Defaulting an unseen route to 0 sends every first-time Single schedule to +/// the same subscriber: the cursor is per route, so a fleet of one-shot +/// schedules never rotates at all. Seeding from the route spreads them +/// deterministically while later fires still advance the stored cursor. +pub(super) fn initial_round_robin_cursor(route: &str) -> usize { + use std::hash::{Hash, Hasher}; + + let mut hasher = rustc_hash::FxHasher::default(); + route.hash(&mut hasher); + usize::try_from(hasher.finish()).unwrap_or(usize::MAX) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn should_spread_first_delivery_of_distinct_routes_across_subscribers() { + // Arrange + // Every one of these routes is firing for the first time, so each has + // no stored cursor. If unseen routes start at zero they all choose the + // same subscriber and one client absorbs the entire fleet's load. + let candidates = [10_u64, 20, 30, 40]; + let routes = (0..200) + .map(|index| format!("schedule://acme/jobs/one-shot-{index:04}/run")) + .collect::>(); + + // Act + let chosen = routes + .iter() + .map(|route| { + let cursor = initial_round_robin_cursor(route); + DeliveryStrategy::select_recipients( + ScheduleDeliveryMode::Single, + &candidates, + cursor, + ) + .recipients()[0] + }) + .collect::>(); + + // The old behaviour, kept explicit so this test cannot silently stop + // discriminating: an unseen route defaulting to cursor 0 puts every + // first delivery on the same subscriber. + let all_at_zero = routes + .iter() + .map(|_| { + DeliveryStrategy::select_recipients(ScheduleDeliveryMode::Single, &candidates, 0) + .recipients()[0] + }) + .collect::>(); + + // Assert + assert_eq!( + all_at_zero.len(), + 1, + "a zero cursor should concentrate; if not, this test proves nothing" + ); + for candidate in candidates { + let share = chosen.iter().filter(|picked| **picked == candidate).count(); + assert!( + share > 0, + "subscriber {candidate} received none of {} first deliveries", + chosen.len() + ); + } + } + #[test] fn should_rotate_single_delivery_candidates_without_router() { // Arrange diff --git a/src/domains/schedule/sink/domain_sink_impl.rs b/src/domains/schedule/sink/domain_sink_impl.rs deleted file mode 100644 index ca4f079a..00000000 --- a/src/domains/schedule/sink/domain_sink_impl.rs +++ /dev/null @@ -1,887 +0,0 @@ -use super::delivery_strategy::DeliveryStrategy; -use super::model::{ - now_epoch_ms, Arc, AtomicBool, AtomicU64, Entry, Envelope, HashMap, HashSet, Instant, Mutex, - Ordering, PendingFireKey, PendingFireState, PendingFireStates, Router, ScheduleDomainActor, - ScheduleDomainCommand, ScheduleDomainCore, ScheduleDomainRuntime, ScheduleDomainSink, - ScheduleDomainState, ScheduleLiveCounts, ScheduleMetrics, VecDeque, EXECUTIONS_WINDOW_MS, -}; -#[cfg(test)] -use crate::dispatch::protocol::frame_context::FrameContext; -use crate::runtime::routing::{Route, RouteAddress, RouteFamily}; - -type PendingAckRetryMap = HashMap>; -pub(crate) const DEFAULT_SCHEDULE_PRELOAD_TIMEOUT: std::time::Duration = - std::time::Duration::from_secs(120); - -fn duration_millis(duration: std::time::Duration) -> u64 { - u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) -} - -type LivePublishCandidate = ( - crate::runtime::routing::RouteFamily, - u64, - String, - crate::domains::schedule::ScheduleDeliveryMode, - bytes::Bytes, -); - -mod admin_runtime; - -struct DueScanPlan { - live_publish_candidates: Vec, - ack_retry_candidates: PendingAckRetryMap, - snapshot_dirty: bool, -} - -/// Narrow read-only surface used by metrics and administration code. -pub trait ScheduleObservability { - fn subscription_count(&self) -> usize; - fn schedule_count(&self) -> usize; - fn pending_fire_count(&self) -> usize; - fn executions_per_minute(&self) -> f64; - fn notify_failure_count(&self) -> u64; - fn ack_failure_count(&self) -> u64; - fn pending_ack_retry_count(&self) -> usize; - fn oldest_pending_claim_age_seconds(&self) -> u64; - fn overdue_normalization_count(&self) -> u64; -} - -impl ScheduleDomainState { - fn new_with_storage( - store: crate::storage::FitzStorageEngine, - router: Arc, - admin_read_model: Arc, - ) -> Self { - Self { - core: ScheduleDomainCore { - store, - actors: Mutex::new(HashMap::new()), - sub_families: Mutex::new(HashMap::new()), - next_sub_id: AtomicU64::new(1), - router, - admin_read_model, - snapshot_dirty: AtomicBool::new(false), - snapshot_syncing: AtomicBool::new(false), - last_snapshot_elapsed_us: AtomicU64::new(0), - snapshot_epoch: Instant::now(), - live_publish_failures: AtomicU64::new(0), - ack_failures: AtomicU64::new(0), - pending_ack_retries: Mutex::new(HashMap::new()), - recent_acknowledgement_ms: Mutex::new(VecDeque::new()), - write_options: cntryl_midge::WriteOptions::buffered(), - metrics: None, - }, - active: AtomicBool::new(true), - } - } - - pub(super) fn runtime(&self) -> ScheduleDomainRuntime<'_> { - ScheduleDomainRuntime { - core: &self.core, - active: &self.active, - } - } -} - -impl ScheduleDomainActor { - pub(super) fn new(state: Arc) -> Self { - Self { state } - } - - pub(super) fn route_address() -> RouteAddress { - RouteAddress::new( - RouteFamily::new(0), - Route::new("internal://domain/schedule"), - ) - } -} - -impl ScheduleDomainSink { - pub fn new( - store: Arc, - router: Arc, - admin_read_model: Arc, - ) -> Self { - Self::new_with_storage( - crate::storage::FitzStorageEngine::new(store), - router, - admin_read_model, - ) - } - - pub(crate) fn new_with_storage( - store: crate::storage::FitzStorageEngine, - router: Arc, - admin_read_model: Arc, - ) -> Self { - let state = Arc::new(ScheduleDomainState::new_with_storage( - store, - router, - admin_read_model, - )); - let actor = Self::spawn_actor(state.clone()); - Self { state, actor } - } - - fn spawn_actor( - state: Arc, - ) -> crate::runtime::ManagedActor { - let router = state.core.router.clone(); - crate::runtime::ManagedActor::spawn_fail_closed( - router, - ScheduleDomainActor::route_address(), - move || ScheduleDomainActor::new(state.clone()), - crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, - ) - } - - fn rebuild_actor(&mut self) { - self.actor.stop(); - self.actor = Self::spawn_actor(self.state.clone()); - } - - fn state_for_builder(&mut self) -> &mut ScheduleDomainState { - Arc::get_mut(&mut self.state) - .expect("Schedule sink builders must run before sharing the sink") - } - - #[must_use] - pub fn with_write_options(mut self, write_options: cntryl_midge::WriteOptions) -> Self { - self.actor.stop(); - self.state_for_builder().core.write_options = write_options; - self.rebuild_actor(); - self - } - - #[must_use] - pub fn with_metrics( - mut self, - collector: crate::observability::metrics::MetricsCollector, - ) -> Self { - self.actor.stop(); - let state = self.state_for_builder(); - state.core.metrics = Some(ScheduleMetrics::new(collector)); - state.runtime().refresh_metrics_gauges(); - self.rebuild_actor(); - self - } - - pub fn stop(&self) { - self.state.active.store(false, Ordering::Relaxed); - self.actor.stop(); - } - - #[cfg(test)] - pub(super) fn is_actor_running(&self) -> bool { - self.actor.is_running() - } - - pub(crate) fn actor_health_snapshot(&self) -> crate::runtime::ManagedActorHealthSnapshot { - self.actor.health_snapshot() - } - - #[cfg(test)] - pub(crate) fn panic_actor_for_tests(&self) { - let _ = self - .actor - .try_send_high_priority(ScheduleDomainCommand::PanicForTests); - } - - #[cfg(test)] - pub(super) fn stop_actor_for_tests(&self) { - self.actor.stop(); - } - - #[cfg(test)] - pub(super) fn block_actor_for_tests( - &self, - entered: crossbeam_channel::Sender<()>, - release: crossbeam_channel::Receiver<()>, - ) { - self.actor - .try_send_high_priority(ScheduleDomainCommand::BlockForTests(entered, release)) - .expect("enqueue Schedule actor test block"); - } - - /// # Errors - /// - /// Returns an error when listing column families or preloading a persisted - /// schedule actor fails. - pub fn preload_persisted_families(&self) -> Result<(), String> { - self.preload_persisted_families_with_timeout(DEFAULT_SCHEDULE_PRELOAD_TIMEOUT) - } - - /// # Errors - /// - /// Returns an error when the actor cannot be reached, preload fails, or the - /// actor does not reply before `timeout`. - pub(crate) fn preload_persisted_families_with_timeout( - &self, - timeout: std::time::Duration, - ) -> Result<(), String> { - let started_at = std::time::Instant::now(); - let timeout_ms = duration_millis(timeout); - tracing::info!(domain = "schedule", timeout_ms, "Schedule preload started"); - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = self - .actor - .try_send_high_priority(ScheduleDomainCommand::PreloadPersistedFamilies(reply_tx)) - { - return Err(format!("schedule preload enqueue failed: {error}")); - } - - match reply_rx.recv_timeout(timeout) { - Ok(result) => { - result?; - tracing::info!( - domain = "schedule", - elapsed_ms = duration_millis(started_at.elapsed()), - "Schedule preload completed" - ); - Ok(()) - } - Err(crossbeam_channel::RecvTimeoutError::Timeout) => { - tracing::error!( - domain = "schedule", - timeout_ms, - elapsed_ms = duration_millis(started_at.elapsed()), - "Schedule preload timed out" - ); - Err(format!( - "schedule preload reply timed out after {timeout_ms}ms" - )) - } - Err(crossbeam_channel::RecvTimeoutError::Disconnected) => { - Err("schedule preload reply failed: actor reply channel disconnected".to_string()) - } - } - } - - pub(crate) fn is_active(&self) -> bool { - self.state.active.load(Ordering::Relaxed) - } - - pub(crate) fn scan_due_schedules(&self) { - if let Err(error) = self - .actor - .try_send_high_priority(ScheduleDomainCommand::ScanDueSchedules) - { - tracing::warn!(domain = "schedule", error = %error, "Schedule due scan enqueue failed"); - } - } - - pub(crate) fn force_due_scan_for_tests(&self, ready_count: usize) { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = - self.actor - .try_send_high_priority(ScheduleDomainCommand::ForceDueScanForTests( - ready_count, - reply_tx, - )) - { - tracing::warn!(domain = "schedule", error = %error, "Schedule forced due scan enqueue failed"); - return; - } - - if let Err(error) = reply_rx.recv_timeout(std::time::Duration::from_secs(1)) { - tracing::warn!(domain = "schedule", error = %error, "Schedule forced due scan reply failed"); - } - } - - pub fn unsubscribe_all(&self, session_id: u64) { - if let Err(error) = self - .actor - .try_send_high_priority(ScheduleDomainCommand::CleanupSession(session_id)) - { - tracing::warn!(domain = "schedule", error = %error, "Schedule cleanup enqueue failed"); - } - } - - pub fn admin_pending_claims( - &self, - route_family: crate::runtime::routing::RouteFamily, - ) -> Vec { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = - self.actor - .try_send_high_priority(ScheduleDomainCommand::ReadPendingClaims( - route_family, - reply_tx, - )) - { - tracing::warn!(domain = "schedule", error = %error, "Schedule pending claim read enqueue failed"); - return Vec::new(); - } - - reply_rx - .recv_timeout(std::time::Duration::from_secs(1)) - .unwrap_or_default() - } - - fn live_counts(&self) -> ScheduleLiveCounts { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = self - .actor - .try_send_high_priority(ScheduleDomainCommand::ReadLiveCounts(reply_tx)) - { - tracing::warn!(domain = "schedule", error = %error, "Schedule live-count query enqueue failed"); - return ScheduleLiveCounts::default(); - } - - reply_rx - .recv_timeout(std::time::Duration::from_secs(1)) - .unwrap_or_default() - } - - pub(crate) fn refresh_admin_snapshot_if_dirty(&self) { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = self - .actor - .try_send_high_priority(ScheduleDomainCommand::RefreshAdminSnapshotIfDirty(reply_tx)) - { - tracing::warn!(domain = "schedule", error = %error, "Schedule admin snapshot refresh enqueue failed"); - return; - } - - if let Err(error) = reply_rx.recv_timeout(std::time::Duration::from_secs(1)) { - tracing::warn!(domain = "schedule", error = %error, "Schedule admin snapshot refresh reply failed"); - } - } - - #[doc(hidden)] - pub fn bench_publish_event(&self, event: &crate::runtime::DomainPublishEvent) { - let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - if let Err(error) = - self.actor - .try_send_high_priority(ScheduleDomainCommand::BenchPublishEvent( - event.clone(), - reply_tx, - )) - { - tracing::warn!(domain = "schedule", error = %error, "Schedule bench publish enqueue failed"); - return; - } - - if let Err(error) = reply_rx.recv_timeout(std::time::Duration::from_secs(1)) { - tracing::warn!(domain = "schedule", error = %error, "Schedule bench publish reply failed"); - } - } -} - -impl ScheduleObservability for ScheduleDomainSink { - fn subscription_count(&self) -> usize { - self.live_counts().subscriptions - } - - fn schedule_count(&self) -> usize { - self.live_counts().schedules - } - - fn pending_fire_count(&self) -> usize { - self.live_counts().pending_fires - } - - fn executions_per_minute(&self) -> f64 { - self.live_counts().executions_per_minute - } - - fn notify_failure_count(&self) -> u64 { - self.live_counts().notify_failures - } - - fn ack_failure_count(&self) -> u64 { - self.live_counts().ack_failures - } - - fn pending_ack_retry_count(&self) -> usize { - self.live_counts().pending_ack_retries - } - - fn oldest_pending_claim_age_seconds(&self) -> u64 { - self.live_counts().oldest_pending_claim_age_seconds - } - - fn overdue_normalization_count(&self) -> u64 { - self.live_counts().overdue_normalizations - } -} - -impl ScheduleDomainSink { - pub fn subscription_count(&self) -> usize { - ScheduleObservability::subscription_count(self) - } - - pub fn schedule_count(&self) -> usize { - ScheduleObservability::schedule_count(self) - } - - pub fn pending_fire_count(&self) -> usize { - ScheduleObservability::pending_fire_count(self) - } - - pub fn executions_per_minute(&self) -> f64 { - ScheduleObservability::executions_per_minute(self) - } - - pub fn notify_failure_count(&self) -> u64 { - ScheduleObservability::notify_failure_count(self) - } - - pub fn ack_failure_count(&self) -> u64 { - ScheduleObservability::ack_failure_count(self) - } - - pub fn pending_ack_retry_count(&self) -> usize { - ScheduleObservability::pending_ack_retry_count(self) - } - - pub fn oldest_pending_claim_age_seconds(&self) -> u64 { - ScheduleObservability::oldest_pending_claim_age_seconds(self) - } - - pub fn overdue_normalization_count(&self) -> u64 { - ScheduleObservability::overdue_normalization_count(self) - } -} - -impl ScheduleDomainRuntime<'_> { - /// # Errors - /// - /// Returns an error when listing column families or preloading a persisted - /// schedule actor fails. - pub(super) fn preload_persisted_families(&self) -> Result<(), String> { - let started_at = std::time::Instant::now(); - let column_families = self - .core - .store - .list_column_families() - .map_err(|e| format!("list schedule column families failed: {e}"))?; - let persisted_family_count = column_families - .iter() - .filter(|column_family| column_family.id() != 0) - .count(); - tracing::info!( - domain = "schedule", - persisted_family_count, - "Schedule preload discovered persisted families" - ); - - let mut actors = self.core.actors.lock(); - let mut preloaded_family_count = 0_usize; - for column_family in column_families { - if column_family.id() == 0 { - continue; - } - - let family = crate::runtime::routing::RouteFamily::new(column_family.id()); - if actors.contains_key(&family) { - continue; - } - - let actor = crate::domains::schedule::ScheduleActor::try_new_with_storage( - family, - self.core.store.clone(), - self.core.write_options, - )?; - actors.insert(family, actor); - preloaded_family_count = preloaded_family_count.saturating_add(1); - tracing::debug!( - domain = "schedule", - route_family = family.id(), - preloaded_family_count, - persisted_family_count, - "Schedule persisted family preloaded" - ); - } - - // Seed the rolling-window acknowledgement counter from persisted - // last_fire_ms values so executions-per-minute survives restarts for - // occurrences already acknowledged within the last 60 seconds. - let now_ms = now_epoch_ms(); - let cutoff_ms = now_ms.saturating_sub(EXECUTIONS_WINDOW_MS); - let mut deque = self.core.recent_acknowledgement_ms.lock(); - for actor in actors.values() { - for ts in actor.last_fire_timestamps_since(cutoff_ms) { - deque.push_back(ts); - } - } - deque.make_contiguous().sort_unstable(); - drop(deque); - - drop(actors); - - self.schedule_admin_snapshot(true); - tracing::info!( - domain = "schedule", - preloaded_family_count, - persisted_family_count, - elapsed_ms = duration_millis(started_at.elapsed()), - "Schedule actor projection preload completed" - ); - Ok(()) - } - - pub(super) fn scan_due_schedules(&self) { - let DueScanPlan { - live_publish_candidates, - mut ack_retry_candidates, - snapshot_dirty, - } = self.claim_due(); - let had_live_handoffs = - self.deliver_claims(live_publish_candidates, &mut ack_retry_candidates); - let acknowledged_handoffs = self.acknowledge_delivered(ack_retry_candidates); - - if snapshot_dirty || had_live_handoffs || acknowledged_handoffs { - self.schedule_admin_snapshot(false); - } - - self.refresh_metrics_gauges(); - } - - pub(crate) fn force_due_scan_for_tests(&self, ready_count: usize) { - { - let mut actors = self.core.actors.lock(); - for actor in actors.values_mut() { - actor.bench_prepare_scan(ready_count); - } - } - - self.scan_due_schedules(); - self.schedule_admin_snapshot(true); - } - - fn claim_due(&self) -> DueScanPlan { - let mut live_publish_candidates = Vec::new(); - let mut ack_retry_candidates = PendingAckRetryMap::new(); - let mut snapshot_dirty = false; - let mut actors = self.core.actors.lock(); - let mut pending_ack_retries = self.core.pending_ack_retries.lock(); - - for (family, actor) in actors.iter_mut() { - if !actor.claim_due_fires().is_empty() { - snapshot_dirty = true; - } - - Self::collect_family_pending_fires( - *family, - actor, - &mut pending_ack_retries, - &mut live_publish_candidates, - &mut ack_retry_candidates, - ); - } - - DueScanPlan { - live_publish_candidates, - ack_retry_candidates, - snapshot_dirty, - } - } - - fn collect_family_pending_fires( - family: crate::runtime::routing::RouteFamily, - actor: &crate::domains::schedule::ScheduleActor, - pending_ack_retries: &mut HashMap, - live_publish_candidates: &mut Vec, - ack_retry_candidates: &mut PendingAckRetryMap, - ) { - let family_id = family.as_u64(); - let pending_fires = actor.pending_claimed_occurrences_for_publish(); - let mut pending_keys = HashSet::with_capacity(pending_fires.len()); - let remove_retry_entry = { - let tracked_retries = pending_ack_retries.entry(family_id).or_default(); - for pending_fire in pending_fires { - let pending_key = (pending_fire.fire_ms, pending_fire.route.clone()); - pending_keys.insert(pending_key.clone()); - - match tracked_retries - .entry(pending_key.clone()) - .or_insert(PendingFireState::Claimed) - { - PendingFireState::HandedOff => { - ack_retry_candidates - .entry(family) - .or_default() - .push(pending_key); - continue; - } - PendingFireState::Acknowledged => continue, - PendingFireState::Claimed => {} - } - - live_publish_candidates.push(( - family, - pending_fire.fire_ms, - pending_fire.route, - pending_fire.delivery_mode, - pending_fire.payload, - )); - } - - tracked_retries.retain(|pending_key, _| pending_keys.contains(pending_key)); - tracked_retries.is_empty() - }; - - if remove_retry_entry { - pending_ack_retries.remove(&family_id); - } - } - - fn deliver_claims( - &self, - live_publish_candidates: Vec, - ack_retry_candidates: &mut PendingAckRetryMap, - ) -> bool { - let mut had_live_handoffs = false; - - for (family, fire_ms, route, delivery_mode, payload) in live_publish_candidates { - let accepted = self.handle_schedule_publish(family, &route, delivery_mode, &payload); - had_live_handoffs |= accepted; - if !accepted { - self.core - .live_publish_failures - .fetch_add(1, Ordering::Relaxed); - } - ack_retry_candidates - .entry(family) - .or_default() - .push((fire_ms, route)); - } - - had_live_handoffs - } - - fn acknowledge_delivered(&self, ack_retry_candidates: PendingAckRetryMap) -> bool { - let mut acknowledged_handoffs = false; - - if ack_retry_candidates.is_empty() { - return false; - } - - let mut actors = self.core.actors.lock(); - let mut pending_ack_retries = self.core.pending_ack_retries.lock(); - for (family, ack_candidates) in ack_retry_candidates { - if let Some(actor) = actors.get_mut(&family) { - acknowledged_handoffs |= self.acknowledge_family_pending_fire_claims( - family, - actor, - &ack_candidates, - &mut pending_ack_retries, - ); - } - } - - acknowledged_handoffs - } - - fn acknowledge_family_pending_fire_claims( - &self, - family: crate::runtime::routing::RouteFamily, - actor: &mut crate::domains::schedule::ScheduleActor, - ack_candidates: &[PendingFireKey], - pending_ack_retries: &mut HashMap, - ) -> bool { - let family_id = family.as_u64(); - let tracked = pending_ack_retries.entry(family_id).or_default(); - for pending_key in ack_candidates { - tracked.insert(pending_key.clone(), PendingFireState::HandedOff); - } - match actor.ack_pending_fire_claims(ack_candidates) { - Ok((acked, acknowledged_at_ms)) if acked > 0 => { - for pending_key in ack_candidates { - tracked.insert(pending_key.clone(), PendingFireState::Acknowledged); - } - Self::clear_ack_retry_candidates(family_id, ack_candidates, pending_ack_retries); - self.record_recent_acknowledgements(acked, acknowledged_at_ms); - true - } - Ok(_) => { - Self::clear_ack_retry_candidates(family_id, ack_candidates, pending_ack_retries); - false - } - Err(error) => { - self.core.ack_failures.fetch_add(1, Ordering::Relaxed); - tracing::warn!( - route_family = family.as_u64(), - error = %error, - "Failed to acknowledge pending schedule fires" - ); - false - } - } - } - - fn clear_ack_retry_candidates( - family_id: u64, - ack_candidates: &[PendingFireKey], - pending_ack_retries: &mut HashMap, - ) { - let remove_retry_entry = - if let Some(tracked_retries) = pending_ack_retries.get_mut(&family_id) { - for pending_key in ack_candidates { - tracked_retries.remove(pending_key); - } - tracked_retries.is_empty() - } else { - false - }; - if remove_retry_entry { - pending_ack_retries.remove(&family_id); - } - } - - fn record_recent_acknowledgements(&self, acked: usize, acknowledged_at_ms: u64) { - let mut deque = self.core.recent_acknowledgement_ms.lock(); - let cutoff = acknowledged_at_ms.saturating_sub(EXECUTIONS_WINDOW_MS); - while deque.front().copied().is_some_and(|t| t < cutoff) { - deque.pop_front(); - } - for _ in 0..acked { - deque.push_back(acknowledged_at_ms); - } - } - - pub(super) fn get_or_create_actor<'a>( - &'a self, - actors: &'a mut HashMap< - crate::runtime::routing::RouteFamily, - crate::domains::schedule::ScheduleActor, - >, - route_family: crate::runtime::routing::RouteFamily, - ) -> Result<&'a mut crate::domains::schedule::ScheduleActor, String> { - match actors.entry(route_family) { - Entry::Occupied(entry) => Ok(entry.into_mut()), - Entry::Vacant(entry) => { - let actor = crate::domains::schedule::ScheduleActor::try_new_with_storage( - route_family, - self.core.store.clone(), - self.core.write_options, - )?; - Ok(entry.insert(actor)) - } - } - } - - pub(super) fn route_live_notify( - &self, - session_id: u64, - subscription_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - route: &str, - payload: &bytes::Bytes, - ) -> bool { - #[cfg(test)] - let notify_payload = crate::dispatch::protocol::schedule_codec::encode_notify( - subscription_id, - route, - payload.as_ref(), - ); - - #[cfg(test)] - let notify_ctx = FrameContext::new( - session_id, - crate::dispatch::protocol::frame::ChannelId::Sub, - crate::dispatch::protocol::tlv::MessageType::new(705), - bytes::Bytes::from(notify_payload), - *subscriber.family(), - ); - - #[cfg(test)] - let notify_envelope = Envelope::new(subscriber.clone(), notify_ctx); - - #[cfg(not(test))] - let notify_envelope = Envelope::new( - subscriber.clone(), - crate::domains::schedule::ScheduleClientNotification::new( - session_id, - *subscriber.family(), - subscription_id, - route.to_string(), - payload.clone(), - ), - ); - - // Subscriber notify routing is best-effort and must not redefine the - // schedule domain's durable acknowledgement boundary. - self.core.router.route(notify_envelope).is_ok() - } - - fn handle_schedule_publish( - &self, - family: crate::runtime::routing::RouteFamily, - route: &str, - delivery_mode: crate::domains::schedule::ScheduleDeliveryMode, - payload: &bytes::Bytes, - ) -> bool { - let mut families = self.core.sub_families.lock(); - let Some(state) = families.get_mut(&family.as_u64()) else { - return false; - }; - let mut subscription_ids = state.matching_ids(family, route); - subscription_ids.sort_unstable(); - if subscription_ids.is_empty() { - return false; - } - - let cursor = state.round_robin_cursors.get(route).copied().unwrap_or(0); - let strategy = - DeliveryStrategy::select_recipients(delivery_mode, &subscription_ids, cursor); - let mut any_accepted = false; - for subscription_id in strategy.recipients() { - let Some(subscription) = state.subscriptions.get(*subscription_id) else { - continue; - }; - let accepted = self.route_live_notify( - subscription.session_id, - subscription.subscription_id, - &subscription.subscriber, - route, - payload, - ); - any_accepted |= accepted; - if accepted && strategy.stops_after_success() { - let index = subscription_ids - .iter() - .position(|candidate| candidate == subscription_id) - .unwrap_or(cursor); - state - .round_robin_cursors - .insert(route.to_string(), (index + 1) % subscription_ids.len()); - return true; - } - } - if strategy.stops_after_success() { - state - .round_robin_cursors - .insert(route.to_string(), (cursor + 1) % subscription_ids.len()); - } - any_accepted - } - - pub(super) fn handle_domain_publish(&self, event: &crate::runtime::DomainPublishEvent) { - self.handle_schedule_publish( - event.family_id, - event.route.as_str(), - crate::domains::schedule::ScheduleDeliveryMode::Broadcast, - &event.payload, - ); - } - - pub fn unsubscribe_all(&self, session_id: u64) { - let mut families = self.core.sub_families.lock(); - for (family, state) in families.iter_mut() { - state.remove_session( - crate::runtime::routing::RouteFamily::new( - u32::try_from(*family).unwrap_or(u32::MAX), - ), - session_id, - ); - } - families.retain(|_, state| !state.is_empty()); - tracing::debug!( - domain = "schedule", - session = session_id, - "All schedule subscriptions removed for session" - ); - } -} diff --git a/src/domains/schedule/sink/facade.rs b/src/domains/schedule/sink/facade.rs new file mode 100644 index 00000000..a0e0dca7 --- /dev/null +++ b/src/domains/schedule/sink/facade.rs @@ -0,0 +1,366 @@ +//! Public `ScheduleDomainSink` API and actor lifecycle management. + +use super::model::{ + duration_millis, Arc, AtomicBool, AtomicU64, HashMap, Instant, Mutex, Ordering, Router, + ScheduleDomainActor, ScheduleDomainCommand, ScheduleDomainCore, ScheduleDomainRuntime, + ScheduleDomainSink, ScheduleDomainState, ScheduleLiveCounts, ScheduleMetrics, VecDeque, +}; +use crate::runtime::routing::{Route, RouteAddress, RouteFamily}; + +pub(crate) const DEFAULT_SCHEDULE_PRELOAD_TIMEOUT: std::time::Duration = + std::time::Duration::from_secs(120); + +impl ScheduleDomainState { + fn new_with_storage( + store: crate::storage::FitzStorageEngine, + router: Arc, + admin_read_model: Arc, + ) -> Self { + Self { + core: ScheduleDomainCore { + store, + actors: Mutex::new(HashMap::new()), + sub_families: Mutex::new(HashMap::new()), + cleaned_up_sessions: Mutex::new(crate::runtime::CleanedUpSessions::new( + crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, + )), + next_sub_id: AtomicU64::new(1), + router, + admin_read_model, + snapshot_dirty: AtomicBool::new(false), + snapshot_syncing: AtomicBool::new(false), + last_snapshot_elapsed_us: AtomicU64::new(0), + snapshot_epoch: Instant::now(), + live_publish_failures: AtomicU64::new(0), + ack_failures: AtomicU64::new(0), + pending_ack_retries: Mutex::new(HashMap::new()), + recent_acknowledgement_ms: Mutex::new(VecDeque::new()), + write_options: cntryl_midge::WriteOptions::buffered(), + metrics: None, + }, + active: AtomicBool::new(true), + } + } + + pub(super) fn runtime(&self) -> ScheduleDomainRuntime<'_> { + ScheduleDomainRuntime { + core: &self.core, + active: &self.active, + } + } +} + +impl ScheduleDomainActor { + pub(super) fn new(state: Arc) -> Self { + Self { state } + } + + pub(super) fn route_address() -> RouteAddress { + RouteAddress::new( + RouteFamily::new(0), + Route::new("internal://domain/schedule"), + ) + } +} + +impl ScheduleDomainSink { + pub fn new( + store: Arc, + router: Arc, + admin_read_model: Arc, + ) -> Self { + Self::new_with_storage( + crate::storage::FitzStorageEngine::new(store), + router, + admin_read_model, + ) + } + + pub(crate) fn new_with_storage( + store: crate::storage::FitzStorageEngine, + router: Arc, + admin_read_model: Arc, + ) -> Self { + let state = Arc::new(ScheduleDomainState::new_with_storage( + store, + router, + admin_read_model, + )); + let actor = Self::spawn_actor(state.clone()); + Self { state, actor } + } + + fn spawn_actor( + state: Arc, + ) -> crate::runtime::ManagedActor { + let router = state.core.router.clone(); + crate::runtime::ManagedActor::spawn_fail_closed( + router, + ScheduleDomainActor::route_address(), + move || ScheduleDomainActor::new(state.clone()), + crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, + ) + } + + fn rebuild_actor(&mut self) { + self.actor.stop(); + self.actor = Self::spawn_actor(self.state.clone()); + } + + fn state_for_builder(&mut self) -> &mut ScheduleDomainState { + Arc::get_mut(&mut self.state) + .expect("Schedule sink builders must run before sharing the sink") + } + + #[must_use] + pub fn with_write_options(mut self, write_options: cntryl_midge::WriteOptions) -> Self { + self.actor.stop(); + self.state_for_builder().core.write_options = write_options; + self.rebuild_actor(); + self + } + + #[must_use] + pub fn with_metrics( + mut self, + collector: crate::observability::metrics::MetricsCollector, + ) -> Self { + self.actor.stop(); + let state = self.state_for_builder(); + state.core.metrics = Some(ScheduleMetrics::new(collector)); + state.runtime().refresh_metrics_gauges(); + self.rebuild_actor(); + self + } + + pub fn stop(&self) { + self.state.active.store(false, Ordering::Relaxed); + self.actor.stop(); + } + + #[cfg(test)] + pub(super) fn is_actor_running(&self) -> bool { + self.actor.is_running() + } + + pub(crate) fn actor_health_snapshot(&self) -> crate::runtime::ManagedActorHealthSnapshot { + self.actor.health_snapshot() + } + + #[cfg(test)] + pub(crate) fn panic_actor_for_tests(&self) { + let _ = self + .actor + .try_send_high_priority(ScheduleDomainCommand::PanicForTests); + } + + #[cfg(test)] + pub(super) fn stop_actor_for_tests(&self) { + self.actor.stop(); + } + + #[cfg(test)] + pub(super) fn block_actor_for_tests( + &self, + entered: crossbeam_channel::Sender<()>, + release: crossbeam_channel::Receiver<()>, + ) { + self.actor + .try_send_high_priority(ScheduleDomainCommand::BlockForTests(entered, release)) + .expect("enqueue Schedule actor test block"); + } + + /// # Errors + /// + /// Returns an error when listing column families or preloading a persisted + /// schedule actor fails. + pub fn preload_persisted_families(&self) -> Result<(), String> { + self.preload_persisted_families_with_timeout(DEFAULT_SCHEDULE_PRELOAD_TIMEOUT) + } + + /// # Errors + /// + /// Returns an error when the actor cannot be reached, preload fails, or the + /// actor does not reply before `timeout`. + pub(crate) fn preload_persisted_families_with_timeout( + &self, + timeout: std::time::Duration, + ) -> Result<(), String> { + let started_at = std::time::Instant::now(); + let timeout_ms = duration_millis(timeout); + tracing::info!(domain = "schedule", timeout_ms, "Schedule preload started"); + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = self + .actor + .try_send_high_priority(ScheduleDomainCommand::PreloadPersistedFamilies(reply_tx)) + { + return Err(format!("schedule preload enqueue failed: {error}")); + } + + match reply_rx.recv_timeout(timeout) { + Ok(result) => { + result?; + tracing::info!( + domain = "schedule", + elapsed_ms = duration_millis(started_at.elapsed()), + "Schedule preload completed" + ); + Ok(()) + } + Err(crossbeam_channel::RecvTimeoutError::Timeout) => { + tracing::error!( + domain = "schedule", + timeout_ms, + elapsed_ms = duration_millis(started_at.elapsed()), + "Schedule preload timed out" + ); + Err(format!( + "schedule preload reply timed out after {timeout_ms}ms" + )) + } + Err(crossbeam_channel::RecvTimeoutError::Disconnected) => { + Err("schedule preload reply failed: actor reply channel disconnected".to_string()) + } + } + } + + pub(crate) fn is_active(&self) -> bool { + self.state.active.load(Ordering::Relaxed) + } + + pub(crate) fn scan_due_schedules(&self) { + if let Err(error) = self + .actor + .try_send_high_priority(ScheduleDomainCommand::ScanDueSchedules) + { + tracing::warn!(domain = "schedule", error = %error, "Schedule due scan enqueue failed"); + } + } + + pub(crate) fn force_due_scan_for_tests(&self, ready_count: usize) { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = + self.actor + .try_send_high_priority(ScheduleDomainCommand::ForceDueScanForTests( + ready_count, + reply_tx, + )) + { + tracing::warn!(domain = "schedule", error = %error, "Schedule forced due scan enqueue failed"); + return; + } + + if let Err(error) = reply_rx.recv_timeout(std::time::Duration::from_secs(1)) { + tracing::warn!(domain = "schedule", error = %error, "Schedule forced due scan reply failed"); + } + } + + pub fn admin_pending_claims( + &self, + route_family: crate::runtime::routing::RouteFamily, + ) -> Vec { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = + self.actor + .try_send_high_priority(ScheduleDomainCommand::ReadPendingClaims( + route_family, + reply_tx, + )) + { + tracing::warn!(domain = "schedule", error = %error, "Schedule pending claim read enqueue failed"); + return Vec::new(); + } + + reply_rx + .recv_timeout(std::time::Duration::from_secs(1)) + .unwrap_or_default() + } + + fn live_counts(&self) -> ScheduleLiveCounts { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = self + .actor + .try_send_high_priority(ScheduleDomainCommand::ReadLiveCounts(reply_tx)) + { + tracing::warn!(domain = "schedule", error = %error, "Schedule live-count query enqueue failed"); + return ScheduleLiveCounts::default(); + } + + reply_rx + .recv_timeout(std::time::Duration::from_secs(1)) + .unwrap_or_default() + } + + pub(crate) fn refresh_admin_snapshot_if_dirty(&self) { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = self + .actor + .try_send_high_priority(ScheduleDomainCommand::RefreshAdminSnapshotIfDirty(reply_tx)) + { + tracing::warn!(domain = "schedule", error = %error, "Schedule admin snapshot refresh enqueue failed"); + return; + } + + if let Err(error) = reply_rx.recv_timeout(std::time::Duration::from_secs(1)) { + tracing::warn!(domain = "schedule", error = %error, "Schedule admin snapshot refresh reply failed"); + } + } + + #[doc(hidden)] + pub fn bench_publish_event(&self, event: &crate::runtime::DomainPublishEvent) { + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); + if let Err(error) = + self.actor + .try_send_high_priority(ScheduleDomainCommand::BenchPublishEvent( + event.clone(), + reply_tx, + )) + { + tracing::warn!(domain = "schedule", error = %error, "Schedule bench publish enqueue failed"); + return; + } + + if let Err(error) = reply_rx.recv_timeout(std::time::Duration::from_secs(1)) { + tracing::warn!(domain = "schedule", error = %error, "Schedule bench publish reply failed"); + } + } +} + +/// Narrow read-only surface used by metrics and administration code. +impl ScheduleDomainSink { + pub fn subscription_count(&self) -> usize { + self.live_counts().subscriptions + } + + pub fn schedule_count(&self) -> usize { + self.live_counts().schedules + } + + pub fn pending_fire_count(&self) -> usize { + self.live_counts().pending_fires + } + + pub fn executions_per_minute(&self) -> f64 { + self.live_counts().executions_per_minute + } + + pub fn notify_failure_count(&self) -> u64 { + self.live_counts().notify_failures + } + + pub fn ack_failure_count(&self) -> u64 { + self.live_counts().ack_failures + } + + pub fn pending_ack_retry_count(&self) -> usize { + self.live_counts().pending_ack_retries + } + + pub fn oldest_pending_claim_age_seconds(&self) -> u64 { + self.live_counts().oldest_pending_claim_age_seconds + } + + pub fn overdue_normalization_count(&self) -> u64 { + self.live_counts().overdue_normalizations + } +} diff --git a/src/domains/schedule/sink/ingress.rs b/src/domains/schedule/sink/ingress.rs new file mode 100644 index 00000000..a68df897 --- /dev/null +++ b/src/domains/schedule/sink/ingress.rs @@ -0,0 +1,350 @@ +//! Envelope ingress: validate an inbound envelope, parse it into a Schedule +//! request, and dispatch to the subscriptions/definitions/response layers. + +use super::model::{DeliveryError, Envelope, Ordering, ScheduleDomainRuntime}; +#[cfg(test)] +use crate::dispatch::protocol::frame_context::FrameContext; + +impl ScheduleDomainRuntime<'_> { + pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { + if self.handle_cleanup_envelope(envelope) { + return Ok(()); + } + self.ensure_active()?; + + if self.handle_domain_publish_envelope(envelope) { + return Ok(()); + } + + Self::log_delivery(envelope); + + let Some(request) = Self::extract_request(envelope)? else { + return Ok(()); + }; + let meta = request.meta; + let request_started = self.record_request_start(); + + if !Self::valid_request_envelope(envelope, meta) { + let response = crate::domains::schedule::ScheduleResponse::Error( + crate::domains::schedule::ScheduleFailure::new( + crate::domains::schedule::ScheduleFailureCategory::InvalidTarget, + "route family mismatch", + ), + ); + let response_meta = Self::response_meta_for_source(envelope, meta); + self.route_schedule_response(envelope, response_meta, &response, request_started); + return Ok(()); + } + + // This request was already queued (on the normal lane) before this + // session's disconnect cleanup ran (on the high-priority lane) and + // jumped ahead of it. Reject rather than silently recreating a + // subscription for a session that is already gone and will never be + // cleaned up again. + if self.is_cleaned_up_session(meta.session_id) { + let response = crate::domains::schedule::ScheduleResponse::Error( + crate::domains::schedule::ScheduleFailure::new( + crate::domains::schedule::ScheduleFailureCategory::InvalidTarget, + "session already closed", + ), + ); + let response_meta = Self::response_meta_for_source(envelope, meta); + self.route_schedule_response(envelope, response_meta, &response, request_started); + return Ok(()); + } + + let Some(schedule_msg) = + self.parse_request_message(envelope, meta, request.message, request_started) + else { + return Ok(()); + }; + + if !Self::valid_schedule_message(envelope, meta, &schedule_msg) { + let response = crate::domains::schedule::ScheduleResponse::Error( + crate::domains::schedule::ScheduleFailure::new( + crate::domains::schedule::ScheduleFailureCategory::InvalidTarget, + "route family mismatch", + ), + ); + let response_meta = Self::response_meta_for_source(envelope, meta); + self.route_schedule_response(envelope, response_meta, &response, request_started); + return Ok(()); + } + + let route_addr = envelope.destination(); + let route_family = *route_addr.family(); + + let Some((response, schedule_snapshot_dirty)) = self.dispatch_schedule_message( + envelope, + meta, + request_started, + route_family, + schedule_msg, + ) else { + return Ok(()); + }; + + if schedule_snapshot_dirty { + self.schedule_admin_snapshot(false); + } + + self.route_schedule_response(envelope, meta, &response, request_started); + + Ok(()) + } + + fn ensure_active(&self) -> Result<(), DeliveryError> { + crate::runtime::ingress_support::ensure_actor_active(self.active) + } + + fn handle_domain_publish_envelope(&self, envelope: &Envelope) -> bool { + if let Some(event) = envelope.payload::() { + if *envelope.destination().family() != event.family_id { + self.core + .live_publish_failures + .fetch_add(1, Ordering::Relaxed); + return true; + } + self.handle_domain_publish(event); + return true; + } + + false + } + + fn log_delivery(envelope: &Envelope) { + crate::runtime::ingress_support::log_envelope_received( + "schedule", + "Schedule domain sink: received envelope", + envelope, + ); + } + + fn extract_request( + envelope: &Envelope, + ) -> Result, DeliveryError> { + if let Some(request) = Self::request_from_envelope(envelope) { + Ok(Some(request)) + } else { + tracing::warn!( + domain = "schedule", + "Envelope payload was not ScheduleClientRequest" + ); + Err(DeliveryError::ActorStopped) + } + } + + fn record_request_start(&self) -> Option { + self.core + .metrics + .as_ref() + .map(crate::domains::schedule::ScheduleMetrics::record_request_start) + } + + fn parse_request_message( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + message: Result< + crate::domains::schedule::ScheduleMessage, + crate::domains::schedule::ScheduleFailure, + >, + request_started: Option, + ) -> Option { + match message { + Ok(message) => Some(message), + Err(error) => { + tracing::warn!( + domain = "schedule", + error = %error, + "Failed to parse schedule message" + ); + let response = crate::domains::schedule::ScheduleResponse::Error(error); + let response_meta = Self::response_meta_for_source(envelope, meta); + self.route_schedule_response(envelope, response_meta, &response, request_started); + None + } + } + } + + pub(super) fn dispatch_schedule_message( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + request_started: Option, + route_family: crate::runtime::routing::RouteFamily, + schedule_msg: crate::domains::schedule::ScheduleMessage, + ) -> Option<(crate::domains::schedule::ScheduleResponse, bool)> { + use crate::domains::schedule::ScheduleResponse; + + match &schedule_msg { + crate::domains::schedule::ScheduleMessage::Subscribe { + family_id, + route, + session_id, + subscriber, + } => { + return Some(( + self.apply_subscribe_message( + *family_id, + route, + *session_id, + subscriber.clone(), + ), + false, + )); + } + crate::domains::schedule::ScheduleMessage::Unsubscribe { + family_id, + route, + session_id, + .. + } => { + return Some(( + self.apply_unsubscribe_message(*family_id, route, *session_id), + false, + )); + } + crate::domains::schedule::ScheduleMessage::UnsubscribeAll { session_id, .. } => { + self.unsubscribe_all(*session_id); + return Some((ScheduleResponse::Ok, false)); + } + _ => {} + } + + let mut actors = self.core.actors.lock(); + let actor = match self.get_or_create_actor(&mut actors, route_family) { + Ok(actor) => actor, + Err(error) => { + let response = ScheduleResponse::Error( + crate::domains::schedule::ScheduleFailure::parse(error), + ); + self.route_schedule_response(envelope, meta, &response, request_started); + return None; + } + }; + + Some(self.apply_schedule_message(actor, schedule_msg)) + } + + fn request_from_envelope( + envelope: &Envelope, + ) -> Option { + if let Some(request) = envelope.payload::() + { + return Some(request.clone()); + } + + #[cfg(test)] + { + let frame_ctx = envelope.payload::()?.clone(); + let subscriber = envelope.source().cloned().unwrap_or_else(|| { + crate::runtime::routing::RouteAddress::new( + *envelope.destination().family(), + crate::runtime::routing::Route::new(format!( + "inbox://session/{}", + frame_ctx.session_id + )), + ) + }); + let meta = crate::runtime::ClientFrameMeta::new( + frame_ctx.session_id, + test_client_channel_from_protocol(frame_ctx.channel_id), + frame_ctx.msg_type.as_u16(), + frame_ctx.route_family, + ); + let parsed = crate::dispatch::protocol::schedule_codec::parse_request( + &frame_ctx, + &frame_ctx.payload, + *envelope.destination().family(), + crate::session::SessionId(frame_ctx.session_id), + subscriber, + ); + Some(crate::domains::schedule::ScheduleClientRequest::new( + meta, parsed, + )) + } + + #[cfg(not(test))] + { + None + } + } + + fn valid_request_envelope(envelope: &Envelope, meta: crate::runtime::ClientFrameMeta) -> bool { + meta.route_family == *envelope.destination().family() + && envelope + .source() + .is_none_or(|source| *source.family() == meta.route_family) + } + + pub(super) fn response_meta_for_source( + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + ) -> crate::runtime::ClientFrameMeta { + envelope.source().map_or(meta, |source| { + let mut response_meta = meta; + response_meta.route_family = *source.family(); + response_meta + }) + } + + fn valid_schedule_message( + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + message: &crate::domains::schedule::ScheduleMessage, + ) -> bool { + use crate::domains::schedule::ScheduleMessage; + + match message { + ScheduleMessage::Subscribe { + family_id, + session_id, + subscriber, + .. + } + | ScheduleMessage::Unsubscribe { + family_id, + session_id, + subscriber, + .. + } => { + *family_id == meta.route_family + && *session_id == meta.session_id + && *subscriber.family() == *family_id + && envelope.source().is_none_or(|source| source == subscriber) + } + ScheduleMessage::UnsubscribeAll { + session_id, + subscriber, + } => { + *session_id == meta.session_id + && *subscriber.family() == meta.route_family + && envelope.source().is_none_or(|source| source == subscriber) + } + ScheduleMessage::Create { .. } + | ScheduleMessage::CreateBatch { .. } + | ScheduleMessage::Cancel { .. } + | ScheduleMessage::List { .. } + | ScheduleMessage::ListV2 { .. } => true, + } + } +} + +#[cfg(test)] +fn test_client_channel_from_protocol( + channel: crate::dispatch::protocol::frame::ChannelId, +) -> crate::runtime::ClientChannel { + match channel { + crate::dispatch::protocol::frame::ChannelId::Control => { + crate::runtime::ClientChannel::Control + } + crate::dispatch::protocol::frame::ChannelId::Pub => crate::runtime::ClientChannel::Pub, + crate::dispatch::protocol::frame::ChannelId::Sub => crate::runtime::ClientChannel::Sub, + crate::dispatch::protocol::frame::ChannelId::Rpc => crate::runtime::ClientChannel::Rpc, + crate::dispatch::protocol::frame::ChannelId::Lease => crate::runtime::ClientChannel::Lease, + crate::dispatch::protocol::frame::ChannelId::Internal => { + crate::runtime::ClientChannel::Internal + } + } +} diff --git a/src/domains/schedule/sink/mailbox.rs b/src/domains/schedule/sink/mailbox.rs new file mode 100644 index 00000000..5e40a129 --- /dev/null +++ b/src/domains/schedule/sink/mailbox.rs @@ -0,0 +1,70 @@ +//! Mailbox-lane routing and the domain actor's message loop. + +use super::model::{ + DeliveryError, Envelope, MailboxSink, ScheduleDomainActor, ScheduleDomainCommand, + ScheduleDomainSink, +}; +use crate::runtime::{Actor, Context}; + +impl MailboxSink for ScheduleDomainSink { + fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.actor + .try_send(ScheduleDomainCommand::Deliver(envelope)) + } + + fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { + self.actor + .try_send_high_priority(ScheduleDomainCommand::Deliver(envelope)) + } +} + +impl Actor for ScheduleDomainActor { + type Message = ScheduleDomainCommand; + + fn receive(&mut self, msg: Self::Message, _ctx: &mut Context) { + let runtime = self.state.runtime(); + match msg { + ScheduleDomainCommand::Deliver(envelope) => { + if let Err(error) = runtime.deliver_envelope(&envelope) { + tracing::warn!(domain = "schedule", error = %error, "Schedule actor delivery failed"); + } + } + ScheduleDomainCommand::CleanupSession(session_id) => { + runtime.unsubscribe_all(session_id); + } + ScheduleDomainCommand::ReadLiveCounts(reply) => { + let _ = reply.send(runtime.live_counts()); + } + ScheduleDomainCommand::ReadPendingClaims(route_family, reply) => { + let _ = reply.send(runtime.admin_pending_claims(route_family)); + } + ScheduleDomainCommand::RefreshAdminSnapshotIfDirty(reply) => { + runtime.refresh_admin_snapshot_if_dirty(); + let _ = reply.send(()); + } + ScheduleDomainCommand::ScanDueSchedules => { + runtime.scan_due_schedules(); + } + ScheduleDomainCommand::PreloadPersistedFamilies(reply) => { + let _ = reply.send(runtime.preload_persisted_families()); + } + ScheduleDomainCommand::BenchPublishEvent(event, reply) => { + runtime.bench_publish_event(&event); + let _ = reply.send(()); + } + ScheduleDomainCommand::ForceDueScanForTests(ready_count, reply) => { + runtime.force_due_scan_for_tests(ready_count); + let _ = reply.send(()); + } + #[cfg(test)] + ScheduleDomainCommand::PanicForTests => { + panic!("test Schedule domain actor panic"); + } + #[cfg(test)] + ScheduleDomainCommand::BlockForTests(entered, release) => { + let _ = entered.send(()); + let _ = release.recv(); + } + } + } +} diff --git a/src/domains/schedule/sink/mailbox_sink_impl.rs b/src/domains/schedule/sink/mailbox_sink_impl.rs deleted file mode 100644 index 6bd16b0c..00000000 --- a/src/domains/schedule/sink/mailbox_sink_impl.rs +++ /dev/null @@ -1,769 +0,0 @@ -use super::model::{ - DeliveryError, Envelope, MailboxSink, Ordering, ScheduleDomainActor, ScheduleDomainCommand, - ScheduleDomainRuntime, ScheduleDomainSink, ScheduleSubscription, ScheduleSubscriptionSet, -}; -#[cfg(test)] -use crate::dispatch::protocol::frame_context::FrameContext; -use crate::runtime::{Actor, Context}; - -impl MailboxSink for ScheduleDomainSink { - fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { - self.actor - .try_send(ScheduleDomainCommand::Deliver(envelope)) - } - - fn deliver_high_priority(&self, envelope: Envelope) -> Result<(), DeliveryError> { - self.actor - .try_send_high_priority(ScheduleDomainCommand::Deliver(envelope)) - } -} - -impl Actor for ScheduleDomainActor { - type Message = ScheduleDomainCommand; - - fn receive(&mut self, msg: Self::Message, _ctx: &mut Context) { - let runtime = self.state.runtime(); - match msg { - ScheduleDomainCommand::Deliver(envelope) => { - if let Err(error) = runtime.deliver_envelope(&envelope) { - tracing::warn!(domain = "schedule", error = %error, "Schedule actor delivery failed"); - } - } - ScheduleDomainCommand::CleanupSession(session_id) => { - runtime.unsubscribe_all(session_id); - } - ScheduleDomainCommand::ReadLiveCounts(reply) => { - let _ = reply.send(runtime.live_counts()); - } - ScheduleDomainCommand::ReadPendingClaims(route_family, reply) => { - let _ = reply.send(runtime.admin_pending_claims(route_family)); - } - ScheduleDomainCommand::RefreshAdminSnapshotIfDirty(reply) => { - runtime.refresh_admin_snapshot_if_dirty(); - let _ = reply.send(()); - } - ScheduleDomainCommand::ScanDueSchedules => { - runtime.scan_due_schedules(); - } - ScheduleDomainCommand::PreloadPersistedFamilies(reply) => { - let _ = reply.send(runtime.preload_persisted_families()); - } - ScheduleDomainCommand::BenchPublishEvent(event, reply) => { - runtime.bench_publish_event(&event); - let _ = reply.send(()); - } - ScheduleDomainCommand::ForceDueScanForTests(ready_count, reply) => { - runtime.force_due_scan_for_tests(ready_count); - let _ = reply.send(()); - } - #[cfg(test)] - ScheduleDomainCommand::PanicForTests => { - panic!("test Schedule domain actor panic"); - } - #[cfg(test)] - ScheduleDomainCommand::BlockForTests(entered, release) => { - let _ = entered.send(()); - let _ = release.recv(); - } - } - } -} - -impl ScheduleDomainRuntime<'_> { - pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { - if self.handle_cleanup_envelope(envelope) { - return Ok(()); - } - self.ensure_active()?; - - if self.handle_domain_publish_envelope(envelope) { - return Ok(()); - } - - Self::log_delivery(envelope); - - let Some(request) = Self::extract_request(envelope)? else { - return Ok(()); - }; - let meta = request.meta; - let request_started = self.record_request_start(); - - if !Self::valid_request_envelope(envelope, meta) { - let response = crate::domains::schedule::ScheduleResponse::Error( - crate::domains::schedule::ScheduleFailure::new( - crate::domains::schedule::ScheduleFailureCategory::InvalidTarget, - "route family mismatch", - ), - ); - let response_meta = Self::response_meta_for_source(envelope, meta); - self.route_schedule_response(envelope, response_meta, &response, request_started); - return Ok(()); - } - - let Some(schedule_msg) = - self.parse_request_message(envelope, meta, request.message, request_started) - else { - return Ok(()); - }; - - if !Self::valid_schedule_message(envelope, meta, &schedule_msg) { - let response = crate::domains::schedule::ScheduleResponse::Error( - crate::domains::schedule::ScheduleFailure::new( - crate::domains::schedule::ScheduleFailureCategory::InvalidTarget, - "route family mismatch", - ), - ); - let response_meta = Self::response_meta_for_source(envelope, meta); - self.route_schedule_response(envelope, response_meta, &response, request_started); - return Ok(()); - } - - let route_addr = envelope.destination(); - let route_family = *route_addr.family(); - - let Some((response, schedule_snapshot_dirty)) = self.dispatch_schedule_message( - envelope, - meta, - request_started, - route_family, - schedule_msg, - ) else { - return Ok(()); - }; - - if schedule_snapshot_dirty { - self.schedule_admin_snapshot(false); - } - - self.route_schedule_response(envelope, meta, &response, request_started); - - Ok(()) - } - - fn handle_cleanup_envelope(&self, envelope: &Envelope) -> bool { - if let Some(cleanup) = envelope.payload::() { - self.unsubscribe_all(cleanup.session_id); - return true; - } - - false - } - - fn ensure_active(&self) -> Result<(), DeliveryError> { - if !self.active.load(Ordering::Relaxed) { - return Err(DeliveryError::ActorStopped); - } - - Ok(()) - } - - fn handle_domain_publish_envelope(&self, envelope: &Envelope) -> bool { - if let Some(event) = envelope.payload::() { - if *envelope.destination().family() != event.family_id { - self.core - .live_publish_failures - .fetch_add(1, Ordering::Relaxed); - return true; - } - self.handle_domain_publish(event); - return true; - } - - false - } - - fn log_delivery(envelope: &Envelope) { - tracing::debug!( - domain = "schedule", - destination = %envelope.destination(), - source = ?envelope.source(), - "Schedule domain sink: received envelope" - ); - } - - fn extract_request( - envelope: &Envelope, - ) -> Result, DeliveryError> { - if let Some(request) = Self::request_from_envelope(envelope) { - Ok(Some(request)) - } else { - tracing::warn!( - domain = "schedule", - "Envelope payload was not ScheduleClientRequest" - ); - Err(DeliveryError::ActorStopped) - } - } - - fn record_request_start(&self) -> Option { - self.core - .metrics - .as_ref() - .map(crate::domains::schedule::ScheduleMetrics::record_request_start) - } - - fn parse_request_message( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - message: Result< - crate::domains::schedule::ScheduleMessage, - crate::domains::schedule::ScheduleFailure, - >, - request_started: Option, - ) -> Option { - match message { - Ok(message) => Some(message), - Err(error) => { - tracing::warn!( - domain = "schedule", - error = %error, - "Failed to parse schedule message" - ); - let response = crate::domains::schedule::ScheduleResponse::Error(error); - let response_meta = Self::response_meta_for_source(envelope, meta); - self.route_schedule_response(envelope, response_meta, &response, request_started); - None - } - } - } - - fn dispatch_schedule_message( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - request_started: Option, - route_family: crate::runtime::routing::RouteFamily, - schedule_msg: crate::domains::schedule::ScheduleMessage, - ) -> Option<(crate::domains::schedule::ScheduleResponse, bool)> { - use crate::domains::schedule::ScheduleResponse; - - match &schedule_msg { - crate::domains::schedule::ScheduleMessage::Subscribe { - family_id, - route, - session_id, - subscriber, - } => { - return Some(( - self.apply_subscribe_message( - *family_id, - route, - *session_id, - subscriber.clone(), - ), - false, - )); - } - crate::domains::schedule::ScheduleMessage::Unsubscribe { - family_id, - route, - session_id, - .. - } => { - return Some(( - self.apply_unsubscribe_message(*family_id, route, *session_id), - false, - )); - } - crate::domains::schedule::ScheduleMessage::UnsubscribeAll { session_id, .. } => { - self.unsubscribe_all(*session_id); - return Some((ScheduleResponse::Ok, false)); - } - _ => {} - } - - let mut actors = self.core.actors.lock(); - let actor = match self.get_or_create_actor(&mut actors, route_family) { - Ok(actor) => actor, - Err(error) => { - let response = ScheduleResponse::Error( - crate::domains::schedule::ScheduleFailure::parse(error), - ); - self.route_schedule_response(envelope, meta, &response, request_started); - return None; - } - }; - - Some(self.apply_schedule_message(actor, schedule_msg)) - } - - fn apply_schedule_message( - &self, - actor: &mut crate::domains::schedule::ScheduleActor, - schedule_msg: crate::domains::schedule::ScheduleMessage, - ) -> (crate::domains::schedule::ScheduleResponse, bool) { - use crate::domains::schedule::{ScheduleMessage, ScheduleResponse}; - - match schedule_msg { - ScheduleMessage::Create { - route, - cron, - delivery_mode, - payload, - } => Self::apply_create_message(actor, route, cron, delivery_mode, payload), - ScheduleMessage::CreateBatch { entries } => { - Self::apply_create_batch_message(actor, entries) - } - ScheduleMessage::Cancel { route } => Self::apply_cancel_message(actor, &route), - ScheduleMessage::List { offset, limit } => { - let (entries, total_count) = actor.list_entries(offset, limit); - - ( - ScheduleResponse::ListDefs { - entries, - total_count, - }, - false, - ) - } - ScheduleMessage::ListV2 { cursor, limit } => { - let (entries, has_more, continuation) = - actor.list_entries_v2(cursor.as_deref(), limit); - ( - ScheduleResponse::ListPage { - entries, - has_more, - continuation, - }, - false, - ) - } - ScheduleMessage::Subscribe { - family_id, - route, - session_id, - subscriber, - } => ( - self.apply_subscribe_message(family_id, &route, session_id, subscriber), - false, - ), - ScheduleMessage::Unsubscribe { - family_id, - route, - session_id, - .. - } => ( - self.apply_unsubscribe_message(family_id, &route, session_id), - false, - ), - ScheduleMessage::UnsubscribeAll { session_id, .. } => { - self.unsubscribe_all(session_id); - (ScheduleResponse::Ok, false) - } - } - } - - fn apply_create_message( - actor: &mut crate::domains::schedule::ScheduleActor, - route: String, - cron: String, - delivery_mode: crate::domains::schedule::ScheduleDeliveryMode, - payload: bytes::Bytes, - ) -> (crate::domains::schedule::ScheduleResponse, bool) { - use crate::domains::schedule::{ScheduleFailure, ScheduleResponse}; - - if let Some(failure) = - crate::domains::schedule::definition_validation::schedule_definition_failure( - &route, &cron, - ) - { - return (ScheduleResponse::Error(failure), false); - } - - match actor.create_schedule_with_mode(route, cron, delivery_mode, payload) { - Ok(changed) => (ScheduleResponse::Ok, changed), - Err(error) => ( - ScheduleResponse::Error(ScheduleFailure::parse(error)), - false, - ), - } - } - - fn apply_create_batch_message( - actor: &mut crate::domains::schedule::ScheduleActor, - entries: Vec, - ) -> (crate::domains::schedule::ScheduleResponse, bool) { - use crate::domains::schedule::{ScheduleFailure, ScheduleResponse}; - - if let Some(failure) = entries.iter().find_map(|entry| { - crate::domains::schedule::definition_validation::schedule_definition_failure( - &entry.route, - &entry.cron, - ) - }) { - return (ScheduleResponse::Error(failure), false); - } - - match actor.create_schedules(entries) { - Ok(changed) => (ScheduleResponse::Ok, changed > 0), - Err(error) => ( - ScheduleResponse::Error(ScheduleFailure::parse(error)), - false, - ), - } - } - - fn apply_cancel_message( - actor: &mut crate::domains::schedule::ScheduleActor, - route: &str, - ) -> (crate::domains::schedule::ScheduleResponse, bool) { - use crate::domains::schedule::{ - ScheduleFailure, ScheduleFailureCategory, ScheduleResponse, - }; - - if let Err(error) = - crate::domains::schedule::protocol::validate_concrete_schedule_route(route) - { - return ( - ScheduleResponse::Error(ScheduleFailure::new( - ScheduleFailureCategory::InvalidTarget, - error, - )), - false, - ); - } - - match actor.delete_schedule(route) { - Ok(removed) => (ScheduleResponse::Ok, removed), - Err(error) => ( - ScheduleResponse::Error(ScheduleFailure::parse(error)), - false, - ), - } - } - - fn apply_subscribe_message( - &self, - family_id: crate::runtime::routing::RouteFamily, - route: &crate::runtime::routing::Route, - session_id: u64, - subscriber: crate::runtime::routing::RouteAddress, - ) -> crate::domains::schedule::ScheduleResponse { - use crate::domains::schedule::{ - ScheduleFailure, ScheduleFailureCategory, ScheduleResponse, - }; - - match crate::runtime::DomainKind::Schedule - .descriptor() - .compile_registration_pattern(route.as_str()) - { - Ok(pattern) => { - self.insert_schedule_subscription(family_id, route, session_id, subscriber, pattern) - } - Err(error) => ScheduleResponse::Error(ScheduleFailure::new( - ScheduleFailureCategory::InvalidSubscriptionPattern, - error, - )), - } - } - - fn insert_schedule_subscription( - &self, - family_id: crate::runtime::routing::RouteFamily, - route: &crate::runtime::routing::Route, - session_id: u64, - subscriber: crate::runtime::routing::RouteAddress, - pattern: crate::runtime::matcher::Pattern, - ) -> crate::domains::schedule::ScheduleResponse { - use crate::domains::schedule::{ - ScheduleFailure, ScheduleFailureCategory, ScheduleResponse, - }; - - let fam_id = family_id.as_u64(); - let mut families = self.core.sub_families.lock(); - let state = families - .entry(fam_id) - .or_insert_with(ScheduleSubscriptionSet::new); - - let sub_id = if let Some(id) = state.find_existing_id(session_id, route.as_str()) { - tracing::debug!( - domain = "schedule", - session = session_id, - subscription_id = id, - route = route.as_str(), - "Schedule subscription already exists (idempotent)" - ); - id - } else { - if state - .subscriptions - .wildcard_registration_limit_reached(session_id, &pattern) - { - return ScheduleResponse::Error(ScheduleFailure::new( - ScheduleFailureCategory::SubscriptionLimit, - format!( - "wildcard subscription limit exceeded ({} per session)", - crate::domains::subscription_state::MAX_WILDCARD_REGISTRATIONS_PER_SESSION - ), - )); - } - let Ok(new_id) = self.core.next_sub_id.fetch_update( - Ordering::Relaxed, - Ordering::Relaxed, - |current| current.checked_add(1), - ) else { - let state_empty = state.is_empty(); - if state_empty { - families.remove(&fam_id); - } - return ScheduleResponse::Error(ScheduleFailure::new( - ScheduleFailureCategory::SubscriptionLimit, - "subscription ID space exhausted", - )); - }; - state.insert( - family_id, - ScheduleSubscription { - pattern, - session_id, - subscription_id: new_id, - subscriber, - }, - ); - - tracing::debug!( - domain = "schedule", - session = session_id, - subscription_id = new_id, - route = route.as_str(), - "Schedule subscription added" - ); - new_id - }; - - ScheduleResponse::SubscribeOk { - subscription_id: sub_id, - } - } - - fn apply_unsubscribe_message( - &self, - family_id: crate::runtime::routing::RouteFamily, - route: &crate::runtime::routing::Route, - session_id: u64, - ) -> crate::domains::schedule::ScheduleResponse { - use crate::domains::schedule::{ - ScheduleFailure, ScheduleFailureCategory, ScheduleResponse, - }; - - if let Err(error) = crate::runtime::DomainKind::Schedule - .descriptor() - .compile_registration_pattern(route.as_str()) - { - return ScheduleResponse::Error(ScheduleFailure::new( - ScheduleFailureCategory::InvalidSubscriptionPattern, - error, - )); - } - - let fam_id = family_id.as_u64(); - let mut families = self.core.sub_families.lock(); - let remove_family = if let Some(state) = families.get_mut(&fam_id) { - state.remove_session_route(family_id, session_id, route.as_str()); - state.is_empty() - } else { - false - }; - if remove_family { - families.remove(&fam_id); - } - ScheduleResponse::Ok - } - - fn request_from_envelope( - envelope: &Envelope, - ) -> Option { - if let Some(request) = envelope.payload::() - { - return Some(request.clone()); - } - - #[cfg(test)] - { - let frame_ctx = envelope.payload::()?.clone(); - let subscriber = envelope.source().cloned().unwrap_or_else(|| { - crate::runtime::routing::RouteAddress::new( - *envelope.destination().family(), - crate::runtime::routing::Route::new(format!( - "inbox://session/{}", - frame_ctx.session_id - )), - ) - }); - let meta = crate::runtime::ClientFrameMeta::new( - frame_ctx.session_id, - test_client_channel_from_protocol(frame_ctx.channel_id), - frame_ctx.msg_type.as_u16(), - frame_ctx.route_family, - ); - let parsed = crate::dispatch::protocol::schedule_codec::parse_request( - &frame_ctx, - &frame_ctx.payload, - *envelope.destination().family(), - crate::session::SessionId(frame_ctx.session_id), - subscriber, - ); - Some(crate::domains::schedule::ScheduleClientRequest::new( - meta, parsed, - )) - } - - #[cfg(not(test))] - { - None - } - } - - fn route_schedule_response( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - response: &crate::domains::schedule::ScheduleResponse, - request_started: Option, - ) { - #[cfg(test)] - let response_ctx = { - let mut payload_encoder = - crate::dispatch::protocol::payload_codec::PayloadEncoder::with_capacity(256); - let response_bytes = crate::dispatch::protocol::schedule_codec::encode_response_into( - &mut payload_encoder, - meta.message_type, - response, - ); - FrameContext::new( - meta.session_id, - test_protocol_channel_from_client(meta.channel), - crate::dispatch::protocol::tlv::MessageType::new(meta.message_type), - bytes::Bytes::from(response_bytes), - meta.route_family, - ) - }; - - #[cfg(not(test))] - let response_ctx = - crate::domains::schedule::ScheduleClientResponse::new(meta, response.clone()); - - if let Some(response_envelope) = envelope.try_reply_to(response_ctx) { - if let Err(error) = self.core.router.route(response_envelope) { - if let Some(metrics) = self.core.metrics.as_ref() { - metrics.record_response_drop(); - } else { - crate::observability::counter_inc( - crate::domains::schedule::metrics::METRIC_RESPONSE_DROPS_TOTAL, - ); - } - tracing::warn!( - domain = "schedule", - session_id = meta.session_id, - route_family = meta.route_family.as_u64(), - error = %error, - "Dropped best-effort Schedule response" - ); - } - } - - if let (Some(metrics), Some(started_at)) = (self.core.metrics.as_ref(), request_started) { - if Self::schedule_response_is_failure(response) { - metrics.record_failure(started_at); - } else { - metrics.record_success(started_at); - } - } - } - - fn valid_request_envelope(envelope: &Envelope, meta: crate::runtime::ClientFrameMeta) -> bool { - meta.route_family == *envelope.destination().family() - && envelope - .source() - .is_none_or(|source| *source.family() == meta.route_family) - } - - fn response_meta_for_source( - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - ) -> crate::runtime::ClientFrameMeta { - envelope.source().map_or(meta, |source| { - let mut response_meta = meta; - response_meta.route_family = *source.family(); - response_meta - }) - } - - fn valid_schedule_message( - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - message: &crate::domains::schedule::ScheduleMessage, - ) -> bool { - use crate::domains::schedule::ScheduleMessage; - - match message { - ScheduleMessage::Subscribe { - family_id, - session_id, - subscriber, - .. - } - | ScheduleMessage::Unsubscribe { - family_id, - session_id, - subscriber, - .. - } => { - *family_id == meta.route_family - && *session_id == meta.session_id - && *subscriber.family() == *family_id - && envelope.source().is_none_or(|source| source == subscriber) - } - ScheduleMessage::UnsubscribeAll { - session_id, - subscriber, - } => { - *session_id == meta.session_id - && *subscriber.family() == meta.route_family - && envelope.source().is_none_or(|source| source == subscriber) - } - ScheduleMessage::Create { .. } - | ScheduleMessage::CreateBatch { .. } - | ScheduleMessage::Cancel { .. } - | ScheduleMessage::List { .. } - | ScheduleMessage::ListV2 { .. } => true, - } - } -} - -#[cfg(test)] -fn test_client_channel_from_protocol( - channel: crate::dispatch::protocol::frame::ChannelId, -) -> crate::runtime::ClientChannel { - match channel { - crate::dispatch::protocol::frame::ChannelId::Control => { - crate::runtime::ClientChannel::Control - } - crate::dispatch::protocol::frame::ChannelId::Pub => crate::runtime::ClientChannel::Pub, - crate::dispatch::protocol::frame::ChannelId::Sub => crate::runtime::ClientChannel::Sub, - crate::dispatch::protocol::frame::ChannelId::Rpc => crate::runtime::ClientChannel::Rpc, - crate::dispatch::protocol::frame::ChannelId::Lease => crate::runtime::ClientChannel::Lease, - crate::dispatch::protocol::frame::ChannelId::Internal => { - crate::runtime::ClientChannel::Internal - } - } -} - -#[cfg(test)] -fn test_protocol_channel_from_client( - channel: crate::runtime::ClientChannel, -) -> crate::dispatch::protocol::frame::ChannelId { - match channel { - crate::runtime::ClientChannel::Control => { - crate::dispatch::protocol::frame::ChannelId::Control - } - crate::runtime::ClientChannel::Pub => crate::dispatch::protocol::frame::ChannelId::Pub, - crate::runtime::ClientChannel::Sub => crate::dispatch::protocol::frame::ChannelId::Sub, - crate::runtime::ClientChannel::Rpc => crate::dispatch::protocol::frame::ChannelId::Rpc, - crate::runtime::ClientChannel::Lease => crate::dispatch::protocol::frame::ChannelId::Lease, - crate::runtime::ClientChannel::Internal => { - crate::dispatch::protocol::frame::ChannelId::Internal - } - } -} diff --git a/src/domains/schedule/sink/mod.rs b/src/domains/schedule/sink/mod.rs index fe92c4c8..b96e1500 100644 --- a/src/domains/schedule/sink/mod.rs +++ b/src/domains/schedule/sink/mod.rs @@ -1,12 +1,18 @@ +mod cleanup; +mod definitions; +mod delivery; mod delivery_strategy; -mod domain_sink_impl; -mod mailbox_sink_impl; +mod facade; +mod ingress; +mod mailbox; mod model; +mod observability; +mod responses; +mod subscriptions; #[cfg(test)] mod test_helpers; -pub use domain_sink_impl::ScheduleObservability; -pub(crate) use domain_sink_impl::DEFAULT_SCHEDULE_PRELOAD_TIMEOUT; +pub(crate) use facade::DEFAULT_SCHEDULE_PRELOAD_TIMEOUT; pub use model::ScheduleDomainSink; #[cfg(test)] diff --git a/src/domains/schedule/sink/model.rs b/src/domains/schedule/sink/model.rs index c25f94c3..5f2bad3a 100644 --- a/src/domains/schedule/sink/model.rs +++ b/src/domains/schedule/sink/model.rs @@ -11,6 +11,10 @@ pub(super) use std::time::Instant; pub(super) const SCHEDULE_ADMIN_SNAPSHOT_INTERVAL_US: u64 = 250_000; pub(super) const EXECUTIONS_WINDOW_MS: u64 = 60_000; +pub(super) fn duration_millis(duration: std::time::Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + pub(super) fn now_epoch_ms() -> u64 { u64::try_from( std::time::SystemTime::now() @@ -145,7 +149,11 @@ pub(super) struct ScheduleDomainCore { pub(super) actors: Mutex< HashMap, >, - pub(super) sub_families: Mutex>, + pub(super) sub_families: + Mutex>, + /// Sessions disconnect cleanup has already run for; guards against a + /// stale queued request recreating a subscription. See `cleanup.rs`. + pub(super) cleaned_up_sessions: Mutex, pub(super) next_sub_id: AtomicU64, pub(super) router: Arc, #[cfg_attr(feature = "bench-no-snapshot", allow(dead_code))] diff --git a/src/domains/schedule/sink/domain_sink_impl/admin_runtime.rs b/src/domains/schedule/sink/observability.rs similarity index 81% rename from src/domains/schedule/sink/domain_sink_impl/admin_runtime.rs rename to src/domains/schedule/sink/observability.rs index 1afaa2b0..ba97addc 100644 --- a/src/domains/schedule/sink/domain_sink_impl/admin_runtime.rs +++ b/src/domains/schedule/sink/observability.rs @@ -1,18 +1,23 @@ -use super::super::model::{ +//! Admin read-model projection and metrics glue: when and how live Schedule +//! state is mirrored into the admin snapshot and metric gauges. +//! +//! Projection failure must never affect domain correctness. + +use super::model::{ now_epoch_ms, schedule_admin_snapshot_due, Ordering, ScheduleDomainRuntime, ScheduleLiveCounts, EXECUTIONS_WINDOW_MS, }; impl ScheduleDomainRuntime<'_> { - pub fn subscription_count(&self) -> usize { + pub(super) fn subscription_count(&self) -> usize { let families = self.core.sub_families.lock(); families .values() - .map(super::super::model::ScheduleSubscriptionSet::subscription_count) + .map(super::model::ScheduleSubscriptionSet::subscription_count) .sum() } - pub fn schedule_count(&self) -> usize { + pub(super) fn schedule_count(&self) -> usize { let actors = self.core.actors.lock(); actors .values() @@ -20,7 +25,7 @@ impl ScheduleDomainRuntime<'_> { .sum() } - pub fn pending_fire_count(&self) -> usize { + pub(super) fn pending_fire_count(&self) -> usize { let actors = self.core.actors.lock(); actors .values() @@ -29,7 +34,7 @@ impl ScheduleDomainRuntime<'_> { } /// Legacy metric name: counts acknowledged live handoffs over the last minute. - pub fn executions_per_minute(&self) -> f64 { + pub(super) fn executions_per_minute(&self) -> f64 { let now_ms = now_epoch_ms(); let cutoff = now_ms.saturating_sub(EXECUTIONS_WINDOW_MS); let mut deque = self.core.recent_acknowledgement_ms.lock(); @@ -39,15 +44,15 @@ impl ScheduleDomainRuntime<'_> { f64::from(u32::try_from(deque.len()).unwrap_or(u32::MAX)) } - pub fn notify_failure_count(&self) -> u64 { + pub(super) fn notify_failure_count(&self) -> u64 { self.core.live_publish_failures.load(Ordering::Relaxed) } - pub fn ack_failure_count(&self) -> u64 { + pub(super) fn ack_failure_count(&self) -> u64 { self.core.ack_failures.load(Ordering::Relaxed) } - pub fn pending_ack_retry_count(&self) -> usize { + pub(super) fn pending_ack_retry_count(&self) -> usize { let pending_ack_retries = self.core.pending_ack_retries.lock(); pending_ack_retries .values() @@ -55,7 +60,7 @@ impl ScheduleDomainRuntime<'_> { .sum() } - pub fn admin_pending_claims( + pub(super) fn admin_pending_claims( &self, route_family: crate::runtime::routing::RouteFamily, ) -> Vec { @@ -66,7 +71,7 @@ impl ScheduleDomainRuntime<'_> { .unwrap_or_default() } - pub fn oldest_pending_claim_age_seconds(&self) -> u64 { + pub(super) fn oldest_pending_claim_age_seconds(&self) -> u64 { let now_ms = now_epoch_ms(); let actors = self.core.actors.lock(); actors @@ -76,7 +81,7 @@ impl ScheduleDomainRuntime<'_> { .unwrap_or(0) } - pub fn overdue_normalization_count(&self) -> u64 { + pub(super) fn overdue_normalization_count(&self) -> u64 { let actors = self.core.actors.lock(); actors .values() @@ -84,7 +89,7 @@ impl ScheduleDomainRuntime<'_> { .sum() } - pub(in crate::domains::schedule::sink) fn live_counts(&self) -> ScheduleLiveCounts { + pub(super) fn live_counts(&self) -> ScheduleLiveCounts { ScheduleLiveCounts { subscriptions: self.subscription_count(), schedules: self.schedule_count(), @@ -120,7 +125,7 @@ impl ScheduleDomainRuntime<'_> { } } - pub(in crate::domains::schedule::sink) fn schedule_response_is_failure( + pub(super) fn schedule_response_is_failure( response: &crate::domains::schedule::ScheduleResponse, ) -> bool { matches!( @@ -129,7 +134,7 @@ impl ScheduleDomainRuntime<'_> { ) } - pub(in crate::domains::schedule::sink) fn schedule_admin_snapshot(&self, force: bool) { + pub(super) fn schedule_admin_snapshot(&self, force: bool) { self.core.snapshot_dirty.store(true, Ordering::Relaxed); self.maybe_sync_admin_snapshot(force); } @@ -176,12 +181,11 @@ impl ScheduleDomainRuntime<'_> { self.core.snapshot_syncing.store(false, Ordering::Release); } - pub(crate) fn refresh_admin_snapshot_if_dirty(&self) { + pub(super) fn refresh_admin_snapshot_if_dirty(&self) { self.maybe_sync_admin_snapshot(true); } - #[doc(hidden)] - pub fn bench_publish_event(&self, event: &crate::runtime::DomainPublishEvent) { + pub(super) fn bench_publish_event(&self, event: &crate::runtime::DomainPublishEvent) { self.handle_domain_publish(event); } } diff --git a/src/domains/schedule/sink/responses.rs b/src/domains/schedule/sink/responses.rs new file mode 100644 index 00000000..7b4e0b47 --- /dev/null +++ b/src/domains/schedule/sink/responses.rs @@ -0,0 +1,64 @@ +//! Response encoding and best-effort routing back to the requester. + +use super::model::{Envelope, ScheduleDomainRuntime}; +#[cfg(test)] +use crate::dispatch::protocol::frame_context::FrameContext; + +impl ScheduleDomainRuntime<'_> { + pub(super) fn route_schedule_response( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + response: &crate::domains::schedule::ScheduleResponse, + request_started: Option, + ) { + #[cfg(test)] + let response_ctx = { + let mut payload_encoder = + crate::dispatch::protocol::payload_codec::PayloadEncoder::with_capacity(256); + let response_bytes = crate::dispatch::protocol::schedule_codec::encode_response_into( + &mut payload_encoder, + meta.message_type, + response, + ); + FrameContext::new( + meta.session_id, + crate::protocol::test_support::channel_id_from_client(meta.channel), + crate::dispatch::protocol::tlv::MessageType::new(meta.message_type), + bytes::Bytes::from(response_bytes), + meta.route_family, + ) + }; + + #[cfg(not(test))] + let response_ctx = + crate::domains::schedule::ScheduleClientResponse::new(meta, response.clone()); + + if let Some(response_envelope) = envelope.try_reply_to(response_ctx) { + if let Err(error) = self.core.router.route(response_envelope) { + if let Some(metrics) = self.core.metrics.as_ref() { + metrics.record_response_drop(); + } else { + crate::observability::counter_inc( + crate::domains::schedule::metrics::METRIC_RESPONSE_DROPS_TOTAL, + ); + } + tracing::warn!( + domain = "schedule", + session_id = meta.session_id, + route_family = meta.route_family.as_u64(), + error = %error, + "Dropped best-effort Schedule response" + ); + } + } + + if let (Some(metrics), Some(started_at)) = (self.core.metrics.as_ref(), request_started) { + if Self::schedule_response_is_failure(response) { + metrics.record_failure(started_at); + } else { + metrics.record_success(started_at); + } + } + } +} diff --git a/src/domains/schedule/sink/subscriptions.rs b/src/domains/schedule/sink/subscriptions.rs new file mode 100644 index 00000000..1cb6634a --- /dev/null +++ b/src/domains/schedule/sink/subscriptions.rs @@ -0,0 +1,144 @@ +//! Subscribe/unsubscribe message handling: mutation of the live subscription +//! index in response to a client request. + +use super::model::{ + Ordering, ScheduleDomainRuntime, ScheduleSubscription, ScheduleSubscriptionSet, +}; + +impl ScheduleDomainRuntime<'_> { + pub(super) fn apply_subscribe_message( + &self, + family_id: crate::runtime::routing::RouteFamily, + route: &crate::runtime::routing::Route, + session_id: u64, + subscriber: crate::runtime::routing::RouteAddress, + ) -> crate::domains::schedule::ScheduleResponse { + use crate::domains::schedule::{ + ScheduleFailure, ScheduleFailureCategory, ScheduleResponse, + }; + + match crate::runtime::DomainKind::Schedule + .descriptor() + .compile_registration_pattern(route.as_str()) + { + Ok(pattern) => { + self.insert_schedule_subscription(family_id, route, session_id, subscriber, pattern) + } + Err(error) => ScheduleResponse::Error(ScheduleFailure::new( + ScheduleFailureCategory::InvalidSubscriptionPattern, + error, + )), + } + } + + fn insert_schedule_subscription( + &self, + family_id: crate::runtime::routing::RouteFamily, + route: &crate::runtime::routing::Route, + session_id: u64, + subscriber: crate::runtime::routing::RouteAddress, + pattern: crate::runtime::matcher::Pattern, + ) -> crate::domains::schedule::ScheduleResponse { + use crate::domains::schedule::{ + ScheduleFailure, ScheduleFailureCategory, ScheduleResponse, + }; + + let mut families = self.core.sub_families.lock(); + let state = families + .entry(family_id) + .or_insert_with(ScheduleSubscriptionSet::new); + + let sub_id = if let Some(id) = state.find_existing_id(session_id, route.as_str()) { + tracing::debug!( + domain = "schedule", + session = session_id, + subscription_id = id, + route = route.as_str(), + "Schedule subscription already exists (idempotent)" + ); + id + } else { + if state + .subscriptions + .wildcard_registration_limit_reached(session_id, &pattern) + { + return ScheduleResponse::Error(ScheduleFailure::new( + ScheduleFailureCategory::SubscriptionLimit, + format!( + "wildcard subscription limit exceeded ({} per session)", + crate::domains::subscription_state::MAX_WILDCARD_REGISTRATIONS_PER_SESSION + ), + )); + } + let Ok(new_id) = self.core.next_sub_id.fetch_update( + Ordering::Relaxed, + Ordering::Relaxed, + |current| current.checked_add(1), + ) else { + let state_empty = state.is_empty(); + if state_empty { + families.remove(&family_id); + } + return ScheduleResponse::Error(ScheduleFailure::new( + ScheduleFailureCategory::SubscriptionLimit, + "subscription ID space exhausted", + )); + }; + state.insert( + family_id, + ScheduleSubscription { + pattern, + session_id, + subscription_id: new_id, + subscriber, + }, + ); + + tracing::debug!( + domain = "schedule", + session = session_id, + subscription_id = new_id, + route = route.as_str(), + "Schedule subscription added" + ); + new_id + }; + + ScheduleResponse::SubscribeOk { + subscription_id: sub_id, + } + } + + pub(super) fn apply_unsubscribe_message( + &self, + family_id: crate::runtime::routing::RouteFamily, + route: &crate::runtime::routing::Route, + session_id: u64, + ) -> crate::domains::schedule::ScheduleResponse { + use crate::domains::schedule::{ + ScheduleFailure, ScheduleFailureCategory, ScheduleResponse, + }; + + if let Err(error) = crate::runtime::DomainKind::Schedule + .descriptor() + .compile_registration_pattern(route.as_str()) + { + return ScheduleResponse::Error(ScheduleFailure::new( + ScheduleFailureCategory::InvalidSubscriptionPattern, + error, + )); + } + + let mut families = self.core.sub_families.lock(); + let remove_family = if let Some(state) = families.get_mut(&family_id) { + state.remove_session_route(family_id, session_id, route.as_str()); + state.is_empty() + } else { + false + }; + if remove_family { + families.remove(&family_id); + } + ScheduleResponse::Ok + } +} diff --git a/src/domains/schedule/sink/test_helpers.rs b/src/domains/schedule/sink/test_helpers.rs index 2f48837e..bc923b42 100644 --- a/src/domains/schedule/sink/test_helpers.rs +++ b/src/domains/schedule/sink/test_helpers.rs @@ -61,7 +61,7 @@ impl ScheduleDomainSink { pub(super) fn insert_subscriptions_for_tests( &self, - family_id: u64, + family_id: crate::runtime::routing::RouteFamily, subscriptions: ScheduleSubscriptionSet, ) { self.state diff --git a/src/domains/schedule/sink/tests/correctness.rs b/src/domains/schedule/sink/tests/correctness.rs index 4895c899..6d32c490 100644 --- a/src/domains/schedule/sink/tests/correctness.rs +++ b/src/domains/schedule/sink/tests/correctness.rs @@ -84,6 +84,68 @@ fn should_reject_schedule_subscription_when_identity_does_not_match_request() { assert!(sink.subscriptions_are_empty_for_tests()); } +#[test] +fn should_reject_stale_subscribe_after_disconnect_cleanup_marks_session() { + // Arrange + let family = RouteFamily::new(1); + let session_id = 9; + let route = "schedule://acme/jobs/nightly/run"; + let source = RouteAddress::new(family, Route::new("inbox://session/9")); + let destination = RouteAddress::new(family, Route::new(route)); + let mailbox = Arc::new(Mailbox::new(8)); + let router = Arc::new(Router::new()); + router.register(source.clone(), mailbox.clone()); + let sink = new_correctness_schedule_sink(router); + let subscribe = || { + crate::domains::schedule::ScheduleClientRequest::new( + crate::runtime::ClientFrameMeta::new( + session_id, + crate::runtime::ClientChannel::Sub, + 703, + family, + ), + Ok(crate::domains::schedule::ScheduleMessage::Subscribe { + family_id: family, + route: Route::new(route), + session_id, + subscriber: source.clone(), + }), + ) + }; + sink.deliver(Envelope::from_route( + source.clone(), + destination.clone(), + subscribe(), + )) + .expect("subscribe before disconnect"); + let _subscribe_ack = receive_envelope(&mailbox, "subscribe ack envelope"); + wait_for_subscription_count(&sink, 1); + + // Act: cleanup for this session runs and completes before the stale + // subscribe below is processed - equivalent to what the high-priority + // mailbox lane guarantees a real disconnect races against a queued + // normal-lane request. + sink.deliver(Envelope::new( + RouteAddress::new(family, Route::new("schedule://cleanup")), + crate::runtime::SessionCleanup { session_id }, + )) + .expect("cleanup session"); + wait_for_subscription_count(&sink, 0); + + sink.deliver(Envelope::from_route( + source.clone(), + destination, + subscribe(), + )) + .expect("deliver stale subscribe"); + + // Assert: the stale subscribe from the now-cleaned-up session is + // rejected instead of resurrecting a subscription for it. + let error = schedule_error_message(&mailbox, "stale subscribe rejection response"); + assert_eq!(error, "session already closed"); + assert_eq!(sink.subscription_count(), 0); +} + #[test] fn should_reject_schedule_subscription_when_decoded_family_differs_from_request() { // Arrange diff --git a/src/domains/schedule/sink/tests/lifecycle_and_admin.rs b/src/domains/schedule/sink/tests/lifecycle_and_admin.rs index c4341437..303e351e 100644 --- a/src/domains/schedule/sink/tests/lifecycle_and_admin.rs +++ b/src/domains/schedule/sink/tests/lifecycle_and_admin.rs @@ -304,7 +304,7 @@ fn should_route_schedule_live_stats_through_actor_command() { subscriber: RouteAddress::new(family, Route::new("inbox://session/7")), }, ); - sink.insert_subscriptions_for_tests(family.as_u64(), subscriptions); + sink.insert_subscriptions_for_tests(family, subscriptions); sink.push_recent_acknowledgement_for_tests(now_epoch_ms()); sink.set_live_publish_failures_for_tests(2); sink.set_ack_failures_for_tests(3); diff --git a/src/domains/stream/actor.rs b/src/domains/stream/actor.rs index 0407c79a..59c70e63 100644 --- a/src/domains/stream/actor.rs +++ b/src/domains/stream/actor.rs @@ -156,21 +156,22 @@ impl StreamActor { expected_offset }; - let event_size = body + let payload_bytes = body .len() .checked_add(metadata.as_ref().map_or(0, Bytes::len)) - .and_then(|size| { - size.checked_add( - discriminator - .as_ref() - .map_or(0, |value| value.as_str().len()), - ) - }) .ok_or_else(|| "event too large".to_string())?; - if event_size > MAX_EVENT_SIZE { + if payload_bytes > MAX_EVENT_SIZE { return Err("event too large".to_string()); } + let event_size = payload_bytes + .checked_add( + discriminator + .as_ref() + .map_or(0, |value| value.as_str().len()), + ) + .ok_or_else(|| "event too large".to_string())?; + let limits = self.store.batch_limits(); let next_event_count = session .staged_events @@ -532,6 +533,98 @@ mod tests { assert!(!actor.has_active_session()); } + #[test] + fn should_reject_an_append_larger_than_a_read_response_can_carry() { + // Arrange + // Every read plane frames one response as a single TLV value with a + // `u16` length, so a record whose wire cost exceeds that ceiling can + // never be replayed - and because the read accumulator refuses to + // build an unencodable response, it permanently blocks pagination at + // its own offset on the resource, area, realm, and global planes + // alike. Stream guarantees exact replay of committed history, so such + // an append must be refused rather than committed and then wedged. + let store = Arc::new(StreamStore::new(create_test_engine_with_cfs(vec![1]))); + let mut actor = StreamActor::new( + RouteFamily::new(1), + "test".to_string(), + "events".to_string(), + "orders".to_string(), + store, + ) + .expect("create actor"); + actor + .begin_append_session(10, 100, None) + .expect("begin append session"); + + // Act + let rejected = actor.append_to_session_with_discriminator_for_owner( + 10, + 100, + 0, + Bytes::from(vec![b'a'; 200_000]), + None, + None, + ); + + // Assert + let error = rejected.expect_err("an unreadable append must be refused"); + assert!( + error.contains("event too large"), + "unexpected error: {error}" + ); + assert!( + actor + .commit_session_for_owner(10, 100, StreamWriteMode::Buffered) + .is_err(), + "the refused event must not have been staged for commit" + ); + } + + #[test] + fn should_accept_an_append_at_the_published_event_size_limit() { + // Arrange + // The public limit reserves enough space for the largest valid route, + // so an event at that boundary must remain readable here. + let store = Arc::new(StreamStore::new(create_test_engine_with_cfs(vec![1]))); + let realm = "r".repeat( + crate::utils::route_shape::MAX_ROUTE_BYTES - "stream://".len() - "/events/orders".len(), + ); + let mut actor = StreamActor::new( + RouteFamily::new(1), + realm, + "events".to_string(), + "orders".to_string(), + store.clone(), + ) + .expect("create actor"); + actor + .begin_append_session(10, 100, None) + .expect("begin append session"); + let largest_body = MAX_EVENT_SIZE; + + // Act + let accepted = actor.append_to_session_with_discriminator_for_owner( + 10, + 100, + 0, + Bytes::from(vec![b'a'; largest_body]), + None, + None, + ); + actor + .commit_session_for_owner(10, 100, StreamWriteMode::Buffered) + .expect("commit the largest readable event"); + let read = actor.read(0, 10, None).expect("read it back"); + + // Assert + assert_eq!(accepted, Ok(0)); + assert_eq!( + read.items.len(), + 1, + "the largest accepted event must read back" + ); + } + #[test] fn should_preserve_active_session_given_uninitialized_staged_state() { // Arrange diff --git a/src/domains/stream/events.rs b/src/domains/stream/events.rs deleted file mode 100644 index d8052050..00000000 --- a/src/domains/stream/events.rs +++ /dev/null @@ -1,32 +0,0 @@ -/// Semantic state transitions emitted by the stream domain. -/// -/// Events are emitted after successful operations, outside the hot path. -/// They contain identifiers and timestamps only — no aggregates or metric summaries. -#[derive(Debug, Clone)] -pub enum StreamEvent { - EventAppended { - realm: String, - area: String, - resource: String, - offset: u64, - committed_at_epoch_ms: u64, - }, - Subscribed { - realm: String, - area: String, - resource: String, - session_id: u64, - }, - Unsubscribed { - realm: String, - area: String, - resource: String, - session_id: u64, - }, - WatermarkAdvanced { - realm: String, - area: String, - resource: String, - offset: u64, - }, -} diff --git a/src/domains/stream/metrics.rs b/src/domains/stream/metrics.rs index 78989cb6..958823b9 100644 --- a/src/domains/stream/metrics.rs +++ b/src/domains/stream/metrics.rs @@ -10,6 +10,12 @@ pub const METRIC_SUBSCRIPTIONS_GAUGE: &str = "fitz_stream_subscriptions_gauge"; pub const METRIC_APPEND_SESSIONS_GAUGE: &str = "fitz_stream_append_sessions_active"; pub const METRIC_RESPONSE_DROPS_TOTAL: &str = "fitz_stream_response_drops_total"; pub const METRIC_NOTIFY_DROPS_TOTAL: &str = "fitz_stream_notify_drops_total"; +/// Incremented once per route family whose handler panics and fails closed. +/// Non-fatal and scoped to that family only (see +/// `KeyedFamilyExecutor::is_family_running`) — this is the only +/// operator-visible signal for a permanently degraded realm, since a +/// per-family failure deliberately does not flip domain-wide health/liveness. +pub const METRIC_FAMILY_FAILED_CLOSED_TOTAL: &str = "fitz_stream_family_failed_closed_total"; pub const METRIC_WATERMARK_COORDINATION_DROPS_TOTAL: &str = "fitz_stream_watermark_coordination_drops_total"; pub const METRIC_MAINTENANCE_ATTEMPTS_TOTAL: &str = "fitz_stream_maintenance_attempts_total"; @@ -17,6 +23,8 @@ pub const METRIC_MAINTENANCE_FAILURES_TOTAL: &str = "fitz_stream_maintenance_fai pub const METRIC_MAINTENANCE_RETRIES_TOTAL: &str = "fitz_stream_maintenance_retries_total"; pub const METRIC_MAINTENANCE_BUCKETS_COMPACTED_TOTAL: &str = "fitz_stream_maintenance_buckets_compacted_total"; +pub const METRIC_ADMIN_PROJECTION_FAILURES_TOTAL: &str = + "fitz_stream_admin_projection_failures_total"; #[derive(Clone)] pub struct StreamMetrics { diff --git a/src/domains/stream/mod.rs b/src/domains/stream/mod.rs index dcb4bae9..13e8a5ae 100644 --- a/src/domains/stream/mod.rs +++ b/src/domains/stream/mod.rs @@ -39,8 +39,7 @@ pub mod actor; pub mod constants; -pub mod events; -pub mod metrics; +pub(crate) mod metrics; pub mod protocol; pub(crate) mod route_grammar; pub mod sink; @@ -55,7 +54,7 @@ pub use constants::{ INTERNAL_AREA_SEGMENT, INTERNAL_REALM_SEGMENT, MAX_POSTING_ENTRIES_EXAMINED, MAX_POSTING_FRAGMENTS_FETCHED, MAX_READ_ITEMS, MAX_WATERMARK_COORDINATORS, NOTICE_DEBOUNCE_MS, }; -pub use metrics::StreamMetrics; +pub(crate) use metrics::StreamMetrics; pub use protocol::{ AppendResponse, GetMetadataResponse, ReadResponse, StreamClientFrame, StreamClientNotification, StreamClientRequest, StreamClientResponse, StreamClientResponseBody, StreamDiscriminator, diff --git a/src/domains/stream/protocol.rs b/src/domains/stream/protocol.rs index 2e27cb3e..f72547b2 100644 --- a/src/domains/stream/protocol.rs +++ b/src/domains/stream/protocol.rs @@ -4,36 +4,28 @@ use bytes::Bytes; use serde::{Deserialize, Serialize}; use crate::dispatch::protocol::payload_codec::{PayloadDecoder, PayloadEncoder}; -use crate::runtime::routing::{route_exact_quad, Route, RouteAddress, RouteFamily}; +use crate::runtime::routing::{Route, RouteAddress, RouteFamily}; use crate::runtime::ClientFrameMeta; -/// Parse a stream route into (realm, area, resource, operation). -/// -/// Expected format: `{scheme}://{realm}/{area}/{resource}/{operation}` -/// or `/{realm}/{area}/{resource}/{operation}` -/// -/// # Errors -/// -/// Returns an error when `route` does not contain exactly four path segments. -pub fn parse_stream_route(route: &Route) -> Result<(String, String, String, String), String> { - route_exact_quad(route.as_str()) - .map(|parts| { - ( - parts.realm.to_string(), - parts.area.to_string(), - parts.resource.to_string(), - parts.operation.to_string(), - ) - }) - .ok_or_else(|| "Stream routes require exactly 4 segments".to_string()) -} - // ═══════════════════════════════════════════════════════════════════════════ // CONSTANTS // ═══════════════════════════════════════════════════════════════════════════ -/// Maximum size for a single event (body + metadata combined) -pub const MAX_EVENT_SIZE: usize = 1_048_576; // 1 MB +/// Conservative fixed envelope/cursor reserve for a Stream READ response. +pub(crate) const STREAM_READ_RESPONSE_ENVELOPE_OVERHEAD_BYTES: usize = 128; + +/// Conservative fixed wire overhead for one event in a Stream READ response. +pub(crate) const STREAM_READ_ITEM_FIXED_WIRE_OVERHEAD_BYTES: usize = 64; + +/// Maximum combined body and metadata size for one event. +/// +/// Stream READ responses use a `u16` TLV payload. This limit reserves the +/// conservative response/item overhead plus the maximum valid route length, +/// so every accepted event can be replayed on every valid Stream route. +pub const MAX_EVENT_SIZE: usize = (u16::MAX as usize) + - STREAM_READ_RESPONSE_ENVELOPE_OVERHEAD_BYTES + - STREAM_READ_ITEM_FIXED_WIRE_OVERHEAD_BYTES + - crate::utils::route_shape::MAX_ROUTE_BYTES; // ═══════════════════════════════════════════════════════════════════════════ // CORE DATA TYPES @@ -475,8 +467,6 @@ pub enum StreamWriteMode { Buffered, /// Sync: correctness-first, writes are committed synchronously Sync, - /// `CloudStrict`: internal broker mode for cloud provider-ack commits - CloudStrict, } /// Batch committed notification from `StreamActor` to `AreaActor` @@ -665,45 +655,6 @@ impl StreamError { mod tests { use super::*; - #[test] - fn should_parse_stream_route_with_operation() { - // Arrange - let route = Route::new("stream://acme/orders/checkout/append"); - - // Act - let result = parse_stream_route(&route).unwrap(); - - // Assert - assert_eq!(result.0, "acme"); - assert_eq!(result.1, "orders"); - assert_eq!(result.2, "checkout"); - assert_eq!(result.3, "append"); - } - - #[test] - fn should_reject_stream_route_missing_operation() { - // Arrange - let route = Route::new("stream://acme/orders/checkout"); - - // Act - let result = parse_stream_route(&route); - - // Assert - assert!(result.is_err()); - } - - #[test] - fn should_reject_stream_route_given_extra_segment() { - // Arrange - let route = Route::new("stream://acme/orders/checkout/append/extra"); - - // Act - let result = parse_stream_route(&route); - - // Assert - assert!(result.is_err()); - } - #[test] fn should_match_discriminator_when_all_clauses_match() { // Arrange diff --git a/src/domains/stream/sink/cleanup.rs b/src/domains/stream/sink/cleanup.rs new file mode 100644 index 00000000..4092081b --- /dev/null +++ b/src/domains/stream/sink/cleanup.rs @@ -0,0 +1,64 @@ +//! Disconnect cleanup: removal of all Stream state owned by one session. +//! +//! `cleanup_session` marks the session in `cleaned_up_sessions` before doing +//! any mutation, and `mailbox_sink_impl`'s envelope dispatch rejects any +//! session-mutating frame for a session already in that set - see +//! `sink/mailbox_sink_impl/envelope_dispatch.rs`. That check-before-dispatch +//! guard is what actually prevents a stale queued request from recreating +//! state after cleanup; this file only owns the mutation itself. + +use super::model::{RouteFamily, StreamDomainCore}; + +impl StreamDomainCore { + pub(in crate::domains::stream::sink) fn unsubscribe_all(&self, session_id: u64) { + let mut families = self.subscriptions.families.lock(); + for (family_id, state) in families.iter_mut() { + state.remove_session( + RouteFamily::try_from(*family_id) + .expect("stream family IDs originate from RouteFamily"), + session_id, + ); + } + families.retain(|_, state| !state.is_empty()); + drop(families); + self.remove_pending_notifications_for_session(session_id); + self.refresh_metrics_gauges(); + } + + pub(in crate::domains::stream::sink) fn cleanup_session(&self, session_id: u64) { + self.cleaned_up_sessions.lock().mark(session_id); + self.unsubscribe_all(session_id); + + let actors = self + .actors + .lock() + .iter() + .map(|(key, actor)| (key.family.as_u64(), actor.clone())) + .collect::>(); + let mut removed_sessions = Vec::new(); + let mut advanced_families = std::collections::BTreeSet::new(); + for (family_id, actor) in actors { + if let Some(stream_session_id) = actor.lock().cleanup_session(session_id) { + removed_sessions.push(stream_session_id); + advanced_families.insert(family_id); + } + } + + for family_id in advanced_families { + self.handle_visibility_advance( + RouteFamily::try_from(family_id) + .expect("stream family IDs originate from RouteFamily"), + ); + } + + if !removed_sessions.is_empty() { + let removed_count = super::model::usize_to_u64_saturating(removed_sessions.len()); + let mut session_owners = self.session_owners.lock(); + for stream_session_id in removed_sessions { + session_owners.remove(&stream_session_id); + } + self.counter_add("fitz_stream_append_sessions_ended_total", removed_count); + self.admin_snapshot.mark_dirty(); + } + } +} diff --git a/src/domains/stream/sink/delivery.rs b/src/domains/stream/sink/delivery.rs new file mode 100644 index 00000000..21db8cf6 --- /dev/null +++ b/src/domains/stream/sink/delivery.rs @@ -0,0 +1,108 @@ +//! Publish/notification fan-out: matching subscribers to a committed event, +//! delivering to live subscribers, and notifying watermark coordinators. + +#[cfg(test)] +use super::model::PayloadEncoder; +use super::model::{Envelope, StreamDomainCore}; + +mod notification_gating; +mod watermark_coordination; + +impl StreamDomainCore { + pub(in crate::domains::stream::sink) fn handle_domain_publish( + &self, + event: &crate::runtime::DomainPublishEvent, + ) { + self.route_ready_notifications(self.collect_ready_notifications(event)); + } + + pub(in crate::domains::stream::sink) fn handle_visibility_advance( + &self, + family: crate::runtime::routing::RouteFamily, + ) { + self.route_ready_notifications(self.collect_visible_pending_notifications(family.as_u64())); + } + + fn route_ready_notifications(&self, ready: Vec) { + #[cfg(test)] + let mut payload_encoder = PayloadEncoder::with_capacity(256); + for notification in ready { + let target = notification.target; + let event = notification.event; + if *target.subscriber.family() != event.family_id { + crate::observability::counter_inc( + crate::domains::stream::metrics::METRIC_NOTIFY_DROPS_TOTAL, + ); + continue; + } + #[cfg(test)] + self.route_commit_notify( + target.session_id, + target.subscription_id, + &target.subscriber, + &event, + &mut payload_encoder, + ); + #[cfg(not(test))] + self.route_commit_notify( + target.session_id, + target.subscription_id, + &target.subscriber, + &event, + ); + } + } + + #[cfg(test)] + pub(in crate::domains::stream::sink) fn route_commit_notify( + &self, + session_id: u64, + subscription_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + event: &crate::runtime::DomainPublishEvent, + payload_encoder: &mut PayloadEncoder, + ) { + let notify_payload = crate::dispatch::protocol::stream_codec::encode_notify_into( + payload_encoder, + subscription_id, + &event.route, + &event.payload, + ); + let notify_ctx = crate::dispatch::protocol::FrameContext::new( + session_id, + crate::dispatch::protocol::frame::ChannelId::Sub, + crate::dispatch::protocol::tlv::MessageType::new(609), + bytes::Bytes::from(notify_payload), + event.family_id, + ); + let notify_envelope = Envelope::new(subscriber.clone(), notify_ctx); + if self.router.route(notify_envelope).is_err() { + crate::observability::counter_inc( + crate::domains::stream::metrics::METRIC_NOTIFY_DROPS_TOTAL, + ); + } + } + + #[cfg(not(test))] + pub(in crate::domains::stream::sink) fn route_commit_notify( + &self, + session_id: u64, + subscription_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + event: &crate::runtime::DomainPublishEvent, + ) { + let notify = crate::domains::stream::StreamClientNotification::new( + session_id, + event.family_id, + subscription_id, + event.route.clone(), + event.payload.clone(), + ); + let notify_envelope = Envelope::new(subscriber.clone(), notify); + if self.router.route(notify_envelope).is_err() { + crate::observability::counter_inc( + crate::domains::stream::metrics::METRIC_NOTIFY_DROPS_TOTAL, + ); + } + } +} diff --git a/src/domains/stream/sink/domain_sink_impl/domain_core_impl/notification_gating.rs b/src/domains/stream/sink/delivery/notification_gating.rs similarity index 98% rename from src/domains/stream/sink/domain_sink_impl/domain_core_impl/notification_gating.rs rename to src/domains/stream/sink/delivery/notification_gating.rs index 583a6b9d..2c0d5b6e 100644 --- a/src/domains/stream/sink/domain_sink_impl/domain_core_impl/notification_gating.rs +++ b/src/domains/stream/sink/delivery/notification_gating.rs @@ -1,4 +1,4 @@ -use super::{ +use super::super::model::{ route_triplet, PendingStreamNotification, ReadyStreamNotification, StreamDomainCore, StreamNotificationTarget, StreamVisibilityFrontier, }; @@ -209,7 +209,10 @@ impl StreamDomainCore { self.drain_visible_pending(family, &mut VisibilityCache::default()) } - pub(super) fn remove_pending_notifications_for_session(&self, session_id: u64) { + pub(in crate::domains::stream::sink) fn remove_pending_notifications_for_session( + &self, + session_id: u64, + ) { self.subscriptions .pending .lock() diff --git a/src/domains/stream/sink/domain_sink_impl/domain_core_impl/watermark_coordination.rs b/src/domains/stream/sink/delivery/watermark_coordination.rs similarity index 90% rename from src/domains/stream/sink/domain_sink_impl/domain_core_impl/watermark_coordination.rs rename to src/domains/stream/sink/delivery/watermark_coordination.rs index e37da714..d4f8f882 100644 --- a/src/domains/stream/sink/domain_sink_impl/domain_core_impl/watermark_coordination.rs +++ b/src/domains/stream/sink/delivery/watermark_coordination.rs @@ -1,5 +1,6 @@ -use super::{Envelope, Route, RouteFamily, StreamDomainCore}; +use super::super::model::{Envelope, Route, RouteFamily, StreamDomainCore}; use crate::domains::stream::metrics::METRIC_WATERMARK_COORDINATION_DROPS_TOTAL; +use crate::domains::stream::sink::model::{StreamAreaScope, StreamRealmScope}; struct WatermarkDispatch<'a, K> { address: crate::runtime::routing::RouteAddress, @@ -34,7 +35,10 @@ impl StreamDomainCore { let store = self.stream_store.clone(); let realm_owned = realm.to_string(); self.watermark_coordinators.realm.ensure_spawned( - (family_id.as_u64(), realm.to_string()), + StreamRealmScope { + family: family_id, + realm: realm.to_string(), + }, realm_address.clone(), move || { crate::domains::stream::realm_actor::RealmActor::new( @@ -58,7 +62,11 @@ impl StreamDomainCore { let realm_owned = realm.to_string(); let area_owned = area.to_string(); self.watermark_coordinators.area.ensure_spawned( - (family_id.as_u64(), realm.to_string(), area.to_string()), + StreamAreaScope { + family: family_id, + realm: realm.to_string(), + area: area.to_string(), + }, area_address.clone(), move || { crate::domains::stream::area_actor::AreaActor::new( diff --git a/src/domains/stream/sink/domain_sink_impl/domain_core_impl.rs b/src/domains/stream/sink/domain_sink_impl/domain_core_impl.rs deleted file mode 100644 index db452dac..00000000 --- a/src/domains/stream/sink/domain_sink_impl/domain_core_impl.rs +++ /dev/null @@ -1,867 +0,0 @@ -#[cfg(test)] -use super::FrameContext; -use super::{ - route_triplet, u64_to_usize_saturating, usize_to_u32_saturating, usize_to_u64_saturating, - AdminStreamReadRequest, Arc, BTreeMap, Envelope, Mutex, PayloadEncoder, - PendingStreamNotification, ReadResponse, ReadyStreamNotification, Route, RouteFamily, - StreamActor, StreamActorKey, StreamAdminRecord, StreamAdminSnapshotMap, StreamAreaSnapshotMap, - StreamClientResponseBody, StreamDomainCore, StreamDomainRuntime, StreamFilteredReason, - StreamLiveCounts, StreamMetadata, StreamNotificationTarget, StreamReadExecution, - StreamReadItem, StreamRealmSnapshotMap, StreamRecord, StreamStorageLayout, - StreamVisibilityFrontier, -}; - -mod global_read_support; -mod notification_gating; -mod read_finalization; -mod watermark_coordination; -mod wire_encoding; - -use read_finalization::apply_global_snapshot_boundary; - -#[derive(Clone, Copy, PartialEq, Eq)] -enum ReadScope { - Resource, - Area, - Realm, - Global, -} - -impl StreamDomainCore { - pub(in crate::domains::stream::sink) fn run_maintenance_slice(&self, family: u64) { - if let Err(error) = self.stream_store.run_maintenance(family) { - tracing::warn!( - domain = "stream", - family, - error, - "Stream maintenance slice failed; queued work will be retried" - ); - } - } - - fn cursor_integrity_token( - &self, - selector_fingerprint: u64, - captured_watermark: u64, - next_offset: u64, - ) -> u64 { - use hmac::{Hmac, KeyInit, Mac}; - - let mut mac = Hmac::::new_from_slice(self.cursor_integrity_key.as_ref()) - .expect("Stream cursor HMAC key has a valid fixed length"); - mac.update(&[1]); - mac.update(&selector_fingerprint.to_le_bytes()); - mac.update(&captured_watermark.to_le_bytes()); - mac.update(&next_offset.to_le_bytes()); - let bytes = mac.finalize().into_bytes(); - u64::from_le_bytes( - bytes[..8] - .try_into() - .expect("HMAC-SHA256 output is 32 bytes"), - ) - } - - pub(in crate::domains::stream::sink) fn storage_layout(&self) -> StreamStorageLayout { - self.stream_store.storage_layout() - } - - pub(in crate::domains::stream::sink) fn actor_key_for_route( - family_id: RouteFamily, - route: &Route, - ) -> Result { - let parts = - route_triplet(route.as_str()).ok_or_else(|| "invalid stream route".to_string())?; - if parts.realm.is_empty() - || parts.area.is_empty() - || parts.resource.is_empty() - || parts.realm.contains('*') - || parts.area.contains('*') - || parts.resource.contains('*') - { - return Err("stream append routes require concrete realm/area/resource".to_string()); - } - if parts.area == crate::domains::stream::INTERNAL_REALM_SEGMENT { - return Err(format!( - "area '{}' is reserved for internal broker use", - crate::domains::stream::INTERNAL_REALM_SEGMENT - )); - } - if parts.resource == crate::domains::stream::INTERNAL_AREA_SEGMENT { - return Err(format!( - "resource '{}' is reserved for internal broker use", - crate::domains::stream::INTERNAL_AREA_SEGMENT - )); - } - Ok(StreamActorKey { - family_id: family_id.as_u64(), - realm: parts.realm.to_string(), - area: parts.area.to_string(), - resource: parts.resource.to_string(), - }) - } - - pub(in crate::domains::stream::sink) fn get_or_create_actor( - &self, - key: &StreamActorKey, - ) -> Result>, String> { - use std::collections::hash_map::Entry; - - let mut actors = self.actors.lock(); - match actors.entry(key.clone()) { - Entry::Occupied(entry) => Ok(entry.get().clone()), - Entry::Vacant(entry) => { - let actor = Arc::new(Mutex::new(StreamActor::new( - RouteFamily::try_from(key.family_id) - .expect("stream family IDs originate from RouteFamily"), - key.realm.clone(), - key.area.clone(), - key.resource.clone(), - self.stream_store.clone(), - )?)); - entry.insert(actor.clone()); - Ok(actor) - } - } - } - - pub(in crate::domains::stream::sink) fn mark_admin_snapshot_dirty(&self) { - self.admin_snapshot.mark_dirty(); - self.refresh_metrics_gauges(); - } - - pub(in crate::domains::stream::sink) fn refresh_metrics_gauges(&self) { - let counts = self.aggregate_live_counts(); - - if let Some(metrics) = &self.metrics { - metrics.set_stream_count(counts.streams); - metrics.set_subscription_count(counts.subscriptions); - metrics.set_append_session_count(counts.append_sessions); - } else { - crate::observability::gauge_set("fitz_stream_active_gauge", counts.streams as u64); - crate::observability::gauge_set( - "fitz_stream_subscriptions_gauge", - counts.subscriptions as u64, - ); - crate::observability::gauge_set( - "fitz_stream_append_sessions_active", - counts.append_sessions as u64, - ); - } - } - - pub(in crate::domains::stream::sink) fn counter_inc(&self, name: &str) { - if let Some(metrics) = &self.metrics { - metrics.counter_inc(name); - } else { - crate::observability::counter_inc(name); - } - } - - pub(in crate::domains::stream::sink) fn counter_add(&self, name: &str, amount: u64) { - if let Some(metrics) = &self.metrics { - metrics.counter_add(name, amount); - } else { - crate::observability::counter_add(name, amount); - } - } - - pub(in crate::domains::stream::sink) fn stream_response_is_failure( - response: &StreamClientResponseBody, - ) -> bool { - matches!( - response, - StreamClientResponseBody::Error(_) | StreamClientResponseBody::SubscriptionError(_) - ) - } - - pub(in crate::domains::stream::sink) fn refresh_admin_snapshot_if_dirty(&self) { - if self.admin_snapshot.take_dirty() { - self.sync_admin_snapshot(); - } - } - - /// # Errors - /// - /// Returns an error if the requested route cannot be read or if the stream - /// store rejects the read parameters. - pub(in crate::domains::stream::sink) fn admin_read_resource_records( - &self, - request: AdminStreamReadRequest<'_>, - ) -> Result< - ( - Vec, - crate::domains::stream::protocol::ReadCursor, - ), - String, - > { - let filter = - request - .discriminator - .map(|value| crate::domains::stream::protocol::StreamFilterSet { - clauses: vec![ - crate::domains::stream::protocol::StreamFilterClause::Equals(value), - ], - }); - let params = crate::domains::stream::store::ReadResourceParams { - family: request.family.as_u64(), - realm: request.realm, - area: request.area, - resource: request.resource, - from_offset: request.from_offset, - limit: request.limit, - max_bytes: None, - }; - - self.stream_store - .read_resource_with_filter(¶ms, filter.as_ref()) - } - - pub(in crate::domains::stream::sink) fn sync_admin_snapshot(&self) { - let (mut streams, realm_snapshots, area_snapshots, committed_events_total) = - self.collect_committed_stream_snapshots(); - let stream_realm_watermarks = self.collect_stream_realm_watermarks(realm_snapshots); - let stream_area_watermarks = self.collect_stream_area_watermarks(area_snapshots); - self.overlay_live_actor_snapshots(&mut streams); - self.publish_admin_snapshot( - streams, - stream_realm_watermarks, - stream_area_watermarks, - committed_events_total, - ); - } - - fn collect_committed_stream_snapshots( - &self, - ) -> ( - StreamAdminSnapshotMap, - StreamRealmSnapshotMap, - StreamAreaSnapshotMap, - usize, - ) { - let mut streams: StreamAdminSnapshotMap = BTreeMap::new(); - let mut realm_snapshots: StreamRealmSnapshotMap = BTreeMap::new(); - let mut area_snapshots: StreamAreaSnapshotMap = BTreeMap::new(); - let mut committed_events_total = 0usize; - - if let Ok(families) = self.store.list_column_families() { - for family in families { - let family_id = u64::from(family.id()); - if let Ok(records) = self.stream_store.list_resource_metadata(family_id) { - for StreamAdminRecord { - realm, - area, - resource, - next_offset, - committed_size_bytes, - } in records - { - committed_events_total = committed_events_total - .saturating_add(u64_to_usize_saturating(next_offset)); - let last_offset = next_offset.saturating_sub(1); - streams.insert( - (family_id, realm.clone(), area.clone(), resource.clone()), - crate::control::admin::StreamInfo::snapshot( - crate::control::admin::StreamInfoSnapshot { - route_family: family_id, - realm: &realm, - area: &area, - resource: &resource, - offset: last_offset, - watermark: last_offset, - size_bytes: committed_size_bytes, - sessions_active: 0, - }, - ), - ); - - let realm_snapshot = realm_snapshots.entry(realm.clone()).or_default(); - realm_snapshot.areas.insert(area.clone()); - realm_snapshot.resource_count = - realm_snapshot.resource_count.saturating_add(1); - realm_snapshot.families.insert(family_id); - - let area_snapshot = area_snapshots - .entry((realm.clone(), area.clone())) - .or_default(); - area_snapshot.resource_count = - area_snapshot.resource_count.saturating_add(1); - area_snapshot.families.insert(family_id); - } - } - } - } - - ( - streams, - realm_snapshots, - area_snapshots, - committed_events_total, - ) - } - - fn collect_stream_realm_watermarks( - &self, - realm_snapshots: StreamRealmSnapshotMap, - ) -> Vec { - realm_snapshots - .into_iter() - .map(|(realm, snapshot)| { - let family_watermarks = snapshot - .families - .into_iter() - .filter_map(|family_id| { - self.stream_store - .get_realm_watermark(family_id, &realm) - .ok() - .map(|watermark| { - crate::control::admin::StreamRealmWatermark::snapshot( - family_id, watermark, - ) - }) - }) - .collect(); - - crate::control::admin::StreamRealmWatermarkDetail::snapshot( - &realm, - snapshot.areas.len(), - snapshot.resource_count, - family_watermarks, - ) - }) - .collect() - } - - fn collect_stream_area_watermarks( - &self, - area_snapshots: StreamAreaSnapshotMap, - ) -> Vec { - area_snapshots - .into_iter() - .map(|((realm, area), snapshot)| { - let family_watermarks = snapshot - .families - .into_iter() - .filter_map(|family_id| { - self.stream_store - .get_watermark(family_id, &realm, &area) - .ok() - .map(|watermark| { - crate::control::admin::StreamAreaWatermark::snapshot( - family_id, watermark, - ) - }) - }) - .collect(); - - crate::control::admin::StreamAreaWatermarkDetail::snapshot( - &realm, - &area, - snapshot.resource_count, - family_watermarks, - ) - }) - .collect() - } - - fn overlay_live_actor_snapshots(&self, streams: &mut StreamAdminSnapshotMap) { - let family_cores = self.registered_family_cores(); - if family_cores.is_empty() { - self.overlay_live_actor_snapshots_from(streams); - return; - } - - for family_core in family_cores { - family_core.overlay_live_actor_snapshots_from(streams); - } - } - - fn overlay_live_actor_snapshots_from(&self, streams: &mut StreamAdminSnapshotMap) { - let actors = self.actors.lock(); - for (key, actor) in actors.iter() { - let actor = actor.lock(); - let last_offset = actor - .metadata() - .ok() - .and_then(|response| response.metadata.last_resource_offset); - let sessions_active = usize::from(actor.has_active_session()); - let stream_key = ( - key.family_id, - key.realm.clone(), - key.area.clone(), - key.resource.clone(), - ); - let committed_snapshot = streams.get(&stream_key); - if committed_snapshot.is_none() && last_offset.is_none() { - continue; - } - let committed_size_bytes = committed_snapshot.map_or(0, |item| item.size_bytes); - let committed_offset = committed_snapshot.map(|item| item.offset); - let visible_offset = last_offset.or(committed_offset).unwrap_or(0); - - streams.insert( - stream_key, - crate::control::admin::StreamInfo::snapshot( - crate::control::admin::StreamInfoSnapshot { - route_family: key.family_id, - realm: &key.realm, - area: &key.area, - resource: &key.resource, - offset: visible_offset, - watermark: visible_offset, - size_bytes: committed_size_bytes, - sessions_active, - }, - ), - ); - } - } - - fn publish_admin_snapshot( - &self, - streams: StreamAdminSnapshotMap, - stream_realm_watermarks: Vec, - stream_area_watermarks: Vec, - committed_events_total: usize, - ) { - self.admin_snapshot - .read_model - .replace_streams(streams.into_values().collect()); - self.admin_snapshot - .read_model - .replace_stream_realm_watermarks(stream_realm_watermarks); - self.admin_snapshot - .read_model - .replace_stream_area_watermarks(stream_area_watermarks); - self.admin_snapshot - .read_model - .replace_stream_events_total(committed_events_total); - } - - fn empty_global_read_cursor( - &self, - request: &StreamReadExecution<'_>, - selector_fingerprint: u64, - captured_watermark: u64, - ) -> crate::domains::stream::protocol::ReadCursor { - crate::domains::stream::protocol::ReadCursor { - last_resource_offset: 0, - last_area_offset: None, - last_realm_offset: None, - last_global_offset: None, - has_more: request.from_offset < captured_watermark, - cursor_fingerprint: Some(self.cursor_integrity_token( - selector_fingerprint, - captured_watermark, - request.from_offset, - )), - captured_watermark: Some(captured_watermark), - } - } - - fn execute_read_plan( - &self, - scope: ReadScope, - route_filter_area: Option<&str>, - route_filter_resource: Option<&str>, - request: &StreamReadExecution<'_>, - ) -> Result { - let parts = route_triplet(request.route.as_str()); - let (items, cursor) = match scope { - ReadScope::Realm => { - let parts = parts.ok_or_else(|| "invalid stream route".to_string())?; - if let Some(resource) = route_filter_resource { - self.stream_store.read_realm_resource_posting( - &crate::domains::stream::store::ReadRealmPostingParams { - family: request.family_id.as_u64(), - realm: parts.realm, - resource, - from_offset: request.from_offset, - limit: request.limit, - max_bytes: request.max_bytes, - }, - request.filter, - )? - } else { - self.stream_store.read_realm_with_filter( - request.family_id.as_u64(), - parts.realm, - request.from_offset, - request.limit, - request.max_bytes, - request.filter, - )? - } - } - ReadScope::Area => { - let parts = parts.ok_or_else(|| "invalid stream route".to_string())?; - self.stream_store.read_area_with_filter( - &crate::domains::stream::store::ReadAreaParams { - family: request.family_id.as_u64(), - realm: parts.realm, - area: parts.area, - from_offset: request.from_offset, - limit: request.limit, - max_bytes: request.max_bytes, - }, - request.filter, - )? - } - ReadScope::Resource => { - let key = Self::actor_key_for_route(request.family_id, request.route)?; - let response = self.get_or_create_actor(&key)?.lock().read_with_filter( - request.from_offset, - request.limit, - request.max_bytes, - request.filter, - )?; - (response.items, response.cursor) - } - ReadScope::Global => self.stream_store.read_global_posting( - &crate::domains::stream::store::ReadGlobalPostingParams { - family: request.family_id.as_u64(), - from_offset: request.from_offset, - limit: request.limit, - max_bytes: request.max_bytes, - area: route_filter_area, - resource: route_filter_resource, - }, - request.filter, - )?, - }; - Ok(ReadResponse { items, cursor }) - } - - fn finalize_read_response( - &self, - request: &StreamReadExecution<'_>, - selector_fingerprint: u64, - captured_watermark: u64, - mut response: ReadResponse, - ) -> ReadResponse { - apply_global_snapshot_boundary(request.from_offset, captured_watermark, &mut response); - let next_offset = response - .cursor - .last_global_offset - .map_or(request.from_offset, |offset| offset.saturating_add(1)); - response.cursor.cursor_fingerprint = Some(self.cursor_integrity_token( - selector_fingerprint, - captured_watermark, - next_offset, - )); - response.cursor.captured_watermark = Some(captured_watermark); - response - } - - pub(in crate::domains::stream::sink) fn encode_read_response_data( - &self, - request: StreamReadExecution<'_>, - ) -> Result, String> { - use crate::domains::stream::route_grammar::StreamRouteShape; - - let shape = crate::domains::stream::route_grammar::classify_stream_route_shape( - request.route.as_str(), - )?; - let (scope, area_filter, resource_filter) = match &shape { - StreamRouteShape::Resource { .. } => (ReadScope::Resource, None, None), - StreamRouteShape::Area { .. } => (ReadScope::Area, None, None), - StreamRouteShape::Realm { .. } => (ReadScope::Realm, None, None), - StreamRouteShape::RealmFilterResource { resource, .. } => { - (ReadScope::Realm, None, Some(*resource)) - } - StreamRouteShape::Global => (ReadScope::Global, None, None), - StreamRouteShape::GlobalFilterArea { area } => (ReadScope::Global, Some(*area), None), - StreamRouteShape::GlobalFilterResource { resource } => { - (ReadScope::Global, None, Some(*resource)) - } - StreamRouteShape::GlobalFilterAreaResource { area, resource } => { - (ReadScope::Global, Some(*area), Some(*resource)) - } - }; - if scope != ReadScope::Global { - if request.cursor_fingerprint.is_some() || request.captured_watermark.is_some() { - return Err( - "ERR_CURSOR_UNSUPPORTED: snapshot cursors require a global stream selector" - .to_string(), - ); - } - let response = self.execute_read_plan(scope, area_filter, resource_filter, &request)?; - return Ok(Self::encode_stream_read_data( - &response.items, - &response.cursor, - false, - )); - } - - let selector_fingerprint = crate::domains::stream::route_grammar::cursor_fingerprint( - request.family_id, - &shape, - request.filter, - ); - let current_frontier = self - .stream_store - .get_global_watermark(request.family_id.as_u64())?; - let captured_watermark = request.captured_watermark.unwrap_or(current_frontier); - if captured_watermark > current_frontier { - return Err( - "ERR_CURSOR_WATERMARK_INVALID: captured watermark is ahead of the visible frontier" - .to_string(), - ); - } - let is_continuation = - request.cursor_fingerprint.is_some() || request.captured_watermark.is_some(); - let expected_token = self.cursor_integrity_token( - selector_fingerprint, - captured_watermark, - request.from_offset, - ); - if is_continuation && request.cursor_fingerprint != Some(expected_token) { - return Err( - "ERR_CURSOR_SELECTOR_MISMATCH: cursor was issued for a different stream route \ - family, selector, filter, snapshot, or position" - .to_string(), - ); - } - if request.limit == 0 { - let cursor = - self.empty_global_read_cursor(&request, selector_fingerprint, captured_watermark); - return Ok(Self::encode_stream_read_data(&[], &cursor, true)); - } - let response = self.execute_read_plan(scope, area_filter, resource_filter, &request)?; - let response = self.finalize_read_response( - &request, - selector_fingerprint, - captured_watermark, - response, - ); - Ok(Self::encode_stream_read_data( - &response.items, - &response.cursor, - true, - )) - } - - pub(in crate::domains::stream::sink) fn handle_domain_publish( - &self, - event: &crate::runtime::DomainPublishEvent, - ) { - self.route_ready_notifications(self.collect_ready_notifications(event)); - } - - pub(in crate::domains::stream::sink) fn handle_visibility_advance(&self, family: RouteFamily) { - self.route_ready_notifications(self.collect_visible_pending_notifications(family.as_u64())); - } - - fn route_ready_notifications(&self, ready: Vec) { - #[cfg(test)] - let mut payload_encoder = PayloadEncoder::with_capacity(256); - for notification in ready { - let target = notification.target; - let event = notification.event; - if *target.subscriber.family() != event.family_id { - crate::observability::counter_inc( - crate::domains::stream::metrics::METRIC_NOTIFY_DROPS_TOTAL, - ); - continue; - } - #[cfg(test)] - self.route_commit_notify( - target.session_id, - target.subscription_id, - &target.subscriber, - &event, - &mut payload_encoder, - ); - #[cfg(not(test))] - self.route_commit_notify( - target.session_id, - target.subscription_id, - &target.subscriber, - &event, - ); - } - } - - #[cfg(test)] - pub(in crate::domains::stream::sink) fn route_commit_notify( - &self, - session_id: u64, - subscription_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - event: &crate::runtime::DomainPublishEvent, - payload_encoder: &mut PayloadEncoder, - ) { - let notify_payload = crate::dispatch::protocol::stream_codec::encode_notify_into( - payload_encoder, - subscription_id, - &event.route, - &event.payload, - ); - let notify_ctx = FrameContext::new( - session_id, - crate::dispatch::protocol::frame::ChannelId::Sub, - crate::dispatch::protocol::tlv::MessageType::new(609), - bytes::Bytes::from(notify_payload), - event.family_id, - ); - let notify_envelope = Envelope::new(subscriber.clone(), notify_ctx); - if self.router.route(notify_envelope).is_err() { - crate::observability::counter_inc( - crate::domains::stream::metrics::METRIC_NOTIFY_DROPS_TOTAL, - ); - } - } - - #[cfg(not(test))] - pub(in crate::domains::stream::sink) fn route_commit_notify( - &self, - session_id: u64, - subscription_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - event: &crate::runtime::DomainPublishEvent, - ) { - let notify = crate::domains::stream::StreamClientNotification::new( - session_id, - event.family_id, - subscription_id, - event.route.clone(), - event.payload.clone(), - ); - let notify_envelope = Envelope::new(subscriber.clone(), notify); - if self.router.route(notify_envelope).is_err() { - crate::observability::counter_inc( - crate::domains::stream::metrics::METRIC_NOTIFY_DROPS_TOTAL, - ); - } - } - - pub(in crate::domains::stream::sink) fn unsubscribe_all(&self, session_id: u64) { - let mut families = self.subscriptions.families.lock(); - for (family_id, state) in families.iter_mut() { - state.remove_session( - RouteFamily::try_from(*family_id) - .expect("stream family IDs originate from RouteFamily"), - session_id, - ); - } - families.retain(|_, state| !state.is_empty()); - drop(families); - self.remove_pending_notifications_for_session(session_id); - self.refresh_metrics_gauges(); - } - - pub(in crate::domains::stream::sink) fn cleanup_session(&self, session_id: u64) { - self.unsubscribe_all(session_id); - - let actors = self - .actors - .lock() - .iter() - .map(|(key, actor)| (key.family_id, actor.clone())) - .collect::>(); - let mut removed_sessions = Vec::new(); - let mut advanced_families = std::collections::BTreeSet::new(); - for (family_id, actor) in actors { - if let Some(stream_session_id) = actor.lock().cleanup_session(session_id) { - removed_sessions.push(stream_session_id); - advanced_families.insert(family_id); - } - } - - for family_id in advanced_families { - self.handle_visibility_advance( - RouteFamily::try_from(family_id) - .expect("stream family IDs originate from RouteFamily"), - ); - } - - if !removed_sessions.is_empty() { - let removed_count = usize_to_u64_saturating(removed_sessions.len()); - let mut session_owners = self.session_owners.lock(); - for stream_session_id in removed_sessions { - session_owners.remove(&stream_session_id); - } - self.counter_add("fitz_stream_append_sessions_ended_total", removed_count); - self.admin_snapshot.mark_dirty(); - } - } - - pub(in crate::domains::stream::sink) fn live_counts(&self) -> StreamLiveCounts { - let subscriptions = self - .subscriptions - .families - .lock() - .values() - .map(crate::domains::subscription_state::RoutedSubscriptionSet::subscription_count) - .sum(); - - StreamLiveCounts { - streams: self.actors.lock().len(), - append_sessions: self.session_owners.lock().len(), - subscriptions, - } - } - - fn registered_family_cores(&self) -> Vec> { - let mut family_cores = self.family_cores.lock(); - let mut live = Vec::with_capacity(family_cores.len()); - family_cores.retain(|_, weak| { - if let Some(core) = weak.upgrade() { - live.push(core); - true - } else { - false - } - }); - live - } - - fn aggregate_live_counts(&self) -> StreamLiveCounts { - let family_cores = self.registered_family_cores(); - if family_cores.is_empty() { - return self.live_counts(); - } - - family_cores - .into_iter() - .fold(StreamLiveCounts::default(), |mut total, family_core| { - let counts = family_core.live_counts(); - total.streams = total.streams.saturating_add(counts.streams); - total.append_sessions = - total.append_sessions.saturating_add(counts.append_sessions); - total.subscriptions = total.subscriptions.saturating_add(counts.subscriptions); - total - }) - } -} - -impl StreamDomainRuntime<'_> { - pub(in crate::domains::stream::sink) fn run_maintenance_slice(&self, family: u64) { - self.core.run_maintenance_slice(family); - } - - pub(in crate::domains::stream::sink) fn refresh_admin_snapshot_if_dirty(&self) { - self.core.refresh_admin_snapshot_if_dirty(); - } - - #[cfg(test)] - pub(in crate::domains::stream::sink) fn sync_admin_snapshot(&self) { - self.core.sync_admin_snapshot(); - } - - pub(in crate::domains::stream::sink) fn live_counts(&self) -> StreamLiveCounts { - self.core.live_counts() - } - - pub(in crate::domains::stream::sink) fn admin_read_resource_records( - &self, - request: AdminStreamReadRequest<'_>, - ) -> Result< - ( - Vec, - crate::domains::stream::protocol::ReadCursor, - ), - String, - > { - self.core.admin_read_resource_records(request) - } -} diff --git a/src/domains/stream/sink/domain_sink_impl/domain_core_impl/global_read_support.rs b/src/domains/stream/sink/domain_sink_impl/domain_core_impl/global_read_support.rs deleted file mode 100644 index 24484a37..00000000 --- a/src/domains/stream/sink/domain_sink_impl/domain_core_impl/global_read_support.rs +++ /dev/null @@ -1,41 +0,0 @@ -use super::{route_triplet, Route, RouteFamily, StreamDomainCore}; - -impl StreamDomainCore { - pub(in crate::domains::stream::sink) fn encode_last_response_data( - &self, - family_id: RouteFamily, - route: &Route, - ) -> Result, String> { - let parts = - route_triplet(route.as_str()).ok_or_else(|| "invalid stream route".to_string())?; - if parts.area == "*" || parts.resource == "*" { - return Ok(Vec::new()); - } - let key = Self::actor_key_for_route(family_id, route)?; - let actor = self.get_or_create_actor(&key)?; - let data = actor - .lock() - .last()? - .record - .as_ref() - .map(Self::encode_stream_last_data) - .unwrap_or_default(); - Ok(data) - } - - pub(in crate::domains::stream::sink) fn encode_metadata_response_data( - &self, - family_id: RouteFamily, - route: &Route, - ) -> Result, String> { - let parts = - route_triplet(route.as_str()).ok_or_else(|| "invalid stream route".to_string())?; - if parts.area == "*" || parts.resource == "*" { - return Ok(Vec::new()); - } - let key = Self::actor_key_for_route(family_id, route)?; - let actor = self.get_or_create_actor(&key)?; - let metadata = actor.lock().metadata()?.metadata; - Ok(Self::encode_stream_metadata_data(&metadata)) - } -} diff --git a/src/domains/stream/sink/domain_sink_impl.rs b/src/domains/stream/sink/facade.rs similarity index 77% rename from src/domains/stream/sink/domain_sink_impl.rs rename to src/domains/stream/sink/facade.rs index 98d0c139..8cf770c0 100644 --- a/src/domains/stream/sink/domain_sink_impl.rs +++ b/src/domains/stream/sink/facade.rs @@ -1,37 +1,16 @@ +#[cfg(test)] +use super::model::StreamReadExecution; use super::model::{ - route_triplet, AdminSnapshotState, AdminStreamReadRequest, AdminStreamReadRequestOwned, Arc, - AtomicBool, AtomicU64, BTreeMap, Envelope, HashMap, Mutex, Ordering, PayloadEncoder, - PendingStreamNotification, ReadResponse, ReadyStreamNotification, Route, RouteAddress, - RouteFamily, Router, StreamActor, StreamActorKey, StreamAdminReadCommand, StreamAdminRecord, - StreamAreaSnapshot, StreamClientResponseBody, StreamDomainActor, StreamDomainCommand, - StreamDomainCore, StreamDomainRuntime, StreamDomainSink, StreamFilteredReason, - StreamLiveCounts, StreamMetadata, StreamMetrics, StreamNotificationTarget, StreamReadExecution, - StreamReadItem, StreamRealmSnapshot, StreamRecord, StreamStorageLayout, StreamStore, - StreamVisibilityFrontier, SubscriptionRegistry, WatermarkCoordinators, + stream_assumed_service_us, AdminSnapshotState, AdminStreamReadRequest, + AdminStreamReadRequestOwned, Arc, AtomicBool, AtomicU64, AtomicUsize, BTreeMap, + CleanedUpSessions, HashMap, Mutex, Ordering, Route, RouteFamily, Router, + StreamAdminReadCommand, StreamDomainActor, StreamDomainCommand, StreamDomainCore, + StreamDomainSink, StreamLiveCounts, StreamMetrics, StreamReadItem, StreamStorageLayout, + StreamStore, StreamWorkKey, SubscriptionRegistry, WatermarkCoordinators, }; -#[cfg(test)] -use crate::dispatch::protocol::FrameContext; +use crate::runtime::routing::RouteAddress; use crate::runtime::DeliveryError; -fn u64_to_usize_saturating(value: u64) -> usize { - usize::try_from(value).unwrap_or(usize::MAX) -} - -fn usize_to_u32_saturating(value: usize) -> u32 { - u32::try_from(value).unwrap_or(u32::MAX) -} - -fn usize_to_u64_saturating(value: usize) -> u64 { - u64::try_from(value).unwrap_or(u64::MAX) -} - -type StreamAdminSnapshotMap = - BTreeMap<(u64, String, String, String), crate::control::admin::StreamInfo>; -type StreamRealmSnapshotMap = BTreeMap; -type StreamAreaSnapshotMap = BTreeMap<(String, String), StreamAreaSnapshot>; - -mod domain_core_impl; - impl StreamDomainActor { pub(super) fn new(core: Arc) -> Self { Self { core } @@ -40,41 +19,46 @@ impl StreamDomainActor { pub(super) fn route_address() -> RouteAddress { RouteAddress::new(RouteFamily::new(0), Route::new("internal://domain/stream")) } - - pub(super) fn runtime(&self) -> StreamDomainRuntime<'_> { - StreamDomainRuntime { core: &self.core } - } } impl StreamDomainSink { - pub fn new( + /// Construct a Stream sink using the default storage layout. + /// + /// # Errors + /// + /// Returns an initialization error when storage activation, persisted + /// state validation, or cursor-key generation fails. + pub fn try_new( store: Arc, router: Arc, admin_read_model: Arc, write_options: super::StreamStorageWriteOptions, - ) -> Self { - Self::new_with_storage( + ) -> Result { + Self::new_with_storage_layout( crate::storage::FitzStorageEngine::new(store), router, admin_read_model, + StreamStorageLayout::default(), write_options, ) + .map_err(super::StreamSinkInitError::new) } - pub(crate) fn new_with_storage( - store: crate::storage::FitzStorageEngine, + /// Compatibility constructor retaining the historical panic-on-init + /// behavior. New callers should use [`Self::try_new`]. + /// + /// # Panics + /// + /// Panics when Stream storage initialization or persisted-state + /// validation fails. + pub fn new( + store: Arc, router: Arc, admin_read_model: Arc, write_options: super::StreamStorageWriteOptions, ) -> Self { - Self::new_with_storage_layout( - store, - router, - admin_read_model, - StreamStorageLayout::default(), - write_options, - ) - .expect("create stream domain sink with default stream layout") + Self::try_new(store, router, admin_read_model, write_options) + .expect("create stream domain sink with default stream layout") } /// # Errors @@ -138,6 +122,9 @@ impl StreamDomainSink { store, actors: Mutex::new(HashMap::new()), session_owners: Mutex::new(HashMap::new()), + cleaned_up_sessions: Mutex::new(CleanedUpSessions::new( + crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, + )), subscriptions: SubscriptionRegistry::new(Arc::new(AtomicU64::new(1))), next_session_id: Arc::new(AtomicU64::new(1)), cursor_integrity_key: Arc::new(cursor_integrity_key), @@ -162,9 +149,12 @@ impl StreamDomainSink { crate::domains::stream::MAX_WATERMARK_COORDINATORS, )), }, + delivery_service_us: Arc::new(AtomicU64::new(stream_assumed_service_us())), }); - let actor = Self::spawn_actor(core.clone()); let family_families = provisioned_families.map(<[RouteFamily]>::to_vec); + let actor = family_families + .is_none() + .then(|| Self::spawn_actor(core.clone())); let family_runtime = family_families .as_deref() .map(|families| Self::spawn_family_runtime(&core, families)) @@ -174,6 +164,7 @@ impl StreamDomainSink { actor, family_runtime, family_families, + inflight_client_deliveries: Arc::new(AtomicUsize::new(0)), }) } @@ -192,23 +183,36 @@ impl StreamDomainSink { fn spawn_family_runtime( core: &Arc, families: &[RouteFamily], - ) -> Result, String> { - let pool = crate::runtime::FamilyActorPool::new(families) - .map_err(|error| format!("create Stream family actor pool: {error}"))?; - let active = core.active.clone(); + ) -> Result< + crate::runtime::keyed_family_executor::KeyedFamilyExecutor< + StreamWorkKey, + StreamDomainCommand, + Arc, + >, + String, + > { let core_for_factory = core.clone(); - Ok(crate::runtime::FamilyActorPoolRuntime::spawn( - pool, - active, + let core_for_failure = core.clone(); + crate::runtime::keyed_family_executor::KeyedFamilyExecutor::new( + families, + crate::runtime::keyed_family_executor::KeyedFamilyExecutor::< + StreamWorkKey, + StreamDomainCommand, + Arc, + >::production_worker_count(), move |family| Self::family_core_for(&core_for_factory, family), - |core, family, _lane, command| match command { - StreamDomainCommand::Deliver(envelope, reply) => { + |core, family, _lane, _key, command| match command { + StreamDomainCommand::Deliver(envelope, reply, admission) => { let result = if *envelope.destination().family() == family { core.deliver_envelope(&envelope) } else { Err(DeliveryError::ActorStopped) }; let _ = reply.send(result); + // Always None on this path - `deliver_to_family` never + // admits - but drop explicitly for symmetry with the + // non-family actor's release-on-completion. + drop(admission); } StreamDomainCommand::ReadLiveCounts(reply) => { let _ = reply.send(core.live_counts()); @@ -244,7 +248,18 @@ impl StreamDomainSink { panic!("test Stream family actor panic"); } }, - )) + move |_family| { + if let Some(metrics) = core_for_failure.metrics.as_ref() { + metrics.counter_inc( + crate::domains::stream::metrics::METRIC_FAMILY_FAILED_CLOSED_TOTAL, + ); + } else { + crate::observability::counter_inc( + crate::domains::stream::metrics::METRIC_FAMILY_FAILED_CLOSED_TOTAL, + ); + } + }, + ) } fn family_core_for( @@ -256,6 +271,9 @@ impl StreamDomainSink { stream_store: shared.stream_store.clone(), actors: Mutex::new(HashMap::new()), session_owners: Mutex::new(HashMap::new()), + cleaned_up_sessions: Mutex::new(CleanedUpSessions::new( + crate::domains::DOMAIN_ACTOR_MAILBOX_CAPACITY, + )), subscriptions: SubscriptionRegistry::new(shared.subscriptions.next_id.clone()), next_session_id: shared.next_session_id.clone(), cursor_integrity_key: shared.cursor_integrity_key.clone(), @@ -272,6 +290,9 @@ impl StreamDomainSink { area: shared.watermark_coordinators.area.clone(), realm: shared.watermark_coordinators.realm.clone(), }, + // Unused by family cores: `deliver_to_family` never blocks its + // caller and so never admits against this estimate. + delivery_service_us: Arc::new(AtomicU64::new(stream_assumed_service_us())), }); shared .family_cores @@ -282,14 +303,19 @@ impl StreamDomainSink { fn stop_family_runtime(&mut self) { if let Some(runtime) = self.family_runtime.take() { - runtime.stop(); + runtime.join(); } } fn rebuild_actor(&mut self) { - self.actor.stop(); + if let Some(actor) = self.actor.take() { + actor.stop(); + } self.stop_family_runtime(); - self.actor = Self::spawn_actor(self.core.clone()); + self.actor = self + .family_families + .is_none() + .then(|| Self::spawn_actor(self.core.clone())); self.family_runtime = self .family_families .as_deref() @@ -308,7 +334,6 @@ impl StreamDomainSink { mut self, collector: crate::observability::metrics::MetricsCollector, ) -> Self { - self.actor.stop(); self.core_for_builder().metrics = Some(StreamMetrics::new(collector)); self.core.refresh_metrics_gauges(); self.rebuild_actor(); @@ -320,7 +345,9 @@ impl StreamDomainSink { if let Some(runtime) = self.family_runtime.as_ref() { runtime.stop(); } - self.actor.stop(); + if let Some(actor) = self.actor.as_ref() { + actor.stop(); + } } pub(crate) fn is_active(&self) -> bool { @@ -356,14 +383,12 @@ impl StreamDomainSink { continue; }; runtime - .try_enqueue( - RouteFamily::new(family_id), - crate::runtime::FamilyActorLane::Control, - command, - ) + .try_enqueue_control(RouteFamily::new(family_id), command) .map_err(|error| error.to_string()) } else { self.actor + .as_ref() + .expect("direct Stream mode has a managed actor") .try_send_high_priority(command) .map_err(|error| error.to_string()) }; @@ -421,11 +446,14 @@ impl StreamDomainSink { #[cfg(test)] pub(super) fn is_actor_running(&self) -> bool { - self.actor.is_running() - && self - .family_runtime - .as_ref() - .is_none_or(crate::runtime::FamilyActorPoolRuntime::is_running) + self.family_runtime.as_ref().map_or_else( + || { + self.actor + .as_ref() + .is_some_and(crate::runtime::ManagedActor::is_running) + }, + crate::runtime::keyed_family_executor::KeyedFamilyExecutor::is_running, + ) } #[cfg(test)] @@ -437,10 +465,12 @@ impl StreamDomainSink { }; if let Some(runtime) = self.family_runtime.as_ref() { runtime - .try_enqueue(family, crate::runtime::FamilyActorLane::Control, command) + .try_enqueue_control(family, command) .expect("enqueue test Stream maintenance command"); } else { self.actor + .as_ref() + .expect("direct Stream mode has a managed actor") .try_send_high_priority(command) .expect("enqueue test Stream maintenance command"); } @@ -451,8 +481,24 @@ impl StreamDomainSink { pub(crate) fn actor_health_snapshot(&self) -> crate::runtime::ManagedActorHealthSnapshot { self.family_runtime.as_ref().map_or_else( - || self.actor.health_snapshot(), - crate::runtime::FamilyActorPoolRuntime::managed_actor_health_snapshot, + || { + self.actor + .as_ref() + .expect("direct Stream mode has a managed actor") + .health_snapshot() + }, + |runtime| { + let failed_family_count = runtime.failed_family_count(); + let running = runtime.is_running(); + crate::runtime::ManagedActorHealthSnapshot { + running, + restart_count: 0, + panic_count: u64::try_from(failed_family_count).unwrap_or(u64::MAX), + // Same "every family failed" threshold `running` uses -- + // a single failed family must not report exhaustion. + restart_exhausted: !running, + } + }, ) } @@ -463,14 +509,31 @@ impl StreamDomainSink { .fail_next_promotion_frontier_commit_for_tests(); } + /// Panic every provisioned family's handler (or the single actor in + /// non-sharded mode). Used by `panic_all_domain_actors_for_tests` to + /// drive the pool to full exhaustion; a single family's panic must never + /// be conflated with domain-wide health, so covering every family here + /// is required to actually observe pool-wide fail-closed behavior. #[cfg(test)] pub(crate) fn panic_actor_for_tests(&self) { - let _ = self.dispatch_family_control(None, StreamDomainCommand::PanicForTests); + match self.family_families.as_deref() { + Some(families) => { + for family in families { + let _ = self + .dispatch_family_control(Some(*family), StreamDomainCommand::PanicForTests); + } + } + None => { + let _ = self.dispatch_family_control(None, StreamDomainCommand::PanicForTests); + } + } } #[cfg(test)] pub(super) fn stop_actor_for_tests(&self) { - self.actor.stop(); + if let Some(actor) = self.actor.as_ref() { + actor.stop(); + } } #[cfg(test)] @@ -665,7 +728,10 @@ impl StreamDomainSink { "live-count query", StreamDomainCommand::ReadLiveCounts, ) - .unwrap_or_default() + .unwrap_or_else(|error| { + tracing::warn!(domain = "stream", error, "Stream live-count query failed"); + StreamLiveCounts::default() + }) } fn dispatch_family_command( @@ -699,10 +765,12 @@ impl StreamDomainSink { return Err("route family is not provisioned".to_string()); } runtime - .try_enqueue(family, crate::runtime::FamilyActorLane::Control, command) + .try_enqueue_control(family, command) .map_err(|error| error.to_string()) } else { self.actor + .as_ref() + .expect("direct Stream mode has a managed actor") .try_send_high_priority(command) .map_err(|error| error.to_string()) } diff --git a/src/domains/stream/sink/mailbox_sink_impl.rs b/src/domains/stream/sink/mailbox_sink_impl.rs index f52410a4..82f2a744 100644 --- a/src/domains/stream/sink/mailbox_sink_impl.rs +++ b/src/domains/stream/sink/mailbox_sink_impl.rs @@ -1,9 +1,10 @@ use super::model::{ - Arc, DeliveryError, Envelope, MailboxSink, Mutex, Ordering, PayloadEncoder, Route, RouteFamily, + admit_stream_client_delivery, record_stream_service_sample, Arc, DeliveryError, Envelope, + Instant, MailboxSink, Mutex, Ordering, PayloadEncoder, Route, RouteFamily, RoutedSubscriptionSet, StreamActor, StreamClientFrame, StreamClientRequest, StreamClientResponseBody, StreamDomainActor, StreamDomainCommand, StreamDomainCore, - StreamDomainRuntime, StreamDomainSink, StreamReadExecution, StreamSessionOwner, - StreamSubscription, STREAM_OPERATIONS_TOTAL, + StreamDomainSink, StreamReadExecution, StreamSessionOwner, StreamSubscription, StreamWorkKey, + STREAM_ACTOR_REPLY_TIMEOUT, STREAM_OPERATIONS_TOTAL, }; #[cfg(test)] use crate::dispatch::protocol::FrameContext; @@ -13,13 +14,21 @@ use crate::domains::stream::store::StreamStoreError; use crate::runtime::routing::RouteAddress; use crate::runtime::{Actor, Context}; +mod envelope_dispatch; +mod session_operations; +mod subscription_frames; + impl MailboxSink for StreamDomainSink { fn deliver(&self, envelope: Envelope) -> Result<(), DeliveryError> { - if !self.actor.is_running() - || self - .family_runtime + // Family liveness is gated per-family inside `try_enqueue` below + // (`FamilyActorPoolRuntime::is_family_running`) -- a panic scoped to + // one route family must not reject delivery to every other family + // sharing this pool. + if self.family_runtime.is_none() + && !self + .actor .as_ref() - .is_some_and(|runtime| !runtime.is_running()) + .is_some_and(crate::runtime::ManagedActor::is_running) { return Err(DeliveryError::ActorStopped); } @@ -44,33 +53,38 @@ impl Actor for StreamDomainActor { type Message = StreamDomainCommand; fn receive(&mut self, msg: Self::Message, _ctx: &mut Context) { - let runtime = self.runtime(); match msg { - StreamDomainCommand::Deliver(envelope, reply) => { - let _ = reply.send(runtime.deliver_envelope(&envelope)); + StreamDomainCommand::Deliver(envelope, reply, admission) => { + let started_at = Instant::now(); + let outcome = self.core.deliver_envelope(&envelope); + record_stream_service_sample(&self.core.delivery_service_us, started_at); + let _ = reply.send(outcome); + // Explicit: the slot is released here, once the work is + // actually done, and not when the caller gave up waiting. + drop(admission); } StreamDomainCommand::ReadLiveCounts(reply) => { - let _ = reply.send(runtime.live_counts()); + let _ = reply.send(self.core.live_counts()); } StreamDomainCommand::ReadResourceRecords(command) => { let request = command.request.as_borrowed(); let _ = command .reply - .send(runtime.admin_read_resource_records(request)); + .send(self.core.admin_read_resource_records(request)); } StreamDomainCommand::RefreshAdminSnapshotIfDirty(reply) => { - runtime.refresh_admin_snapshot_if_dirty(); + self.core.refresh_admin_snapshot_if_dirty(); let _ = reply.send(()); } StreamDomainCommand::RunMaintenance { family, reply } => { - runtime.run_maintenance_slice(family); + self.core.run_maintenance_slice(family); if let Some(reply) = reply { let _ = reply.send(()); } } #[cfg(test)] StreamDomainCommand::SyncAdminSnapshot(reply) => { - runtime.sync_admin_snapshot(); + self.core.sync_admin_snapshot(); let _ = reply.send(()); } #[cfg(test)] @@ -92,15 +106,20 @@ impl StreamDomainSink { }; let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); let family = *envelope.destination().family(); - let command = StreamDomainCommand::Deliver(envelope, reply_tx); - let lane = if high_priority { - crate::runtime::FamilyActorLane::Control + if !runtime.is_family_running(family) { + return Err(DeliveryError::ActorStopped); + } + // Never blocks its caller, so it never needs an admission slot. + let key = self.work_key_for_envelope(&envelope); + let command = StreamDomainCommand::Deliver(envelope, reply_tx, None); + if high_priority { + runtime.try_enqueue_control(family, command) + } else if let Some(key) = key { + runtime.try_enqueue(family, key, command) } else { - crate::runtime::FamilyActorLane::Normal - }; - runtime - .try_enqueue(family, lane, command) - .map_err(Self::family_enqueue_error)?; + runtime.try_enqueue_control(family, command) + } + .map_err(Self::family_enqueue_error)?; // Family delivery is called synchronously by the async transport edge. // The actor routes client responses through the router, so waiting for @@ -111,6 +130,73 @@ impl StreamDomainSink { Ok(()) } + fn work_key_for_envelope(&self, envelope: &Envelope) -> Option { + if envelope + .payload::() + .is_some() + { + return None; + } + if let Some(event) = envelope.payload::() { + return Some(StreamWorkKey::Notification(event.route.as_str().to_owned())); + } + let request = StreamDomainCore::request_from_envelope(envelope)?; + let session_id = request.meta.session_id; + match request.frame { + Err(_) => Some(StreamWorkKey::UnresolvedSession(session_id)), + Ok(StreamClientFrame::Sub(message)) => match message { + crate::domains::stream::protocol::StreamSubscriptionMessage::Subscribe { + session_id, + .. + } + | crate::domains::stream::protocol::StreamSubscriptionMessage::Unsubscribe { + session_id, + .. + } => Some(StreamWorkKey::SubscriptionSession(session_id)), + }, + Ok(StreamClientFrame::Op(message)) => { + use crate::domains::stream::protocol::StreamMessage; + match message { + StreamMessage::Begin { + family_id, route, .. + } => StreamDomainCore::actor_key_for_route(family_id, &route) + .ok() + .map(StreamWorkKey::Resource), + StreamMessage::Read { + family_id, route, .. + } + | StreamMessage::Last { family_id, route } + | StreamMessage::GetMetadata { family_id, route } => { + Some(Self::selector_work_key(family_id, &route)) + } + StreamMessage::Append { session_id, .. } + | StreamMessage::Commit { session_id, .. } + | StreamMessage::Rollback { session_id } => self + .core + .session_owners + .lock() + .get(&session_id) + .map_or_else( + || Some(StreamWorkKey::UnresolvedSession(session_id)), + |owner| Some(StreamWorkKey::Resource(owner.key.clone())), + ), + } + } + } + } + + fn selector_work_key(family: RouteFamily, route: &Route) -> StreamWorkKey { + match crate::domains::stream::route_grammar::classify_stream_route_shape(route.as_str()) { + Ok(crate::domains::stream::route_grammar::StreamRouteShape::Resource { .. }) => { + StreamDomainCore::actor_key_for_route(family, route).map_or_else( + |_| StreamWorkKey::Selector(route.as_str().to_owned()), + StreamWorkKey::Resource, + ) + } + Ok(_) | Err(_) => StreamWorkKey::Selector(route.as_str().to_owned()), + } + } + fn family_enqueue_error(error: crate::runtime::FamilyActorEnqueueError) -> DeliveryError { match error { crate::runtime::FamilyActorEnqueueError::NormalLaneFull => DeliveryError::MailboxFull { @@ -133,844 +219,73 @@ impl StreamDomainSink { envelope: Envelope, high_priority: bool, ) -> Result<(), DeliveryError> { + // Refuse surplus client work before enqueue; never ration control-plane work. + let is_control_plane = high_priority + || envelope + .payload::() + .is_some(); + let admission = if is_control_plane { + None + } else { + Some(admit_stream_client_delivery( + &self.inflight_client_deliveries, + &self.core.delivery_service_us, + self.actor + .as_ref() + .is_some_and(crate::runtime::ManagedActor::is_running), + )?) + }; + let (reply_tx, reply_rx) = crossbeam_channel::bounded(1); - let command = StreamDomainCommand::Deliver(envelope, reply_tx); + let command = StreamDomainCommand::Deliver(envelope, reply_tx, admission); let enqueue_result = if high_priority { - self.actor.try_send_high_priority(command) + self.actor + .as_ref() + .expect("direct Stream mode has a managed actor") + .try_send_high_priority(command) } else { - self.actor.try_send(command) + self.actor + .as_ref() + .expect("direct Stream mode has a managed actor") + .try_send(command) }; enqueue_result?; reply_rx - .recv_timeout(std::time::Duration::from_secs(1)) - .unwrap_or(Err(DeliveryError::ActorStopped)) + .recv_timeout(STREAM_ACTOR_REPLY_TIMEOUT) + .unwrap_or_else(|error| Err(crate::runtime::reply_wait::map_reply_wait_error(error))) } } -impl StreamDomainRuntime<'_> { - pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { - self.core.deliver_envelope(envelope) - } -} - -impl StreamDomainCore { - pub(super) fn deliver_envelope(&self, envelope: &Envelope) -> Result<(), DeliveryError> { - if self.handle_cleanup_envelope(envelope) { - return Ok(()); - } - self.ensure_active()?; - - if self.handle_domain_publish_envelope(envelope) { - return Ok(()); - } - - let Some(request) = Self::extract_request(envelope)? else { - return Ok(()); - }; - let meta = request.meta; - let request_started = self.record_request_start(); - - if meta.route_family != *envelope.destination().family() - || envelope - .source() - .is_some_and(|source| *source.family() != meta.route_family) - { - let response = Self::stream_error_response("route family mismatch"); - let response_meta = envelope.source().map_or(meta, |source| { - let mut response_meta = meta; - response_meta.route_family = *source.family(); - response_meta - }); - self.route_stream_response(envelope, response_meta, &response, request_started); - return Ok(()); - } - - let Some(parsed_frame) = - self.parse_request_frame(envelope, meta, request.frame, request_started) - else { - return Ok(()); - }; - - self.record_operation(); - - match parsed_frame { - StreamClientFrame::Sub(sub_msg) => { - self.handle_subscription_frame(envelope, meta, request_started, sub_msg); - Ok(()) - } - StreamClientFrame::Op(stream_msg) => { - self.handle_actor_operation_frame(envelope, meta, request_started, stream_msg); - Ok(()) - } - } - } - - fn handle_cleanup_envelope(&self, envelope: &Envelope) -> bool { - if let Some(cleanup) = envelope.payload::() { - self.cleanup_session(cleanup.session_id); - return true; - } - - false - } - - fn ensure_active(&self) -> Result<(), DeliveryError> { - if !self.active.load(Ordering::Relaxed) { - return Err(DeliveryError::ActorStopped); - } - - Ok(()) - } - - fn handle_domain_publish_envelope(&self, envelope: &Envelope) -> bool { - if let Some(event) = envelope.payload::() { - if *envelope.destination().family() != event.family_id { - crate::observability::counter_inc("fitz_stream_publish_family_mismatch_total"); - return true; - } - self.handle_domain_publish(event); - return true; - } - - false - } - - fn extract_request(envelope: &Envelope) -> Result, DeliveryError> { - Ok(Some( - Self::request_from_envelope(envelope).ok_or(DeliveryError::ActorStopped)?, - )) - } - - fn record_request_start(&self) -> Option { - self.metrics - .as_ref() - .map(crate::domains::stream::StreamMetrics::record_request_start) - } - - fn parse_request_frame( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - frame: Result, - request_started: Option, - ) -> Option { - match frame { - Ok(frame) => Some(frame), - Err(error) => { - let response = Self::stream_error_response(error); - self.route_stream_response(envelope, meta, &response, request_started); - None - } - } - } - - fn record_operation(&self) { - if let Some(metrics) = &self.metrics { - metrics.counter_inc(STREAM_OPERATIONS_TOTAL); - } else { - crate::observability::counter_inc(STREAM_OPERATIONS_TOTAL); - } - } - - fn handle_subscription_frame( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - request_started: Option, - sub_msg: crate::domains::stream::protocol::StreamSubscriptionMessage, - ) { - use crate::domains::stream::protocol::StreamSubscriptionMessage; - - let response = match sub_msg { - StreamSubscriptionMessage::Subscribe { - family_id, - pattern, - session_id, - subscriber, - } => self.handle_stream_subscribe( - envelope, meta, family_id, &pattern, session_id, subscriber, - ), - StreamSubscriptionMessage::Unsubscribe { - family_id, - pattern, - session_id, - subscriber, - } => self.handle_stream_unsubscribe( - envelope, - meta, - family_id, - &pattern, - session_id, - &subscriber, - ), - }; - - self.refresh_metrics_gauges(); - self.route_stream_response(envelope, meta, &response, request_started); - } - - fn handle_stream_subscribe( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - family_id: crate::runtime::routing::RouteFamily, - pattern: &crate::runtime::routing::Route, - session_id: u64, - subscriber: crate::runtime::routing::RouteAddress, - ) -> StreamClientResponseBody { - if Self::valid_stream_subscription_request( - envelope, - meta, - family_id, - session_id, - &subscriber, - ) { - let compiled = match Self::compile_stream_subscription_pattern(pattern) { - Ok(compiled) => compiled, - Err(response) => return response, - }; - let mut families = self.subscriptions.families.lock(); - let state = families - .entry(family_id.as_u64()) - .or_insert_with(RoutedSubscriptionSet::new); - if let Some(subscription_id) = state.find_existing_id(session_id, pattern.as_str()) { - return StreamClientResponseBody::Ok { - session_id: Some(subscription_id), - data: vec![], - }; - } - if state.wildcard_registration_limit_reached(session_id, &compiled) { - return StreamClientResponseBody::SubscriptionError( - crate::domains::stream::StreamSubscriptionFailure::Limit, - ); - } - if let Ok(subscription_id) = self.subscriptions.next_id.fetch_update( - Ordering::Relaxed, - Ordering::Relaxed, - |current| current.checked_add(1), - ) { - state.insert( - family_id, - StreamSubscription { - pattern: compiled, - session_id, - subscription_id, - subscriber, - }, - ); - StreamClientResponseBody::Ok { - session_id: Some(subscription_id), - data: vec![], - } - } else { - if state.is_empty() { - families.remove(&family_id.as_u64()); - } - Self::stream_error_response("subscription ID space exhausted") - } - } else { - Self::stream_error_response("route family mismatch") - } - } - - fn handle_stream_unsubscribe( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - family_id: crate::runtime::routing::RouteFamily, - pattern: &crate::runtime::routing::Route, - session_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - ) -> StreamClientResponseBody { - if Self::valid_stream_subscription_request( - envelope, meta, family_id, session_id, subscriber, - ) { - if let Err(response) = Self::compile_stream_subscription_pattern(pattern) { - return response; - } - let mut families = self.subscriptions.families.lock(); - let remove_family = if let Some(state) = families.get_mut(&family_id.as_u64()) { - state.remove_session_pattern(family_id, session_id, pattern.as_str()); - state.is_empty() - } else { - false - }; - if remove_family { - families.remove(&family_id.as_u64()); - } - drop(families); - self.remove_pending_notifications_for_pattern(session_id, pattern.as_str()); - StreamClientResponseBody::Ok { - session_id: None, - data: vec![], - } - } else { - Self::stream_error_response("route family mismatch") - } - } - - fn compile_stream_subscription_pattern( - pattern: &crate::runtime::routing::Route, - ) -> Result { - let invalid_pattern = |error: String| { - StreamClientResponseBody::SubscriptionError( - crate::domains::stream::StreamSubscriptionFailure::InvalidPattern(error), - ) - }; - let compiled = crate::runtime::DomainKind::Stream - .descriptor() - .compile_registration_pattern(pattern.as_str()) - .map_err(invalid_pattern)?; - crate::domains::stream::route_grammar::classify_stream_route_shape(pattern.as_str()) - .map_err(invalid_pattern)?; - Ok(compiled) - } - - fn valid_stream_subscription_request( - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - family_id: crate::runtime::routing::RouteFamily, - session_id: u64, - subscriber: &crate::runtime::routing::RouteAddress, - ) -> bool { - family_id == meta.route_family - && *subscriber.family() == family_id - && session_id == meta.session_id - && envelope.source().is_none_or(|source| source == subscriber) - } - - fn handle_actor_operation_frame( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - request_started: Option, - stream_msg: crate::domains::stream::protocol::StreamMessage, - ) { - use crate::domains::stream::protocol::StreamMessage; - - let message_family = match &stream_msg { - StreamMessage::Begin { family_id, .. } - | StreamMessage::Read { family_id, .. } - | StreamMessage::Last { family_id, .. } - | StreamMessage::GetMetadata { family_id, .. } => Some(*family_id), - StreamMessage::Append { .. } - | StreamMessage::Commit { .. } - | StreamMessage::Rollback { .. } => None, - }; - if message_family.is_some_and(|family_id| family_id != meta.route_family) { - let response = Self::stream_error_response("route family mismatch"); - self.route_stream_response(envelope, meta, &response, request_started); - return; - } - - let (response, commit_notify, should_refresh_admin_snapshot) = match stream_msg { - StreamMessage::Begin { - family_id, - route, - ingest_metadata, - } => self.handle_begin_operation(meta, family_id, &route, ingest_metadata), - StreamMessage::Append { - session_id, - expected_offset, - body, - metadata, - discriminator, - } => self.handle_append_operation( - meta, - session_id, - expected_offset, - body, - metadata, - discriminator, - ), - StreamMessage::Commit { session_id, mode } => { - self.handle_commit_operation(meta, session_id, mode) - } - StreamMessage::Rollback { session_id } => { - self.handle_rollback_operation(meta, session_id) - } - StreamMessage::Read { - family_id, - route, - from_offset, - limit, - max_bytes, - filter, - cursor_fingerprint, - captured_watermark, - } => self.handle_read_operation(StreamReadExecution { - family_id, - route: &route, - from_offset, - limit, - max_bytes, - filter: filter.as_ref(), - cursor_fingerprint, - captured_watermark, - }), - StreamMessage::Last { family_id, route } => { - self.handle_last_operation(family_id, &route) - } - StreamMessage::GetMetadata { family_id, route } => { - self.handle_metadata_operation(family_id, &route) - } - }; - - if should_refresh_admin_snapshot { - self.mark_admin_snapshot_dirty(); - } - - if let Some((family_id, route, payload)) = commit_notify { - let event = crate::runtime::DomainPublishEvent::new(family_id, route, payload); - self.handle_domain_publish(&event); - } - - self.route_stream_response(envelope, meta, &response, request_started); - } - - fn handle_begin_operation( - &self, - meta: crate::runtime::ClientFrameMeta, - family_id: crate::runtime::routing::RouteFamily, - route: &Route, - ingest_metadata: Option, - ) -> ( - StreamClientResponseBody, - Option<(RouteFamily, Route, bytes::Bytes)>, - bool, - ) { - if family_id != meta.route_family { - return ( - Self::stream_error_response("route family mismatch"), - None, - false, - ); - } - - match Self::actor_key_for_route(family_id, route) { - Ok(key) => { - let Ok(stream_session_id) = self.next_session_id.fetch_update( - Ordering::Relaxed, - Ordering::Relaxed, - |current| current.checked_add(1), - ) else { - return ( - Self::stream_error_response("stream session ID space exhausted"), - None, - false, - ); - }; - - match self.get_or_create_actor(&key) { - Ok(actor) => { - match actor.lock().begin_append_session( - meta.session_id, - stream_session_id, - ingest_metadata, - ) { - Ok(session_id) => { - self.session_owners.lock().insert( - session_id, - StreamSessionOwner { - key, - owner_session_id: meta.session_id, - actor: actor.clone(), - }, - ); - self.counter_inc("fitz_stream_append_sessions_started_total"); - ( - StreamClientResponseBody::Ok { - session_id: Some(session_id), - data: vec![], - }, - None, - true, - ) - } - Err(error) => { - crate::observability::counter_inc( - "fitz_stream_append_conflicts_total", - ); - (Self::stream_error_response(error), None, false) - } - } - } - Err(error) => (Self::stream_error_response(error), None, false), - } - } - Err(error) => (Self::stream_error_response(error), None, false), - } - } - - fn session_owner_for( - &self, - owner_session_id: u64, - family_id: RouteFamily, - stream_session_id: u64, - ) -> Option { - self.session_owners - .lock() - .get(&stream_session_id) - .filter(|owner| { - owner.owner_session_id == owner_session_id - && owner.key.family_id == family_id.as_u64() - }) - .cloned() - } - - fn session_actor_for( - &self, - owner_session_id: u64, - family_id: RouteFamily, - stream_session_id: u64, - ) -> Option>> { - self.session_owners - .lock() - .get(&stream_session_id) - .filter(|owner| { - owner.owner_session_id == owner_session_id - && owner.key.family_id == family_id.as_u64() - }) - .map(|owner| owner.actor.clone()) - } - - fn handle_append_operation( - &self, - meta: crate::runtime::ClientFrameMeta, - session_id: u64, - expected_offset: u64, - body: bytes::Bytes, - metadata: Option, - discriminator: Option, - ) -> ( - StreamClientResponseBody, - Option<(RouteFamily, Route, bytes::Bytes)>, - bool, - ) { - let Some(actor) = self.session_actor_for(meta.session_id, meta.route_family, session_id) - else { - return ( - Self::stream_error_response(StreamStoreError::SessionNotFound.client_message()), - None, - false, - ); - }; - let append_result = { - let mut actor = actor.lock(); - actor.append_to_session_with_discriminator_for_owner( - meta.session_id, - session_id, - expected_offset, - body, - metadata, - discriminator, - ) - }; - match append_result { - Ok(assigned_offset) => { - let mut encoder = PayloadEncoder::new(); - encoder.put_u64(assigned_offset); - ( - StreamClientResponseBody::Ok { - session_id: None, - data: encoder.finish(), - }, - None, - false, - ) - } - Err(error) => (Self::stream_error_response(error), None, false), - } - } - - fn handle_commit_operation( - &self, - meta: crate::runtime::ClientFrameMeta, - session_id: u64, - mode: crate::domains::stream::protocol::StreamWriteMode, - ) -> ( - StreamClientResponseBody, - Option<(RouteFamily, Route, bytes::Bytes)>, - bool, - ) { - let mode = if mode == crate::domains::stream::protocol::StreamWriteMode::Sync { - self.sync_write_mode - } else { - mode - }; - let Some(owner) = self.session_owner_for(meta.session_id, meta.route_family, session_id) - else { - return ( - Self::stream_error_response(StreamStoreError::SessionNotFound.client_message()), - None, - false, - ); - }; - let commit_result = { - let mut actor = owner.actor.lock(); - actor.commit_session_for_owner(meta.session_id, session_id, mode) - }; - match commit_result { - Ok(commit) => { - self.session_owners.lock().remove(&session_id); - self.counter_inc("fitz_stream_append_sessions_ended_total"); - self.notify_area_batch_committed( - RouteFamily::try_from(owner.key.family_id) - .expect("stream family IDs originate from RouteFamily"), - &owner.key.realm, - &owner.key.area, - &crate::domains::stream::protocol::BatchCommitted { - first_area_offset: commit.first_area_offset, - last_area_offset: commit.last_area_offset, - first_realm_offset: commit.first_realm_offset, - last_realm_offset: commit.last_realm_offset, - first_global_offset: commit.first_global_offset, - last_global_offset: commit.last_global_offset, - }, - ); - let payload = Self::encode_stream_commit_notify_payload(&commit); - ( - StreamClientResponseBody::Ok { - session_id: None, - data: vec![], - }, - Some(( - RouteFamily::try_from(owner.key.family_id) - .expect("stream family IDs originate from RouteFamily"), - owner.key.resource_route(), - payload, - )), - true, - ) - } - Err(error) => { - self.handle_visibility_advance(meta.route_family); - (Self::stream_error_response(error), None, false) - } - } - } - - fn handle_rollback_operation( - &self, - meta: crate::runtime::ClientFrameMeta, - session_id: u64, - ) -> ( - StreamClientResponseBody, - Option<(RouteFamily, Route, bytes::Bytes)>, - bool, - ) { - let Some(owner) = self.session_owner_for(meta.session_id, meta.route_family, session_id) - else { - return ( - Self::stream_error_response(StreamStoreError::SessionNotFound.client_message()), - None, - false, - ); - }; - let rollback_result = { - let mut actor = owner.actor.lock(); - actor.rollback_session_for_owner(meta.session_id, session_id) - }; - match rollback_result { - Ok(()) => { - self.session_owners.lock().remove(&session_id); - self.counter_inc("fitz_stream_append_sessions_ended_total"); - self.handle_visibility_advance(meta.route_family); - ( - StreamClientResponseBody::Ok { - session_id: None, - data: vec![], - }, - None, - true, - ) - } - Err(error) => (Self::stream_error_response(error), None, false), - } - } - - fn encode_operation_result( - result: Result, String>, - ) -> ( - StreamClientResponseBody, - Option<(RouteFamily, Route, bytes::Bytes)>, - bool, - ) { - match result { - Ok(data) => ( - StreamClientResponseBody::Ok { - session_id: None, - data, - }, - None, - false, - ), - Err(error) => (Self::stream_error_response(error), None, false), - } - } +#[cfg(test)] +mod work_key_tests { + use super::*; - fn handle_read_operation( - &self, - request: StreamReadExecution<'_>, - ) -> ( - StreamClientResponseBody, - Option<(RouteFamily, Route, bytes::Bytes)>, - bool, - ) { - Self::encode_operation_result(self.encode_read_response_data(request)) - } + #[test] + fn should_share_resource_key_between_exact_selectors_and_writes() { + // Arrange + let family = RouteFamily::new(7); + let route = Route::new("stream://acme/orders/42"); + let write_key = StreamDomainCore::actor_key_for_route(family, &route) + .map(StreamWorkKey::Resource) + .unwrap(); - fn handle_last_operation( - &self, - family_id: crate::runtime::routing::RouteFamily, - route: &Route, - ) -> ( - StreamClientResponseBody, - Option<(RouteFamily, Route, bytes::Bytes)>, - bool, - ) { - Self::encode_operation_result(self.encode_last_response_data(family_id, route)) - } + // Act + let read_key = StreamDomainSink::selector_work_key(family, &route); - fn handle_metadata_operation( - &self, - family_id: crate::runtime::routing::RouteFamily, - route: &Route, - ) -> ( - StreamClientResponseBody, - Option<(RouteFamily, Route, bytes::Bytes)>, - bool, - ) { - Self::encode_operation_result(self.encode_metadata_response_data(family_id, route)) + // Assert + assert_eq!(read_key, write_key); } - fn request_from_envelope(envelope: &Envelope) -> Option { - if let Some(request) = envelope.payload::() { - return Some(request.clone()); - } + #[test] + fn should_keep_broad_selectors_on_selector_keys() { + // Arrange + let route = Route::new("stream://acme/orders/*"); - #[cfg(test)] - { - let frame_ctx = envelope.payload::()?.clone(); - let subscriber = envelope.source().cloned().unwrap_or_else(|| { - RouteAddress::new( - *envelope.destination().family(), - Route::new(format!("inbox://session/{}", frame_ctx.session_id)), - ) - }); - let meta = crate::runtime::ClientFrameMeta::new( - frame_ctx.session_id, - test_client_channel_from_protocol(frame_ctx.channel_id), - frame_ctx.msg_type.as_u16(), - frame_ctx.route_family, - ); - let parsed = crate::dispatch::protocol::stream_codec::parse_request( - &frame_ctx, - &frame_ctx.payload, - *envelope.destination().family(), - crate::session::SessionId(frame_ctx.session_id), - subscriber, - ); - Some(StreamClientRequest::new(meta, parsed)) - } + // Act + let key = StreamDomainSink::selector_work_key(RouteFamily::new(7), &route); - #[cfg(not(test))] - { - None - } - } - - fn route_stream_response( - &self, - envelope: &Envelope, - meta: crate::runtime::ClientFrameMeta, - response: &StreamClientResponseBody, - request_started: Option, - ) { - #[cfg(test)] - let response_ctx = { - let mut payload_encoder = - crate::dispatch::protocol::payload_codec::PayloadEncoder::with_capacity(256); - let response_bytes = crate::dispatch::protocol::stream_codec::encode_response_into( - &mut payload_encoder, - meta.message_type, - response, - ); - FrameContext::new( - meta.session_id, - test_protocol_channel_from_client(meta.channel), - crate::dispatch::protocol::tlv::MessageType::new(meta.message_type), - bytes::Bytes::from(response_bytes), - meta.route_family, - ) - }; - - #[cfg(not(test))] - let response_ctx = - crate::domains::stream::StreamClientResponse::new(meta, response.clone()); - - if let Some(response_envelope) = envelope.try_reply_to(response_ctx) { - if let Err(error) = self.router.route(response_envelope) { - if let Some(metrics) = self.metrics.as_ref() { - metrics.record_response_drop(); - } else { - crate::observability::counter_inc( - crate::domains::stream::metrics::METRIC_RESPONSE_DROPS_TOTAL, - ); - } - tracing::warn!( - domain = "stream", - session_id = meta.session_id, - route_family = meta.route_family.as_u64(), - error = %error, - "Dropped best-effort Stream response" - ); - } - } - - if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { - if Self::stream_response_is_failure(response) { - metrics.record_failure(started_at); - } else { - metrics.record_success(started_at); - } - } - } -} - -#[cfg(test)] -fn test_client_channel_from_protocol( - channel: crate::dispatch::protocol::frame::ChannelId, -) -> crate::runtime::ClientChannel { - match channel { - crate::dispatch::protocol::frame::ChannelId::Control => { - crate::runtime::ClientChannel::Control - } - crate::dispatch::protocol::frame::ChannelId::Pub => crate::runtime::ClientChannel::Pub, - crate::dispatch::protocol::frame::ChannelId::Sub => crate::runtime::ClientChannel::Sub, - crate::dispatch::protocol::frame::ChannelId::Rpc => crate::runtime::ClientChannel::Rpc, - crate::dispatch::protocol::frame::ChannelId::Lease => crate::runtime::ClientChannel::Lease, - crate::dispatch::protocol::frame::ChannelId::Internal => { - crate::runtime::ClientChannel::Internal - } - } -} - -#[cfg(test)] -fn test_protocol_channel_from_client( - channel: crate::runtime::ClientChannel, -) -> crate::dispatch::protocol::frame::ChannelId { - match channel { - crate::runtime::ClientChannel::Control => { - crate::dispatch::protocol::frame::ChannelId::Control - } - crate::runtime::ClientChannel::Pub => crate::dispatch::protocol::frame::ChannelId::Pub, - crate::runtime::ClientChannel::Sub => crate::dispatch::protocol::frame::ChannelId::Sub, - crate::runtime::ClientChannel::Rpc => crate::dispatch::protocol::frame::ChannelId::Rpc, - crate::runtime::ClientChannel::Lease => crate::dispatch::protocol::frame::ChannelId::Lease, - crate::runtime::ClientChannel::Internal => { - crate::dispatch::protocol::frame::ChannelId::Internal - } + // Assert + assert_eq!(key, StreamWorkKey::Selector(route.as_str().to_owned())); } } diff --git a/src/domains/stream/sink/mailbox_sink_impl/envelope_dispatch.rs b/src/domains/stream/sink/mailbox_sink_impl/envelope_dispatch.rs new file mode 100644 index 00000000..be0d2790 --- /dev/null +++ b/src/domains/stream/sink/mailbox_sink_impl/envelope_dispatch.rs @@ -0,0 +1,279 @@ +//! Envelope intake: classify what arrived, hand it to the right frame +//! handler, and route the reply back to the client session. + +#[cfg(test)] +use super::FrameContext; +use super::{ + DeliveryError, Envelope, Ordering, StreamClientFrame, StreamClientRequest, + StreamClientResponseBody, StreamDomainCore, STREAM_OPERATIONS_TOTAL, +}; +#[cfg(test)] +use super::{Route, RouteAddress}; + +impl StreamDomainCore { + pub(in crate::domains::stream::sink) fn deliver_envelope( + &self, + envelope: &Envelope, + ) -> Result<(), DeliveryError> { + if self.handle_cleanup_envelope(envelope) { + return Ok(()); + } + self.ensure_active()?; + + if self.handle_domain_publish_envelope(envelope) { + return Ok(()); + } + + let Some(request) = Self::extract_request(envelope)? else { + return Ok(()); + }; + let meta = request.meta; + let request_started = self.record_request_start(); + + if meta.route_family != *envelope.destination().family() + || envelope + .source() + .is_some_and(|source| *source.family() != meta.route_family) + { + let response = Self::stream_error_response("route family mismatch"); + let response_meta = envelope.source().map_or(meta, |source| { + let mut response_meta = meta; + response_meta.route_family = *source.family(); + response_meta + }); + self.route_stream_response(envelope, response_meta, &response, request_started); + return Ok(()); + } + + let Some(parsed_frame) = + self.parse_request_frame(envelope, meta, request.frame, request_started) + else { + return Ok(()); + }; + + let session_mutation = matches!( + parsed_frame, + StreamClientFrame::Sub(_) + | StreamClientFrame::Op( + crate::domains::stream::protocol::StreamMessage::Begin { .. } + | crate::domains::stream::protocol::StreamMessage::Append { .. } + | crate::domains::stream::protocol::StreamMessage::Commit { .. } + | crate::domains::stream::protocol::StreamMessage::Rollback { .. } + ) + ); + if session_mutation && self.cleaned_up_sessions.lock().contains(meta.session_id) { + let response = Self::stream_error_response("session has been cleaned up"); + self.route_stream_response(envelope, meta, &response, request_started); + return Ok(()); + } + + self.record_operation(); + + match parsed_frame { + StreamClientFrame::Sub(sub_msg) => { + self.handle_subscription_frame(envelope, meta, request_started, sub_msg); + Ok(()) + } + StreamClientFrame::Op(stream_msg) => { + self.handle_actor_operation_frame(envelope, meta, request_started, stream_msg); + Ok(()) + } + } + } + + fn handle_cleanup_envelope(&self, envelope: &Envelope) -> bool { + if let Some(cleanup) = envelope.payload::() { + self.cleanup_session(cleanup.session_id); + return true; + } + + false + } + + fn ensure_active(&self) -> Result<(), DeliveryError> { + if !self.active.load(Ordering::Relaxed) { + return Err(DeliveryError::ActorStopped); + } + + Ok(()) + } + + fn handle_domain_publish_envelope(&self, envelope: &Envelope) -> bool { + if let Some(event) = envelope.payload::() { + if *envelope.destination().family() != event.family_id { + crate::observability::counter_inc("fitz_stream_publish_family_mismatch_total"); + return true; + } + self.handle_domain_publish(event); + return true; + } + + false + } + + fn extract_request(envelope: &Envelope) -> Result, DeliveryError> { + Ok(Some( + Self::request_from_envelope(envelope).ok_or(DeliveryError::ActorStopped)?, + )) + } + + fn record_request_start(&self) -> Option { + self.metrics + .as_ref() + .map(crate::domains::stream::StreamMetrics::record_request_start) + } + + fn parse_request_frame( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + frame: Result, + request_started: Option, + ) -> Option { + match frame { + Ok(frame) => Some(frame), + Err(error) => { + let response = Self::stream_error_response(error); + self.route_stream_response(envelope, meta, &response, request_started); + None + } + } + } + + fn record_operation(&self) { + if let Some(metrics) = &self.metrics { + metrics.counter_inc(STREAM_OPERATIONS_TOTAL); + } else { + crate::observability::counter_inc(STREAM_OPERATIONS_TOTAL); + } + } + + pub(super) fn request_from_envelope(envelope: &Envelope) -> Option { + if let Some(request) = envelope.payload::() { + return Some(request.clone()); + } + + #[cfg(test)] + { + let frame_ctx = envelope.payload::()?.clone(); + let subscriber = envelope.source().cloned().unwrap_or_else(|| { + RouteAddress::new( + *envelope.destination().family(), + Route::new(format!("inbox://session/{}", frame_ctx.session_id)), + ) + }); + let meta = crate::runtime::ClientFrameMeta::new( + frame_ctx.session_id, + test_client_channel_from_protocol(frame_ctx.channel_id), + frame_ctx.msg_type.as_u16(), + frame_ctx.route_family, + ); + let parsed = crate::dispatch::protocol::stream_codec::parse_request( + &frame_ctx, + &frame_ctx.payload, + *envelope.destination().family(), + crate::session::SessionId(frame_ctx.session_id), + subscriber, + ); + Some(StreamClientRequest::new(meta, parsed)) + } + + #[cfg(not(test))] + { + None + } + } + + pub(super) fn route_stream_response( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + response: &StreamClientResponseBody, + request_started: Option, + ) { + #[cfg(test)] + let response_ctx = { + let mut payload_encoder = + crate::dispatch::protocol::payload_codec::PayloadEncoder::with_capacity(256); + let response_bytes = crate::dispatch::protocol::stream_codec::encode_response_into( + &mut payload_encoder, + meta.message_type, + response, + ); + FrameContext::new( + meta.session_id, + test_protocol_channel_from_client(meta.channel), + crate::dispatch::protocol::tlv::MessageType::new(meta.message_type), + bytes::Bytes::from(response_bytes), + meta.route_family, + ) + }; + + #[cfg(not(test))] + let response_ctx = + crate::domains::stream::StreamClientResponse::new(meta, response.clone()); + + if let Some(response_envelope) = envelope.try_reply_to(response_ctx) { + if let Err(error) = self.router.route(response_envelope) { + if let Some(metrics) = self.metrics.as_ref() { + metrics.record_response_drop(); + } else { + crate::observability::counter_inc( + crate::domains::stream::metrics::METRIC_RESPONSE_DROPS_TOTAL, + ); + } + tracing::warn!( + domain = "stream", + session_id = meta.session_id, + route_family = meta.route_family.as_u64(), + error = %error, + "Dropped best-effort Stream response" + ); + } + } + + if let (Some(metrics), Some(started_at)) = (self.metrics.as_ref(), request_started) { + if Self::stream_response_is_failure(response) { + metrics.record_failure(started_at); + } else { + metrics.record_success(started_at); + } + } + } +} + +#[cfg(test)] +fn test_client_channel_from_protocol( + channel: crate::dispatch::protocol::frame::ChannelId, +) -> crate::runtime::ClientChannel { + match channel { + crate::dispatch::protocol::frame::ChannelId::Control => { + crate::runtime::ClientChannel::Control + } + crate::dispatch::protocol::frame::ChannelId::Pub => crate::runtime::ClientChannel::Pub, + crate::dispatch::protocol::frame::ChannelId::Sub => crate::runtime::ClientChannel::Sub, + crate::dispatch::protocol::frame::ChannelId::Rpc => crate::runtime::ClientChannel::Rpc, + crate::dispatch::protocol::frame::ChannelId::Lease => crate::runtime::ClientChannel::Lease, + crate::dispatch::protocol::frame::ChannelId::Internal => { + crate::runtime::ClientChannel::Internal + } + } +} + +#[cfg(test)] +fn test_protocol_channel_from_client( + channel: crate::runtime::ClientChannel, +) -> crate::dispatch::protocol::frame::ChannelId { + match channel { + crate::runtime::ClientChannel::Control => { + crate::dispatch::protocol::frame::ChannelId::Control + } + crate::runtime::ClientChannel::Pub => crate::dispatch::protocol::frame::ChannelId::Pub, + crate::runtime::ClientChannel::Sub => crate::dispatch::protocol::frame::ChannelId::Sub, + crate::runtime::ClientChannel::Rpc => crate::dispatch::protocol::frame::ChannelId::Rpc, + crate::runtime::ClientChannel::Lease => crate::dispatch::protocol::frame::ChannelId::Lease, + crate::runtime::ClientChannel::Internal => { + crate::dispatch::protocol::frame::ChannelId::Internal + } + } +} diff --git a/src/domains/stream/sink/mailbox_sink_impl/session_operations.rs b/src/domains/stream/sink/mailbox_sink_impl/session_operations.rs new file mode 100644 index 00000000..956111b8 --- /dev/null +++ b/src/domains/stream/sink/mailbox_sink_impl/session_operations.rs @@ -0,0 +1,412 @@ +//! Append-session and read frames: the operations that reach `StreamActor`. + +use super::{ + Arc, Envelope, IngestMetadata, Mutex, Ordering, PayloadEncoder, Route, RouteFamily, + StreamActor, StreamClientResponseBody, StreamDiscriminator, StreamDomainCore, + StreamReadExecution, StreamSessionOwner, StreamStoreError, +}; +use crate::domains::stream::sink::model::OperationOutcome; + +impl StreamDomainCore { + pub(super) fn handle_actor_operation_frame( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + request_started: Option, + stream_msg: crate::domains::stream::protocol::StreamMessage, + ) { + use crate::domains::stream::protocol::StreamMessage; + + let message_family = match &stream_msg { + StreamMessage::Begin { family_id, .. } + | StreamMessage::Read { family_id, .. } + | StreamMessage::Last { family_id, .. } + | StreamMessage::GetMetadata { family_id, .. } => Some(*family_id), + StreamMessage::Append { .. } + | StreamMessage::Commit { .. } + | StreamMessage::Rollback { .. } => None, + }; + if message_family.is_some_and(|family_id| family_id != meta.route_family) { + let response = Self::stream_error_response("route family mismatch"); + self.route_stream_response(envelope, meta, &response, request_started); + return; + } + + let outcome: OperationOutcome = (match stream_msg { + StreamMessage::Begin { + family_id, + route, + ingest_metadata, + } => self.handle_begin_operation(meta, family_id, &route, ingest_metadata), + StreamMessage::Append { + session_id, + expected_offset, + body, + metadata, + discriminator, + } => self.handle_append_operation( + meta, + session_id, + expected_offset, + body, + metadata, + discriminator, + ), + StreamMessage::Commit { session_id, mode } => { + self.handle_commit_operation(meta, session_id, mode) + } + StreamMessage::Rollback { session_id } => { + self.handle_rollback_operation(meta, session_id) + } + StreamMessage::Read { + family_id, + route, + from_offset, + limit, + max_bytes, + filter, + cursor_fingerprint, + captured_watermark, + } => self.handle_read_operation(StreamReadExecution { + family_id, + route: &route, + from_offset, + limit, + max_bytes, + filter: filter.as_ref(), + cursor_fingerprint, + captured_watermark, + }), + StreamMessage::Last { family_id, route } => { + self.handle_last_operation(family_id, &route) + } + StreamMessage::GetMetadata { family_id, route } => { + self.handle_metadata_operation(family_id, &route) + } + }) + .into(); + + if outcome.admin_dirty { + self.mark_admin_snapshot_dirty(); + } + + if let Some(notification) = outcome.notification { + let event = crate::runtime::DomainPublishEvent::new( + notification.family, + notification.route, + notification.payload, + ); + self.handle_domain_publish(&event); + } + + self.route_stream_response(envelope, meta, &outcome.response, request_started); + } + + fn handle_begin_operation( + &self, + meta: crate::runtime::ClientFrameMeta, + family_id: crate::runtime::routing::RouteFamily, + route: &Route, + ingest_metadata: Option, + ) -> ( + StreamClientResponseBody, + Option<(RouteFamily, Route, bytes::Bytes)>, + bool, + ) { + if family_id != meta.route_family { + return ( + Self::stream_error_response("route family mismatch"), + None, + false, + ); + } + + match Self::actor_key_for_route(family_id, route) { + Ok(key) => { + let Ok(stream_session_id) = self.next_session_id.fetch_update( + Ordering::Relaxed, + Ordering::Relaxed, + |current| current.checked_add(1), + ) else { + return ( + Self::stream_error_response("stream session ID space exhausted"), + None, + false, + ); + }; + + match self.get_or_create_actor(&key) { + Ok(actor) => { + match actor.lock().begin_append_session( + meta.session_id, + stream_session_id, + ingest_metadata, + ) { + Ok(session_id) => { + self.session_owners.lock().insert( + session_id, + StreamSessionOwner { + key, + owner_session_id: meta.session_id, + actor: actor.clone(), + }, + ); + self.counter_inc("fitz_stream_append_sessions_started_total"); + ( + StreamClientResponseBody::Ok { + session_id: Some(session_id), + data: vec![], + }, + None, + true, + ) + } + Err(error) => { + crate::observability::counter_inc( + "fitz_stream_append_conflicts_total", + ); + (Self::stream_error_response(error), None, false) + } + } + } + Err(error) => (Self::stream_error_response(error), None, false), + } + } + Err(error) => (Self::stream_error_response(error), None, false), + } + } + + fn session_owner_for( + &self, + owner_session_id: u64, + family_id: RouteFamily, + stream_session_id: u64, + ) -> Option { + self.session_owners + .lock() + .get(&stream_session_id) + .filter(|owner| { + owner.owner_session_id == owner_session_id && owner.key.family == family_id + }) + .cloned() + } + + fn session_actor_for( + &self, + owner_session_id: u64, + family_id: RouteFamily, + stream_session_id: u64, + ) -> Option>> { + self.session_owners + .lock() + .get(&stream_session_id) + .filter(|owner| { + owner.owner_session_id == owner_session_id && owner.key.family == family_id + }) + .map(|owner| owner.actor.clone()) + } + + fn handle_append_operation( + &self, + meta: crate::runtime::ClientFrameMeta, + session_id: u64, + expected_offset: u64, + body: bytes::Bytes, + metadata: Option, + discriminator: Option, + ) -> ( + StreamClientResponseBody, + Option<(RouteFamily, Route, bytes::Bytes)>, + bool, + ) { + let Some(actor) = self.session_actor_for(meta.session_id, meta.route_family, session_id) + else { + return ( + Self::stream_error_response(StreamStoreError::SessionNotFound.client_message()), + None, + false, + ); + }; + let append_result = { + let mut actor = actor.lock(); + actor.append_to_session_with_discriminator_for_owner( + meta.session_id, + session_id, + expected_offset, + body, + metadata, + discriminator, + ) + }; + match append_result { + Ok(assigned_offset) => { + let mut encoder = PayloadEncoder::new(); + encoder.put_u64(assigned_offset); + ( + StreamClientResponseBody::Ok { + session_id: None, + data: encoder.finish(), + }, + None, + false, + ) + } + Err(error) => (Self::stream_error_response(error), None, false), + } + } + + fn handle_commit_operation( + &self, + meta: crate::runtime::ClientFrameMeta, + session_id: u64, + mode: crate::domains::stream::protocol::StreamWriteMode, + ) -> ( + StreamClientResponseBody, + Option<(RouteFamily, Route, bytes::Bytes)>, + bool, + ) { + let mode = if mode == crate::domains::stream::protocol::StreamWriteMode::Sync { + self.sync_write_mode + } else { + mode + }; + let Some(owner) = self.session_owner_for(meta.session_id, meta.route_family, session_id) + else { + return ( + Self::stream_error_response(StreamStoreError::SessionNotFound.client_message()), + None, + false, + ); + }; + let commit_result = { + let mut actor = owner.actor.lock(); + actor.commit_session_for_owner(meta.session_id, session_id, mode) + }; + match commit_result { + Ok(commit) => { + self.session_owners.lock().remove(&session_id); + self.counter_inc("fitz_stream_append_sessions_ended_total"); + self.notify_area_batch_committed( + owner.key.family, + &owner.key.realm, + &owner.key.area, + &crate::domains::stream::protocol::BatchCommitted { + first_area_offset: commit.first_area_offset, + last_area_offset: commit.last_area_offset, + first_realm_offset: commit.first_realm_offset, + last_realm_offset: commit.last_realm_offset, + first_global_offset: commit.first_global_offset, + last_global_offset: commit.last_global_offset, + }, + ); + let payload = Self::encode_stream_commit_notify_payload(&commit); + ( + StreamClientResponseBody::Ok { + session_id: None, + data: vec![], + }, + Some((owner.key.family, owner.key.resource_route(), payload)), + true, + ) + } + Err(error) => { + self.handle_visibility_advance(meta.route_family); + (Self::stream_error_response(error), None, false) + } + } + } + + fn handle_rollback_operation( + &self, + meta: crate::runtime::ClientFrameMeta, + session_id: u64, + ) -> ( + StreamClientResponseBody, + Option<(RouteFamily, Route, bytes::Bytes)>, + bool, + ) { + let Some(owner) = self.session_owner_for(meta.session_id, meta.route_family, session_id) + else { + return ( + Self::stream_error_response(StreamStoreError::SessionNotFound.client_message()), + None, + false, + ); + }; + let rollback_result = { + let mut actor = owner.actor.lock(); + actor.rollback_session_for_owner(meta.session_id, session_id) + }; + match rollback_result { + Ok(()) => { + self.session_owners.lock().remove(&session_id); + self.counter_inc("fitz_stream_append_sessions_ended_total"); + self.handle_visibility_advance(meta.route_family); + ( + StreamClientResponseBody::Ok { + session_id: None, + data: vec![], + }, + None, + true, + ) + } + Err(error) => (Self::stream_error_response(error), None, false), + } + } + + fn encode_operation_result( + result: Result, String>, + ) -> ( + StreamClientResponseBody, + Option<(RouteFamily, Route, bytes::Bytes)>, + bool, + ) { + match result { + Ok(data) => ( + StreamClientResponseBody::Ok { + session_id: None, + data, + }, + None, + false, + ), + Err(error) => (Self::stream_error_response(error), None, false), + } + } + + fn handle_read_operation( + &self, + request: StreamReadExecution<'_>, + ) -> ( + StreamClientResponseBody, + Option<(RouteFamily, Route, bytes::Bytes)>, + bool, + ) { + Self::encode_operation_result(self.encode_read_response_data(request)) + } + + fn handle_last_operation( + &self, + family_id: crate::runtime::routing::RouteFamily, + route: &Route, + ) -> ( + StreamClientResponseBody, + Option<(RouteFamily, Route, bytes::Bytes)>, + bool, + ) { + Self::encode_operation_result(self.encode_last_response_data(family_id, route)) + } + + fn handle_metadata_operation( + &self, + family_id: crate::runtime::routing::RouteFamily, + route: &Route, + ) -> ( + StreamClientResponseBody, + Option<(RouteFamily, Route, bytes::Bytes)>, + bool, + ) { + Self::encode_operation_result(self.encode_metadata_response_data(family_id, route)) + } +} diff --git a/src/domains/stream/sink/mailbox_sink_impl/subscription_frames.rs b/src/domains/stream/sink/mailbox_sink_impl/subscription_frames.rs new file mode 100644 index 00000000..9e736aad --- /dev/null +++ b/src/domains/stream/sink/mailbox_sink_impl/subscription_frames.rs @@ -0,0 +1,176 @@ +//! Subscribe and unsubscribe frames. The sink owns stream subscription state +//! outright; these never reach `StreamActor`. + +use super::{ + Envelope, Ordering, RoutedSubscriptionSet, StreamClientResponseBody, StreamDomainCore, + StreamSubscription, +}; + +impl StreamDomainCore { + pub(super) fn handle_subscription_frame( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + request_started: Option, + sub_msg: crate::domains::stream::protocol::StreamSubscriptionMessage, + ) { + use crate::domains::stream::protocol::StreamSubscriptionMessage; + + let response = match sub_msg { + StreamSubscriptionMessage::Subscribe { + family_id, + pattern, + session_id, + subscriber, + } => self.handle_stream_subscribe( + envelope, meta, family_id, &pattern, session_id, subscriber, + ), + StreamSubscriptionMessage::Unsubscribe { + family_id, + pattern, + session_id, + subscriber, + } => self.handle_stream_unsubscribe( + envelope, + meta, + family_id, + &pattern, + session_id, + &subscriber, + ), + }; + + self.refresh_metrics_gauges(); + self.route_stream_response(envelope, meta, &response, request_started); + } + + fn handle_stream_subscribe( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + family_id: crate::runtime::routing::RouteFamily, + pattern: &crate::runtime::routing::Route, + session_id: u64, + subscriber: crate::runtime::routing::RouteAddress, + ) -> StreamClientResponseBody { + if Self::valid_stream_subscription_request( + envelope, + meta, + family_id, + session_id, + &subscriber, + ) { + let compiled = match Self::compile_stream_subscription_pattern(pattern) { + Ok(compiled) => compiled, + Err(response) => return response, + }; + let mut families = self.subscriptions.families.lock(); + let state = families + .entry(family_id.as_u64()) + .or_insert_with(RoutedSubscriptionSet::new); + if let Some(subscription_id) = state.find_existing_id(session_id, pattern.as_str()) { + return StreamClientResponseBody::Ok { + session_id: Some(subscription_id), + data: vec![], + }; + } + if state.wildcard_registration_limit_reached(session_id, &compiled) { + return StreamClientResponseBody::SubscriptionError( + crate::domains::stream::StreamSubscriptionFailure::Limit, + ); + } + if let Ok(subscription_id) = self.subscriptions.next_id.fetch_update( + Ordering::Relaxed, + Ordering::Relaxed, + |current| current.checked_add(1), + ) { + state.insert( + family_id, + StreamSubscription { + pattern: compiled, + session_id, + subscription_id, + subscriber, + }, + ); + StreamClientResponseBody::Ok { + session_id: Some(subscription_id), + data: vec![], + } + } else { + if state.is_empty() { + families.remove(&family_id.as_u64()); + } + Self::stream_error_response("subscription ID space exhausted") + } + } else { + Self::stream_error_response("route family mismatch") + } + } + + fn handle_stream_unsubscribe( + &self, + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + family_id: crate::runtime::routing::RouteFamily, + pattern: &crate::runtime::routing::Route, + session_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + ) -> StreamClientResponseBody { + if Self::valid_stream_subscription_request( + envelope, meta, family_id, session_id, subscriber, + ) { + if let Err(response) = Self::compile_stream_subscription_pattern(pattern) { + return response; + } + let mut families = self.subscriptions.families.lock(); + let remove_family = if let Some(state) = families.get_mut(&family_id.as_u64()) { + state.remove_session_pattern(family_id, session_id, pattern.as_str()); + state.is_empty() + } else { + false + }; + if remove_family { + families.remove(&family_id.as_u64()); + } + drop(families); + self.remove_pending_notifications_for_pattern(session_id, pattern.as_str()); + StreamClientResponseBody::Ok { + session_id: None, + data: vec![], + } + } else { + Self::stream_error_response("route family mismatch") + } + } + + fn compile_stream_subscription_pattern( + pattern: &crate::runtime::routing::Route, + ) -> Result { + let invalid_pattern = |error: String| { + StreamClientResponseBody::SubscriptionError( + crate::domains::stream::StreamSubscriptionFailure::InvalidPattern(error), + ) + }; + let compiled = crate::runtime::DomainKind::Stream + .descriptor() + .compile_registration_pattern(pattern.as_str()) + .map_err(invalid_pattern)?; + crate::domains::stream::route_grammar::classify_stream_route_shape(pattern.as_str()) + .map_err(invalid_pattern)?; + Ok(compiled) + } + + fn valid_stream_subscription_request( + envelope: &Envelope, + meta: crate::runtime::ClientFrameMeta, + family_id: crate::runtime::routing::RouteFamily, + session_id: u64, + subscriber: &crate::runtime::routing::RouteAddress, + ) -> bool { + family_id == meta.route_family + && *subscriber.family() == family_id + && session_id == meta.session_id + && envelope.source().is_none_or(|source| source == subscriber) + } +} diff --git a/src/domains/stream/sink/mod.rs b/src/domains/stream/sink/mod.rs index 7a4f8a78..8cbf6000 100644 --- a/src/domains/stream/sink/mod.rs +++ b/src/domains/stream/sink/mod.rs @@ -1,8 +1,14 @@ -mod domain_sink_impl; +mod cleanup; +mod delivery; +mod facade; mod mailbox_sink_impl; mod model; +mod observability; +mod reads; -pub use model::{AdminStreamReadRequest, StreamDomainSink, StreamStorageWriteOptions}; +pub use model::{ + AdminStreamReadRequest, StreamDomainSink, StreamSinkInitError, StreamStorageWriteOptions, +}; #[cfg(test)] use model::*; diff --git a/src/domains/stream/sink/model.rs b/src/domains/stream/sink/model.rs index 125372f7..b19e07cb 100644 --- a/src/domains/stream/sink/model.rs +++ b/src/domains/stream/sink/model.rs @@ -1,21 +1,32 @@ pub(super) use crate::dispatch::protocol::payload_codec::PayloadEncoder; -pub(super) use crate::domains::stream::store::StreamAdminRecord; pub(super) use crate::domains::stream::StreamMetrics; pub(super) use crate::domains::stream::{ - ReadResponse, StreamActor, StreamClientFrame, StreamClientRequest, StreamClientResponseBody, + StreamActor, StreamClientFrame, StreamClientRequest, StreamClientResponseBody, StreamFilteredReason, StreamMetadata, StreamReadItem, StreamRecord, StreamStorageLayout, StreamStore, }; pub(super) use crate::domains::subscription_state::{RoutedSubscription, RoutedSubscriptionSet}; pub(super) use crate::runtime::routing::{route_triplet, Route, RouteAddress, RouteFamily}; pub(super) use crate::runtime::{ - DeliveryError, Envelope, FamilyActorPoolRuntime, KeyedActorPool, MailboxSink, ManagedActor, - Router, + CleanedUpSessions, DeliveryError, Envelope, KeyedActorPool, MailboxSink, ManagedActor, Router, }; pub(super) use parking_lot::Mutex; pub(super) use std::collections::{BTreeMap, BTreeSet, HashMap}; -pub(super) use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +pub(super) use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; pub(super) use std::sync::{Arc, Weak}; +pub(super) use std::time::{Duration, Instant}; + +pub(super) fn u64_to_usize_saturating(value: u64) -> usize { + usize::try_from(value).unwrap_or(usize::MAX) +} + +pub(super) fn usize_to_u32_saturating(value: usize) -> u32 { + u32::try_from(value).unwrap_or(u32::MAX) +} + +pub(super) fn usize_to_u64_saturating(value: usize) -> u64 { + u64::try_from(value).unwrap_or(u64::MAX) +} pub(super) struct StreamSubscription { pub(super) pattern: crate::runtime::matcher::Pattern, @@ -103,6 +114,23 @@ pub struct StreamStorageWriteOptions { buffered_intent: cntryl_midge::WriteOptions, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StreamSinkInitError(String); + +impl StreamSinkInitError { + pub(super) fn new(message: impl Into) -> Self { + Self(message.into()) + } +} + +impl std::fmt::Display for StreamSinkInitError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.0) + } +} + +impl std::error::Error for StreamSinkInitError {} + impl StreamStorageWriteOptions { #[must_use] pub fn new( @@ -186,13 +214,73 @@ impl AdminStreamReadRequestOwned { } #[derive(Debug, Clone, Eq, Hash, PartialEq)] -pub(super) struct StreamActorKey { - pub(super) family_id: u64, +pub(super) struct StreamResourceScope { + pub(super) family: RouteFamily, pub(super) realm: String, pub(super) area: String, pub(super) resource: String, } +#[derive(Debug, Clone, Eq, Hash, PartialEq)] +pub(super) struct StreamAreaScope { + pub(super) family: RouteFamily, + pub(super) realm: String, + pub(super) area: String, +} + +#[derive(Debug, Clone, Eq, Hash, PartialEq)] +pub(super) struct StreamRealmScope { + pub(super) family: RouteFamily, + pub(super) realm: String, +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub(super) enum StreamWorkKey { + Resource(StreamResourceScope), + Selector(String), + SubscriptionSession(u64), + Notification(String), + UnresolvedSession(u64), +} + +pub(super) struct CommitNotification { + pub(super) family: RouteFamily, + pub(super) route: Route, + pub(super) payload: bytes::Bytes, +} + +pub(super) struct OperationOutcome { + pub(super) response: StreamClientResponseBody, + pub(super) notification: Option, + pub(super) admin_dirty: bool, +} + +impl + From<( + StreamClientResponseBody, + Option<(RouteFamily, Route, bytes::Bytes)>, + bool, + )> for OperationOutcome +{ + fn from( + (response, notification, admin_dirty): ( + StreamClientResponseBody, + Option<(RouteFamily, Route, bytes::Bytes)>, + bool, + ), + ) -> Self { + Self { + response, + notification: notification.map(|(family, route, payload)| CommitNotification { + family, + route, + payload, + }), + admin_dirty, + } + } +} + #[derive(Default)] pub(super) struct StreamRealmSnapshot { pub(super) areas: BTreeSet, @@ -208,7 +296,7 @@ pub(super) struct StreamAreaSnapshot { pub(super) const STREAM_OPERATIONS_TOTAL: &str = "fitz_stream_operations_total"; -impl StreamActorKey { +impl StreamResourceScope { pub(super) fn resource_route(&self) -> Route { Route::new(format!( "stream://{}/{}/{}", @@ -219,7 +307,7 @@ impl StreamActorKey { #[derive(Clone)] pub(super) struct StreamSessionOwner { - pub(super) key: StreamActorKey, + pub(super) key: StreamResourceScope, pub(super) owner_session_id: u64, pub(super) actor: Arc>, } @@ -265,20 +353,24 @@ impl AdminSnapshotState { pub(super) struct WatermarkCoordinators { pub(super) area: Arc< KeyedActorPool< - (u64, String, String), + StreamAreaScope, crate::domains::stream::protocol::StreamCoordinationMessage, >, >, pub(super) realm: Arc< - KeyedActorPool<(u64, String), crate::domains::stream::protocol::StreamCoordinationMessage>, + KeyedActorPool< + StreamRealmScope, + crate::domains::stream::protocol::StreamCoordinationMessage, + >, >, } pub(super) struct StreamDomainCore { pub(super) store: crate::storage::FitzStorageEngine, pub(super) stream_store: Arc, - pub(super) actors: Mutex>>>, + pub(super) actors: Mutex>>>, pub(super) session_owners: Mutex>, + pub(super) cleaned_up_sessions: Mutex, pub(super) subscriptions: SubscriptionRegistry, pub(super) next_session_id: Arc, pub(super) cursor_integrity_key: Arc<[u8; 32]>, @@ -291,12 +383,17 @@ pub(super) struct StreamDomainCore { /// Mutable delivery state itself remains owned by each family core. pub(super) family_cores: Arc>>>, pub(super) watermark_coordinators: WatermarkCoordinators, + /// Measured non-family-actor delivery service time, written by the actor + /// and read by admission to size its window. Unused by family cores - + /// `deliver_to_family` never blocks its caller and so never admits. + pub(super) delivery_service_us: StreamServiceEstimateUs, } pub(super) enum StreamDomainCommand { Deliver( Envelope, crossbeam_channel::Sender>, + Option, ), ReadLiveCounts(crossbeam_channel::Sender), ReadResourceRecords(StreamAdminReadCommand), @@ -322,21 +419,154 @@ pub(super) struct StreamDomainActor { pub(super) core: Arc, } -pub(super) struct StreamDomainRuntime<'a> { - pub(super) core: &'a StreamDomainCore, -} - pub struct StreamDomainSink { pub(super) core: Arc, - pub(super) actor: ManagedActor, - pub(super) family_runtime: Option>, + pub(super) actor: Option>, + pub(super) family_runtime: Option< + crate::runtime::keyed_family_executor::KeyedFamilyExecutor< + StreamWorkKey, + StreamDomainCommand, + Arc, + >, + >, pub(super) family_families: Option>, + /// Client requests currently blocked on the (non-family) actor's reply. + /// Only `deliver_to_actor` admits against this - `deliver_to_family` + /// never blocks its caller, so it needs no admission window. + pub(super) inflight_client_deliveries: Arc, } -impl std::ops::Deref for StreamDomainRuntime<'_> { - type Target = StreamDomainCore; +pub(super) type StreamServiceEstimateUs = Arc; + +/// How long `deliver_to_actor` waits for the (non-family) actor's reply. +pub(super) const STREAM_ACTOR_REPLY_TIMEOUT: Duration = Duration::from_secs(1); + +/// Hard ceiling on concurrent client requests blocked on the actor, whatever +/// the measured service time suggests. +pub(super) const STREAM_ADMISSION_MAX_WINDOW: usize = 64; + +/// Fraction of the reply deadline the admitted backlog may consume, leaving +/// headroom for the delivery itself and a slower-than-average request. +const STREAM_ADMISSION_BUDGET_NUMERATOR: u32 = 4; +const STREAM_ADMISSION_BUDGET_DENOMINATOR: u32 = 5; + +/// Assumed per-delivery service time until the actor has measured one. +const STREAM_ADMISSION_ASSUMED_SERVICE_US: u64 = 5_000; + +/// How many client requests may be blocked on the (non-family) Stream actor. +/// +/// See `queue_admission_window` (`domains::queue::sink::model`) for the full +/// rationale: a fixed window cannot bound the caller's reply deadline, since +/// the actor serves deliveries one at a time and admitting `n` requests +/// commits the tail caller to `n x service_time`. Sizing the window from +/// observed service time keeps the admitted backlog inside the deadline as +/// the actor gets slower, and never drops below 1 so the active operation is +/// always admitted. +pub(super) fn stream_admission_window(service_us: u64) -> usize { + let deadline_us = u64::try_from(STREAM_ACTOR_REPLY_TIMEOUT.as_micros()).unwrap_or(u64::MAX); + let budget_us = deadline_us.saturating_mul(u64::from(STREAM_ADMISSION_BUDGET_NUMERATOR)) + / u64::from(STREAM_ADMISSION_BUDGET_DENOMINATOR); + let service_us = service_us.max(1); + let window = usize::try_from(budget_us / service_us).unwrap_or(STREAM_ADMISSION_MAX_WINDOW); + window.clamp(1, STREAM_ADMISSION_MAX_WINDOW) +} - fn deref(&self) -> &Self::Target { - self.core +/// Blend a fresh delivery duration into the running service estimate. +pub(super) fn blend_stream_service_estimate(previous_us: u64, observed_us: u64) -> u64 { + if previous_us == 0 { + return observed_us.max(1); } + ((previous_us * 3) + observed_us.max(1)) / 4 +} + +/// Admit one client delivery, sizing the window from measured service time and +/// preferring terminal actor failure over a retryable rejection. +/// +/// # Errors +/// +/// `MailboxFull` when the live actor already has as much blocked-caller work +/// as its deadline can serve, or `ActorStopped` when the actor has terminated. +pub(super) fn admit_stream_client_delivery( + inflight: &Arc, + service: &StreamServiceEstimateUs, + actor_running: bool, +) -> Result { + let window = stream_admission_window(service.load(Ordering::Relaxed)); + try_admit_stream_delivery(inflight, window) + .map_err(|error| classify_stream_admission_failure(error, actor_running)) +} + +/// Fold one observed delivery duration into the shared estimate. +pub(super) fn record_stream_service_sample( + estimate: &StreamServiceEstimateUs, + started_at: Instant, +) { + let observed_us = u64::try_from(started_at.elapsed().as_micros()).unwrap_or(u64::MAX); + let previous = estimate.load(Ordering::Relaxed); + estimate.store( + blend_stream_service_estimate(previous, observed_us), + Ordering::Relaxed, + ); +} + +/// A full window means "retry later" only while the actor is alive - a worker +/// that fails closed leaves every admitted slot alive for the sink's +/// lifetime, so admission would keep answering `MailboxFull` forever. +pub(super) fn classify_stream_admission_failure( + error: DeliveryError, + actor_running: bool, +) -> DeliveryError { + if actor_running { + error + } else { + DeliveryError::ActorStopped + } +} + +/// Starting value for the service estimate before anything is measured. +pub(super) const fn stream_assumed_service_us() -> u64 { + STREAM_ADMISSION_ASSUMED_SERVICE_US +} + +/// Holds an admission slot until the queued command is finished with. +/// +/// The slot travels with the command rather than with the blocked caller: a +/// caller that gives up on `recv_timeout` has not cancelled anything, so +/// releasing on caller timeout would recycle slots while the work they +/// admitted is still pending. Dropping with the command covers completion, +/// actor death, and mailbox teardown alike. +#[derive(Debug)] +pub(super) struct StreamAdmissionSlot { + inflight: Arc, +} + +impl Drop for StreamAdmissionSlot { + fn drop(&mut self) { + self.inflight.fetch_sub(1, Ordering::AcqRel); + } +} + +/// Reserve an in-flight slot, or refuse the request. +/// +/// # Errors +/// +/// Returns `MailboxFull` when the actor already has as many blocked callers as +/// its deadline can serve - deliberately the same error an actually-full +/// mailbox produces, so nothing was enqueued and ingress answers with a +/// retryable code. +pub(super) fn try_admit_stream_delivery( + inflight: &Arc, + window: usize, +) -> Result { + inflight + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < window).then_some(current + 1) + }) + .map(|_| StreamAdmissionSlot { + inflight: Arc::clone(inflight), + }) + .map_err(|current| DeliveryError::MailboxFull { + capacity: window, + current_len: current, + }) } diff --git a/src/domains/stream/sink/observability.rs b/src/domains/stream/sink/observability.rs new file mode 100644 index 00000000..1805b4b6 --- /dev/null +++ b/src/domains/stream/sink/observability.rs @@ -0,0 +1,393 @@ +//! Admin read-model projection and metrics glue: when and how live Stream +//! state is mirrored into the admin snapshot and metric gauges. +//! +//! Projection failure must never affect domain correctness. + +use super::model::{ + u64_to_usize_saturating, AdminStreamReadRequest, Arc, BTreeMap, StreamAreaSnapshot, + StreamClientResponseBody, StreamDomainCore, StreamLiveCounts, StreamReadItem, + StreamRealmSnapshot, +}; + +type StreamAdminSnapshotMap = + BTreeMap<(u64, String, String, String), crate::control::admin::StreamInfo>; +type StreamRealmSnapshotMap = BTreeMap; +type StreamAreaSnapshotMap = BTreeMap<(String, String), StreamAreaSnapshot>; + +impl StreamDomainCore { + pub(in crate::domains::stream::sink) fn mark_admin_snapshot_dirty(&self) { + self.admin_snapshot.mark_dirty(); + self.refresh_metrics_gauges(); + } + + pub(in crate::domains::stream::sink) fn refresh_metrics_gauges(&self) { + let counts = self.aggregate_live_counts(); + + if let Some(metrics) = &self.metrics { + metrics.set_stream_count(counts.streams); + metrics.set_subscription_count(counts.subscriptions); + metrics.set_append_session_count(counts.append_sessions); + } else { + crate::observability::gauge_set("fitz_stream_active_gauge", counts.streams as u64); + crate::observability::gauge_set( + "fitz_stream_subscriptions_gauge", + counts.subscriptions as u64, + ); + crate::observability::gauge_set( + "fitz_stream_append_sessions_active", + counts.append_sessions as u64, + ); + } + } + + pub(in crate::domains::stream::sink) fn counter_inc(&self, name: &str) { + if let Some(metrics) = &self.metrics { + metrics.counter_inc(name); + } else { + crate::observability::counter_inc(name); + } + } + + pub(in crate::domains::stream::sink) fn counter_add(&self, name: &str, amount: u64) { + if let Some(metrics) = &self.metrics { + metrics.counter_add(name, amount); + } else { + crate::observability::counter_add(name, amount); + } + } + + pub(in crate::domains::stream::sink) fn stream_response_is_failure( + response: &StreamClientResponseBody, + ) -> bool { + matches!( + response, + StreamClientResponseBody::Error(_) | StreamClientResponseBody::SubscriptionError(_) + ) + } + + pub(in crate::domains::stream::sink) fn refresh_admin_snapshot_if_dirty(&self) { + if self.admin_snapshot.take_dirty() { + self.sync_admin_snapshot(); + } + } + + /// # Errors + /// + /// Returns an error if the requested route cannot be read or if the stream + /// store rejects the read parameters. + pub(in crate::domains::stream::sink) fn admin_read_resource_records( + &self, + request: AdminStreamReadRequest<'_>, + ) -> Result< + ( + Vec, + crate::domains::stream::protocol::ReadCursor, + ), + String, + > { + let filter = + request + .discriminator + .map(|value| crate::domains::stream::protocol::StreamFilterSet { + clauses: vec![ + crate::domains::stream::protocol::StreamFilterClause::Equals(value), + ], + }); + let params = crate::domains::stream::store::ReadResourceParams { + family: request.family.as_u64(), + realm: request.realm, + area: request.area, + resource: request.resource, + from_offset: request.from_offset, + limit: request.limit, + max_bytes: None, + }; + + self.stream_store + .read_resource_with_filter(¶ms, filter.as_ref()) + } + + pub(in crate::domains::stream::sink) fn sync_admin_snapshot(&self) { + if let Err(error) = self.try_sync_admin_snapshot() { + self.admin_snapshot.mark_dirty(); + self.counter_inc( + crate::domains::stream::metrics::METRIC_ADMIN_PROJECTION_FAILURES_TOTAL, + ); + tracing::warn!( + domain = "stream", + error, + "Stream admin projection refresh failed; retaining prior snapshot" + ); + } + } + + fn try_sync_admin_snapshot(&self) -> Result<(), String> { + let (mut streams, realm_snapshots, area_snapshots, committed_events_total) = + self.collect_committed_stream_snapshots()?; + let stream_realm_watermarks = self.collect_stream_realm_watermarks(realm_snapshots)?; + let stream_area_watermarks = self.collect_stream_area_watermarks(area_snapshots)?; + self.overlay_live_actor_snapshots(&mut streams); + self.publish_admin_snapshot( + streams, + stream_realm_watermarks, + stream_area_watermarks, + committed_events_total, + ); + Ok(()) + } + + fn collect_committed_stream_snapshots( + &self, + ) -> Result< + ( + StreamAdminSnapshotMap, + StreamRealmSnapshotMap, + StreamAreaSnapshotMap, + usize, + ), + String, + > { + let mut streams: StreamAdminSnapshotMap = BTreeMap::new(); + let mut realm_snapshots: StreamRealmSnapshotMap = BTreeMap::new(); + let mut area_snapshots: StreamAreaSnapshotMap = BTreeMap::new(); + let mut committed_events_total = 0usize; + + let families = self + .store + .list_column_families() + .map_err(|error| error.to_string())?; + for family in families { + let family_id = u64::from(family.id()); + let records = self.stream_store.list_resource_metadata(family_id)?; + for crate::domains::stream::store::StreamAdminRecord { + realm, + area, + resource, + next_offset, + committed_size_bytes, + } in records + { + committed_events_total = + committed_events_total.saturating_add(u64_to_usize_saturating(next_offset)); + let last_offset = next_offset.saturating_sub(1); + streams.insert( + (family_id, realm.clone(), area.clone(), resource.clone()), + crate::control::admin::StreamInfo::snapshot( + crate::control::admin::StreamInfoSnapshot { + route_family: family_id, + realm: &realm, + area: &area, + resource: &resource, + offset: last_offset, + watermark: last_offset, + size_bytes: committed_size_bytes, + sessions_active: 0, + }, + ), + ); + + let realm_snapshot = realm_snapshots.entry(realm.clone()).or_default(); + realm_snapshot.areas.insert(area.clone()); + realm_snapshot.resource_count = realm_snapshot.resource_count.saturating_add(1); + realm_snapshot.families.insert(family_id); + + let area_snapshot = area_snapshots + .entry((realm.clone(), area.clone())) + .or_default(); + area_snapshot.resource_count = area_snapshot.resource_count.saturating_add(1); + area_snapshot.families.insert(family_id); + } + } + + Ok(( + streams, + realm_snapshots, + area_snapshots, + committed_events_total, + )) + } + + fn collect_stream_realm_watermarks( + &self, + realm_snapshots: StreamRealmSnapshotMap, + ) -> Result, String> { + realm_snapshots + .into_iter() + .map(|(realm, snapshot)| { + let family_watermarks = snapshot + .families + .into_iter() + .map(|family_id| { + self.stream_store + .get_realm_watermark(family_id, &realm) + .map(|watermark| { + crate::control::admin::StreamRealmWatermark::snapshot( + family_id, watermark, + ) + }) + }) + .collect::, _>>()?; + + Ok(crate::control::admin::StreamRealmWatermarkDetail::snapshot( + &realm, + snapshot.areas.len(), + snapshot.resource_count, + family_watermarks, + )) + }) + .collect() + } + + fn collect_stream_area_watermarks( + &self, + area_snapshots: StreamAreaSnapshotMap, + ) -> Result, String> { + area_snapshots + .into_iter() + .map(|((realm, area), snapshot)| { + let family_watermarks = snapshot + .families + .into_iter() + .map(|family_id| { + self.stream_store + .get_watermark(family_id, &realm, &area) + .map(|watermark| { + crate::control::admin::StreamAreaWatermark::snapshot( + family_id, watermark, + ) + }) + }) + .collect::, _>>()?; + + Ok(crate::control::admin::StreamAreaWatermarkDetail::snapshot( + &realm, + &area, + snapshot.resource_count, + family_watermarks, + )) + }) + .collect() + } + + fn overlay_live_actor_snapshots(&self, streams: &mut StreamAdminSnapshotMap) { + let family_cores = self.registered_family_cores(); + if family_cores.is_empty() { + self.overlay_live_actor_snapshots_from(streams); + return; + } + + for family_core in family_cores { + family_core.overlay_live_actor_snapshots_from(streams); + } + } + + fn overlay_live_actor_snapshots_from(&self, streams: &mut StreamAdminSnapshotMap) { + let actors = self.actors.lock(); + for (key, actor) in actors.iter() { + let actor = actor.lock(); + let last_offset = actor + .metadata() + .ok() + .and_then(|response| response.metadata.last_resource_offset); + let sessions_active = usize::from(actor.has_active_session()); + let stream_key = ( + key.family.as_u64(), + key.realm.clone(), + key.area.clone(), + key.resource.clone(), + ); + let committed_snapshot = streams.get(&stream_key); + if committed_snapshot.is_none() && last_offset.is_none() { + continue; + } + let committed_size_bytes = committed_snapshot.map_or(0, |item| item.size_bytes); + let committed_offset = committed_snapshot.map(|item| item.offset); + let visible_offset = last_offset.or(committed_offset).unwrap_or(0); + + streams.insert( + stream_key, + crate::control::admin::StreamInfo::snapshot( + crate::control::admin::StreamInfoSnapshot { + route_family: key.family.as_u64(), + realm: &key.realm, + area: &key.area, + resource: &key.resource, + offset: visible_offset, + watermark: visible_offset, + size_bytes: committed_size_bytes, + sessions_active, + }, + ), + ); + } + } + + fn publish_admin_snapshot( + &self, + streams: StreamAdminSnapshotMap, + stream_realm_watermarks: Vec, + stream_area_watermarks: Vec, + committed_events_total: usize, + ) { + self.admin_snapshot + .read_model + .replace_streams(streams.into_values().collect()); + self.admin_snapshot + .read_model + .replace_stream_realm_watermarks(stream_realm_watermarks); + self.admin_snapshot + .read_model + .replace_stream_area_watermarks(stream_area_watermarks); + self.admin_snapshot + .read_model + .replace_stream_events_total(committed_events_total); + } + + pub(in crate::domains::stream::sink) fn live_counts(&self) -> StreamLiveCounts { + let subscriptions = self + .subscriptions + .families + .lock() + .values() + .map(crate::domains::subscription_state::RoutedSubscriptionSet::subscription_count) + .sum(); + + StreamLiveCounts { + streams: self.actors.lock().len(), + append_sessions: self.session_owners.lock().len(), + subscriptions, + } + } + + fn registered_family_cores(&self) -> Vec> { + let mut family_cores = self.family_cores.lock(); + let mut live = Vec::with_capacity(family_cores.len()); + family_cores.retain(|_, weak| { + if let Some(core) = weak.upgrade() { + live.push(core); + true + } else { + false + } + }); + live + } + + fn aggregate_live_counts(&self) -> StreamLiveCounts { + let family_cores = self.registered_family_cores(); + if family_cores.is_empty() { + return self.live_counts(); + } + + family_cores + .into_iter() + .fold(StreamLiveCounts::default(), |mut total, family_core| { + let counts = family_core.live_counts(); + total.streams = total.streams.saturating_add(counts.streams); + total.append_sessions = + total.append_sessions.saturating_add(counts.append_sessions); + total.subscriptions = total.subscriptions.saturating_add(counts.subscriptions); + total + }) + } +} diff --git a/src/domains/stream/sink/reads.rs b/src/domains/stream/sink/reads.rs new file mode 100644 index 00000000..39b52405 --- /dev/null +++ b/src/domains/stream/sink/reads.rs @@ -0,0 +1,359 @@ +//! Resource/area/realm/global read execution, cursor integrity, and the +//! per-family actor lookup reads are executed against. + +use super::model::{ + route_triplet, Arc, Mutex, Route, RouteFamily, StreamActor, StreamDomainCore, + StreamReadExecution, StreamResourceScope, StreamStorageLayout, +}; +use crate::domains::stream::protocol::ReadResponse; + +mod read_finalization; +mod wire_encoding; + +use read_finalization::apply_global_snapshot_boundary; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum ReadScope { + Resource, + Area, + Realm, + Global, +} + +impl StreamDomainCore { + pub(in crate::domains::stream::sink) fn run_maintenance_slice(&self, family: u64) { + if let Err(error) = self.stream_store.run_maintenance(family) { + tracing::warn!( + domain = "stream", + family, + error, + "Stream maintenance slice failed; queued work will be retried" + ); + } + } + + fn cursor_integrity_token( + &self, + selector_fingerprint: u64, + captured_watermark: u64, + next_offset: u64, + ) -> u64 { + use hmac::{Hmac, KeyInit, Mac}; + + let mut mac = Hmac::::new_from_slice(self.cursor_integrity_key.as_ref()) + .expect("Stream cursor HMAC key has a valid fixed length"); + mac.update(&[1]); + mac.update(&selector_fingerprint.to_le_bytes()); + mac.update(&captured_watermark.to_le_bytes()); + mac.update(&next_offset.to_le_bytes()); + let bytes = mac.finalize().into_bytes(); + u64::from_le_bytes( + bytes[..8] + .try_into() + .expect("HMAC-SHA256 output is 32 bytes"), + ) + } + + pub(in crate::domains::stream::sink) fn storage_layout(&self) -> StreamStorageLayout { + self.stream_store.storage_layout() + } + + pub(in crate::domains::stream::sink) fn actor_key_for_route( + family_id: RouteFamily, + route: &Route, + ) -> Result { + let parts = + route_triplet(route.as_str()).ok_or_else(|| "invalid stream route".to_string())?; + if parts.realm.is_empty() + || parts.area.is_empty() + || parts.resource.is_empty() + || parts.realm.contains('*') + || parts.area.contains('*') + || parts.resource.contains('*') + { + return Err("stream append routes require concrete realm/area/resource".to_string()); + } + if parts.area == crate::domains::stream::INTERNAL_REALM_SEGMENT { + return Err(format!( + "area '{}' is reserved for internal broker use", + crate::domains::stream::INTERNAL_REALM_SEGMENT + )); + } + if parts.resource == crate::domains::stream::INTERNAL_AREA_SEGMENT { + return Err(format!( + "resource '{}' is reserved for internal broker use", + crate::domains::stream::INTERNAL_AREA_SEGMENT + )); + } + Ok(StreamResourceScope { + family: family_id, + realm: parts.realm.to_string(), + area: parts.area.to_string(), + resource: parts.resource.to_string(), + }) + } + + pub(in crate::domains::stream::sink) fn get_or_create_actor( + &self, + key: &StreamResourceScope, + ) -> Result>, String> { + use std::collections::hash_map::Entry; + + let mut actors = self.actors.lock(); + match actors.entry(key.clone()) { + Entry::Occupied(entry) => Ok(entry.get().clone()), + Entry::Vacant(entry) => { + let actor = Arc::new(Mutex::new(StreamActor::new( + key.family, + key.realm.clone(), + key.area.clone(), + key.resource.clone(), + self.stream_store.clone(), + )?)); + entry.insert(actor.clone()); + Ok(actor) + } + } + } + + fn empty_global_read_cursor( + &self, + request: &StreamReadExecution<'_>, + selector_fingerprint: u64, + captured_watermark: u64, + ) -> crate::domains::stream::protocol::ReadCursor { + crate::domains::stream::protocol::ReadCursor { + last_resource_offset: 0, + last_area_offset: None, + last_realm_offset: None, + last_global_offset: None, + has_more: request.from_offset < captured_watermark, + cursor_fingerprint: Some(self.cursor_integrity_token( + selector_fingerprint, + captured_watermark, + request.from_offset, + )), + captured_watermark: Some(captured_watermark), + } + } + + fn execute_read_plan( + &self, + scope: ReadScope, + route_filter_area: Option<&str>, + route_filter_resource: Option<&str>, + request: &StreamReadExecution<'_>, + ) -> Result { + let parts = route_triplet(request.route.as_str()); + let (items, cursor) = match scope { + ReadScope::Realm => { + let parts = parts.ok_or_else(|| "invalid stream route".to_string())?; + if let Some(resource) = route_filter_resource { + self.stream_store.read_realm_resource_posting( + &crate::domains::stream::store::ReadRealmPostingParams { + family: request.family_id.as_u64(), + realm: parts.realm, + resource, + from_offset: request.from_offset, + limit: request.limit, + max_bytes: request.max_bytes, + }, + request.filter, + )? + } else { + self.stream_store.read_realm_with_filter( + request.family_id.as_u64(), + parts.realm, + request.from_offset, + request.limit, + request.max_bytes, + request.filter, + )? + } + } + ReadScope::Area => { + let parts = parts.ok_or_else(|| "invalid stream route".to_string())?; + self.stream_store.read_area_with_filter( + &crate::domains::stream::store::ReadAreaParams { + family: request.family_id.as_u64(), + realm: parts.realm, + area: parts.area, + from_offset: request.from_offset, + limit: request.limit, + max_bytes: request.max_bytes, + }, + request.filter, + )? + } + ReadScope::Resource => { + let key = Self::actor_key_for_route(request.family_id, request.route)?; + let response = self.get_or_create_actor(&key)?.lock().read_with_filter( + request.from_offset, + request.limit, + request.max_bytes, + request.filter, + )?; + (response.items, response.cursor) + } + ReadScope::Global => self.stream_store.read_global_posting( + &crate::domains::stream::store::ReadGlobalPostingParams { + family: request.family_id.as_u64(), + from_offset: request.from_offset, + limit: request.limit, + max_bytes: request.max_bytes, + area: route_filter_area, + resource: route_filter_resource, + }, + request.filter, + )?, + }; + Ok(ReadResponse { items, cursor }) + } + + fn finalize_read_response( + &self, + request: &StreamReadExecution<'_>, + selector_fingerprint: u64, + captured_watermark: u64, + mut response: ReadResponse, + ) -> ReadResponse { + apply_global_snapshot_boundary(request.from_offset, captured_watermark, &mut response); + let next_offset = response + .cursor + .last_global_offset + .map_or(request.from_offset, |offset| offset.saturating_add(1)); + response.cursor.cursor_fingerprint = Some(self.cursor_integrity_token( + selector_fingerprint, + captured_watermark, + next_offset, + )); + response.cursor.captured_watermark = Some(captured_watermark); + response + } + + pub(in crate::domains::stream::sink) fn encode_read_response_data( + &self, + request: StreamReadExecution<'_>, + ) -> Result, String> { + use crate::domains::stream::route_grammar::StreamRouteShape; + + let shape = crate::domains::stream::route_grammar::classify_stream_route_shape( + request.route.as_str(), + )?; + let (scope, area_filter, resource_filter) = match &shape { + StreamRouteShape::Resource { .. } => (ReadScope::Resource, None, None), + StreamRouteShape::Area { .. } => (ReadScope::Area, None, None), + StreamRouteShape::Realm { .. } => (ReadScope::Realm, None, None), + StreamRouteShape::RealmFilterResource { resource, .. } => { + (ReadScope::Realm, None, Some(*resource)) + } + StreamRouteShape::Global => (ReadScope::Global, None, None), + StreamRouteShape::GlobalFilterArea { area } => (ReadScope::Global, Some(*area), None), + StreamRouteShape::GlobalFilterResource { resource } => { + (ReadScope::Global, None, Some(*resource)) + } + StreamRouteShape::GlobalFilterAreaResource { area, resource } => { + (ReadScope::Global, Some(*area), Some(*resource)) + } + }; + if scope != ReadScope::Global { + if request.cursor_fingerprint.is_some() || request.captured_watermark.is_some() { + return Err( + "ERR_CURSOR_UNSUPPORTED: snapshot cursors require a global stream selector" + .to_string(), + ); + } + let response = self.execute_read_plan(scope, area_filter, resource_filter, &request)?; + return Ok(Self::encode_stream_read_data( + &response.items, + &response.cursor, + false, + )); + } + + let selector_fingerprint = crate::domains::stream::route_grammar::cursor_fingerprint( + request.family_id, + &shape, + request.filter, + ); + let current_frontier = self + .stream_store + .get_global_watermark(request.family_id.as_u64())?; + let captured_watermark = request.captured_watermark.unwrap_or(current_frontier); + if captured_watermark > current_frontier { + return Err( + "ERR_CURSOR_WATERMARK_INVALID: captured watermark is ahead of the visible frontier" + .to_string(), + ); + } + let is_continuation = + request.cursor_fingerprint.is_some() || request.captured_watermark.is_some(); + let expected_token = self.cursor_integrity_token( + selector_fingerprint, + captured_watermark, + request.from_offset, + ); + if is_continuation && request.cursor_fingerprint != Some(expected_token) { + return Err( + "ERR_CURSOR_SELECTOR_MISMATCH: cursor was issued for a different stream route \ + family, selector, filter, snapshot, or position" + .to_string(), + ); + } + if request.limit == 0 { + let cursor = + self.empty_global_read_cursor(&request, selector_fingerprint, captured_watermark); + return Ok(Self::encode_stream_read_data(&[], &cursor, true)); + } + let response = self.execute_read_plan(scope, area_filter, resource_filter, &request)?; + let response = self.finalize_read_response( + &request, + selector_fingerprint, + captured_watermark, + response, + ); + Ok(Self::encode_stream_read_data( + &response.items, + &response.cursor, + true, + )) + } + + pub(in crate::domains::stream::sink) fn encode_last_response_data( + &self, + family_id: RouteFamily, + route: &Route, + ) -> Result, String> { + let parts = + route_triplet(route.as_str()).ok_or_else(|| "invalid stream route".to_string())?; + if parts.area == "*" || parts.resource == "*" { + return Ok(Vec::new()); + } + let key = Self::actor_key_for_route(family_id, route)?; + let actor = self.get_or_create_actor(&key)?; + let data = actor + .lock() + .last()? + .record + .as_ref() + .map(Self::encode_stream_last_data) + .unwrap_or_default(); + Ok(data) + } + + pub(in crate::domains::stream::sink) fn encode_metadata_response_data( + &self, + family_id: RouteFamily, + route: &Route, + ) -> Result, String> { + let parts = + route_triplet(route.as_str()).ok_or_else(|| "invalid stream route".to_string())?; + if parts.area == "*" || parts.resource == "*" { + return Ok(Vec::new()); + } + let key = Self::actor_key_for_route(family_id, route)?; + let actor = self.get_or_create_actor(&key)?; + let metadata = actor.lock().metadata()?.metadata; + Ok(Self::encode_stream_metadata_data(&metadata)) + } +} diff --git a/src/domains/stream/sink/domain_sink_impl/domain_core_impl/read_finalization.rs b/src/domains/stream/sink/reads/read_finalization.rs similarity index 100% rename from src/domains/stream/sink/domain_sink_impl/domain_core_impl/read_finalization.rs rename to src/domains/stream/sink/reads/read_finalization.rs diff --git a/src/domains/stream/sink/domain_sink_impl/domain_core_impl/wire_encoding.rs b/src/domains/stream/sink/reads/wire_encoding.rs similarity index 99% rename from src/domains/stream/sink/domain_sink_impl/domain_core_impl/wire_encoding.rs rename to src/domains/stream/sink/reads/wire_encoding.rs index 9c35a164..e6009266 100644 --- a/src/domains/stream/sink/domain_sink_impl/domain_core_impl/wire_encoding.rs +++ b/src/domains/stream/sink/reads/wire_encoding.rs @@ -1,4 +1,4 @@ -use super::{ +use super::super::model::{ usize_to_u32_saturating, usize_to_u64_saturating, PayloadEncoder, StreamClientResponseBody, StreamDomainCore, StreamFilteredReason, StreamMetadata, StreamReadItem, StreamRecord, }; diff --git a/src/domains/stream/sink/tests/sink_dispatch.rs b/src/domains/stream/sink/tests/sink_dispatch.rs index 7b3a42fc..f015e7d4 100644 --- a/src/domains/stream/sink/tests/sink_dispatch.rs +++ b/src/domains/stream/sink/tests/sink_dispatch.rs @@ -889,3 +889,87 @@ fn should_encode_exact_resource_metadata_payload_given_empty_stream() { assert_eq!(metadata.area_watermark, 0); assert_eq!(metadata.realm_watermark, 0); } + +#[test] +fn should_reject_surplus_stream_load_instead_of_accepting_then_timing_out() { + // Arrange + // `deliver_to_actor` blocks its caller's thread on the actor's reply with + // no bound on how many callers can pile up concurrently unless admission + // refuses surplus load up front. Work admitted beyond what the deadline + // can serve would otherwise become an indeterminate outcome. + use crate::domains::stream::sink::model::{stream_admission_window, try_admit_stream_delivery}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let inflight = Arc::new(AtomicUsize::new(0)); + let window = stream_admission_window(super::super::model::stream_assumed_service_us()); + + // Act + let held = (0..window) + .map(|_| try_admit_stream_delivery(&inflight, window).expect("slot within the limit")) + .collect::>(); + let refused = try_admit_stream_delivery(&inflight, window); + + // Assert + assert!( + matches!(refused, Err(DeliveryError::MailboxFull { .. })), + "surplus must be refused as never-enqueued, got {refused:?}" + ); + assert_eq!(inflight.load(Ordering::Acquire), window); + + // Slots are released when the COMMAND is finished with, not when a caller + // stops waiting: a `recv_timeout` cancels nothing, so recycling on caller + // timeout would admit fresh work on top of still-pending mutations. + drop(held); + assert_eq!(inflight.load(Ordering::Acquire), 0); + assert!(try_admit_stream_delivery(&inflight, window).is_ok()); +} + +#[test] +fn should_refuse_stream_client_delivery_once_admission_window_is_exhausted() { + // Arrange + // End-to-end through `StreamDomainSink::deliver`: holding every admission + // slot must make a plain (non-control-plane) delivery fail fast with + // `MailboxFull` rather than blocking on the actor's 1s reply wait. + let context = setup_test_context(); + let window = crate::domains::stream::sink::model::stream_admission_window( + context + .sink + .core + .delivery_service_us + .load(std::sync::atomic::Ordering::Relaxed), + ); + let held = (0..window) + .map(|_| { + crate::domains::stream::sink::model::try_admit_stream_delivery( + &context.sink.inflight_client_deliveries, + window, + ) + .expect("slot") + }) + .collect::>(); + + // Act + let result = context.sink.deliver(Envelope::new( + RouteAddress::new( + context.family, + Route::new("stream://admission-under-pressure"), + ), + Bytes::from_static(b"probe"), + )); + + // Assert + assert!( + matches!(result, Err(DeliveryError::MailboxFull { .. })), + "surplus client load must be refused as never-enqueued, got {result:?}" + ); + + // Slots release once held commands finish, restoring normal admission. + drop(held); + assert_eq!( + context + .sink + .inflight_client_deliveries + .load(std::sync::atomic::Ordering::Acquire), + 0 + ); +} diff --git a/src/domains/stream/store/commits_and_sessions.rs b/src/domains/stream/store/commits_and_sessions.rs index 4be26e87..5cd43658 100644 --- a/src/domains/stream/store/commits_and_sessions.rs +++ b/src/domains/stream/store/commits_and_sessions.rs @@ -270,7 +270,7 @@ impl StreamStore { let sequencing_guard = self.resource_sequence_guard(family, realm, area, resource); // Known scaling limit: strict compact-page ordering keeps this guard - // across the storage commit (and therefore fsync in Sync/CloudStrict). + // across the storage commit (and therefore fsync in Sync mode). // Revisit with group commit if per-resource throughput becomes limiting. let _sequencing_lock = sequencing_guard.lock(); diff --git a/src/domains/stream/store/compact_page_writes.rs b/src/domains/stream/store/compact_page_writes.rs index a3ce22ad..f054c228 100644 --- a/src/domains/stream/store/compact_page_writes.rs +++ b/src/domains/stream/store/compact_page_writes.rs @@ -900,7 +900,6 @@ impl StreamStore { let write_options = match mode { StreamWriteMode::Sync => self.sync_write_options, StreamWriteMode::Buffered => self.buffered_write_options, - StreamWriteMode::CloudStrict => cntryl_midge::WriteOptions::cloud_strict(), }; #[cfg(not(test))] let _ = family; diff --git a/src/domains/stream/store/maintenance.rs b/src/domains/stream/store/maintenance.rs index f05c1e1d..298e24d0 100644 --- a/src/domains/stream/store/maintenance.rs +++ b/src/domains/stream/store/maintenance.rs @@ -526,8 +526,20 @@ impl StreamStore { (deadline > now_epoch_ms) .then(|| deadline.saturating_sub(now_epoch_ms).saturating_add(999) / 1_000) }); + // A fragment's key names the offset of its FIRST record, and only the + // positional planes are guaranteed to tile from the bucket start - a + // posting holds just the offsets belonging to one area or resource, so + // its first entry lands wherever that scope's first commit did (and + // moves again when `prune_expired` drops leading entries). Keying the + // replacement at `bucket_start` regardless would make it disagree with + // its own key, and `validate_merged_posting` would reject the bucket + // on the next merge - failing every later slice and requeueing the + // bucket forever. + let replacement_first_offset = merged + .posting_bounds() + .map_or(bucket.bucket_start, |(first, _)| first); let mut replacement_key = bucket.group_key; - replacement_key.extend_from_slice(&bucket.bucket_start.to_be_bytes()); + replacement_key.extend_from_slice(&replacement_first_offset.to_be_bytes()); replacement_key.extend_from_slice(&bucket.replacement_generation.to_be_bytes()); let mut write_txn = self .db diff --git a/src/domains/stream/store/mod.rs b/src/domains/stream/store/mod.rs index 78930925..10cfa87e 100644 --- a/src/domains/stream/store/mod.rs +++ b/src/domains/stream/store/mod.rs @@ -7,7 +7,8 @@ use std::sync::Arc; use super::protocol::{ IngestMetadata, StreamDiscriminator, StreamFilterSet, StreamFilteredReason, StreamReadItem, - StreamRecord, StreamWriteMode, + StreamRecord, StreamWriteMode, STREAM_READ_ITEM_FIXED_WIRE_OVERHEAD_BYTES, + STREAM_READ_RESPONSE_ENVELOPE_OVERHEAD_BYTES, }; use super::storage::{ decode_area_offset_from_key, decode_realm_offset_from_key, decode_resource_offset_from_key, @@ -64,6 +65,125 @@ type ResourceMetaStateHandle = Arc>; const ERR_SESSION_ROUTE_FAMILY_MISMATCH: &str = "ERR_SESSION_ROUTE_FAMILY_MISMATCH"; +/// Hard ceiling on the bytes a single stream read response may accumulate. +/// +/// Every domain response is framed on the wire as one length-prefixed TLV +/// value with a `u16` length (see `encode_single_tlv_frame` in +/// `api/outbound.rs`), so a response assembled past this size can never be +/// sent — it panics at encode time instead. Clients may optionally request a +/// smaller `max_bytes`, but the ceiling applies unconditionally, since +/// `max_bytes` is optional on the wire and commonly omitted. +pub(crate) const MAX_STREAM_RESPONSE_PAYLOAD_BYTES: usize = u16::MAX as usize; + +/// The largest a read response's summed item bytes may be while still +/// guaranteeing the fully encoded wire frame fits `u16::MAX`. +fn stream_response_byte_ceiling() -> usize { + MAX_STREAM_RESPONSE_PAYLOAD_BYTES.saturating_sub(STREAM_READ_RESPONSE_ENVELOPE_OVERHEAD_BYTES) +} + +/// Resolve a client-requested `max_bytes` against the hard wire ceiling. +pub(super) fn bounded_max_bytes(max_bytes: Option) -> usize { + let ceiling = stream_response_byte_ceiling(); + max_bytes.map_or(ceiling, |requested| requested.min(ceiling)) +} + +/// Conservative worst-case wire bytes for one record encoded as an `Event` +/// item. `route_len` is the record's actual encoded route length in bytes. +pub(super) fn stream_record_wire_bytes( + route_len: usize, + body_len: usize, + metadata_len: usize, +) -> usize { + STREAM_READ_ITEM_FIXED_WIRE_OVERHEAD_BYTES + .saturating_add(route_len) + .saturating_add(body_len) + .saturating_add(metadata_len) +} + +/// Conservative worst-case wire bytes for one record encoded as a `Filtered` +/// marker: the route plus the same generous fixed overhead, with no body or +/// metadata. Charging a filter-excluded record its (far larger) `Event` cost +/// would let a body the client never receives stop - or reject - the read. +pub(super) fn stream_filtered_marker_wire_bytes(route_len: usize) -> usize { + stream_record_wire_bytes(route_len, 0, 0) +} + +/// Wire bytes for the item this record will actually become: the full `Event` +/// encoding when it passes the read filter, a cheap `Filtered` marker when it +/// does not. Charging a filter-excluded record its `Event` cost would let a +/// body the client never receives stop - or fail - the page. +pub(super) fn stream_read_item_wire_bytes( + matches_filter: bool, + route_len: usize, + body_len: usize, + metadata_len: usize, +) -> usize { + if matches_filter { + stream_record_wire_bytes(route_len, body_len, metadata_len) + } else { + stream_filtered_marker_wire_bytes(route_len) + } +} + +/// Byte length of `stream://{realm}/{area}/{resource}` without allocating. +pub(super) fn stream_route_len(realm: &str, area: &str, resource: &str) -> usize { + "stream://".len() + realm.len() + 1 + area.len() + 1 + resource.len() +} + +pub(super) enum WireBudgetDecision { + /// The record fits; include it and continue. + Include, + /// The response is full; stop before this record and paginate. + Stop, +} + +/// Charge one item's wire bytes against a read response's running budget. +/// +/// Shared by every posting-based read loop (realm-resource, global, +/// global-posting) so the "stop once full, but always make progress with a +/// lone oversized-for-`max_bytes` record" policy - and the hard-ceiling +/// handling for a record that can never fit *any* response - lives in one +/// place instead of being copy-pasted per loop. +/// +/// `item_bytes` must be the cost of the item the caller will actually push: +/// `stream_record_wire_bytes` for an `Event`, `stream_filtered_marker_wire_bytes` +/// for a `Filtered` marker. Charging a filter-excluded record its full `Event` +/// cost would fail a read that only ever needed to send a cheap marker. +/// +/// # Errors +/// +/// Returns `Err` when `item_bytes` alone exceeds +/// `stream_response_byte_ceiling()`, meaning no response could ever encode +/// this item even alone. Stream guarantees exact replay of committed history +/// (see `docs/development/domain-boundaries-spec.md`), so an event the client +/// asked for that cannot be sent must surface as an explicit, classifiable +/// error naming its offset - never as a marker that silently drops the +/// committed body from a rebuilt aggregate. +pub(super) fn charge_wire_budget( + offset: u64, + item_bytes: usize, + bytes_read: usize, + byte_limit: usize, +) -> Result { + if bytes_read.saturating_add(item_bytes) > byte_limit { + if bytes_read > 0 { + return Ok(WireBudgetDecision::Stop); + } + // A tight client-requested `max_bytes` still forces this lone item + // through so pagination makes progress, as long as it fits in a wire + // frame at all. Only an item that itself exceeds the hard wire + // ceiling is rejected outright. + if item_bytes > stream_response_byte_ceiling() { + return Err(format!( + "ERR_READ_RESPONSE_TOO_LARGE: record at offset {offset} is {item_bytes} \ + bytes, exceeding the {}-byte read response limit", + stream_response_byte_ceiling() + )); + } + } + Ok(WireBudgetDecision::Include) +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(super) enum StreamStoreError { SessionNotFound, @@ -431,13 +551,6 @@ struct ReadPageState<'a> { has_more: &'a mut bool, } -fn resource_page_record_bytes(page_record: &CompactResourcePageRecord) -> usize { - page_record - .body - .len() - .saturating_add(page_record.metadata.as_ref().map_or(0, Bytes::len)) -} - fn update_resource_cursor( state: &mut ReadCursorState, resource_offset: u64, @@ -448,13 +561,6 @@ fn update_resource_cursor( state.last_realm_offset = Some(page_record.realm_offset); } -fn area_page_record_bytes(page_record: &CompactAreaPageRecord) -> usize { - page_record - .body - .len() - .saturating_add(page_record.metadata.as_ref().map_or(0, Bytes::len)) -} - fn update_area_cursor( state: &mut ReadCursorState, area_offset: u64, @@ -465,13 +571,6 @@ fn update_area_cursor( state.last_realm_offset = None; } -fn realm_page_record_bytes(page_record: &CompactRealmPageRecord) -> usize { - page_record - .body - .len() - .saturating_add(page_record.metadata.as_ref().map_or(0, Bytes::len)) -} - fn update_realm_cursor( state: &mut ReadCursorState, realm_offset: u64, @@ -663,7 +762,9 @@ fn collect_filtered_read_page_items< where I: IntoIterator, FLoadDiscriminator: FnMut(u64, &R) -> Result, String>, - FRecordBytes: FnMut(&R) -> usize, + // Returns the wire cost of the item this record becomes, given whether + // it passed the read filter. + FRecordBytes: FnMut(&R, bool) -> usize, FUpdateCursor: FnMut(&mut ReadCursorState, u64, &R), FFilteredItem: FnMut(u64, &R) -> StreamReadItem, FEventItem: FnMut(u64, R) -> StreamReadItem, @@ -682,38 +783,46 @@ where } } + if state.items.len() == state.limit { + *state.has_more = true; + return Ok(true); + } + let discriminator = if state.filter.is_some() { load_discriminator(offset, &record)? } else { None }; - if !StreamStore::record_matches_filter(state.filter, discriminator.as_deref()) { - if state.items.len() == state.limit { + // Charge the cost of the item this record will actually become: a + // filter-excluded record is only ever a cheap `Filtered` marker, so + // charging it the full `Event` cost would let a body the client never + // receives stop the page - or, past the wire ceiling, fail a read + // that had nothing oversized to deliver in the first place. + let matches_filter = + StreamStore::record_matches_filter(state.filter, discriminator.as_deref()); + let item_bytes = record_bytes(&record, matches_filter); + match charge_wire_budget( + offset, + item_bytes, + *state.total_bytes, + state.max_bytes_limit, + )? { + WireBudgetDecision::Stop => { *state.has_more = true; return Ok(true); } - - update_cursor(state.cursor, offset, &record); - state.items.push(filtered_item(offset, &record)); - continue; - } - - if state.items.len() == state.limit { - *state.has_more = true; - return Ok(true); - } - - let record_bytes = record_bytes(&record); - let next_total_bytes = state.total_bytes.saturating_add(record_bytes); - if next_total_bytes > state.max_bytes_limit && !state.items.is_empty() { - *state.has_more = true; - return Ok(true); + WireBudgetDecision::Include => {} } update_cursor(state.cursor, offset, &record); - *state.total_bytes = next_total_bytes; - state.items.push(event_item(offset, record)); + *state.total_bytes = state.total_bytes.saturating_add(item_bytes); + let item = if matches_filter { + event_item(offset, record) + } else { + filtered_item(offset, &record) + }; + state.items.push(item); } Ok(stop_scan) diff --git a/src/domains/stream/store/ordered_reads.rs b/src/domains/stream/store/ordered_reads.rs index 3147ded4..393145b7 100644 --- a/src/domains/stream/store/ordered_reads.rs +++ b/src/domains/stream/store/ordered_reads.rs @@ -4,11 +4,11 @@ use super::read_support::{ GlobalFragmentCache, }; use super::{ - area_page_record_bytes, collect_filtered_read_page_items, decode_area_offset_from_key, + bounded_max_bytes, collect_filtered_read_page_items, decode_area_offset_from_key, decode_realm_offset_from_key, decode_resource_offset_from_key, encode_compact_area_page_key, encode_compact_resource_page_key, encode_compressed_compact_realm_page_key, - read_limit_to_usize, realm_page_record_bytes, record_is_expired, resource_page_record_bytes, - update_area_cursor, update_realm_cursor, update_resource_cursor, CompactAreaPageValue, + read_limit_to_usize, record_is_expired, stream_read_item_wire_bytes, stream_route_len, + update_area_cursor, update_realm_cursor, update_resource_cursor, Bytes, CompactAreaPageValue, CompactResourcePageValue, CompressedCompactRealmPageValue, ReadAreaParams, ReadCursorState, ReadPageState, ReadResourceParams, StreamFilterSet, StreamFilteredReason, StreamReadItem, StreamRecord, StreamStore, @@ -89,7 +89,7 @@ impl StreamStore { last_realm_offset: None, last_global_offset: None, }; - let max_bytes_limit = params.max_bytes.unwrap_or(usize::MAX); + let max_bytes_limit = bounded_max_bytes(params.max_bytes); let mut has_more = false; let mut previous_fragment_end = None; let now_epoch_ms = self.now_epoch_ms(); @@ -115,7 +115,16 @@ impl StreamStore { for (slot, mut page_record) in page.records.into_iter().enumerate() { let offset = page_slot_offset(page_start, slot); if record_is_expired(page_record.expires_at, now_epoch_ms) { - update_resource_cursor(&mut cursor, offset, &page_record); + // Only records the caller has not already paged past may + // move the cursor. This pre-pass walks the whole fragment, + // which starts at the enclosing 64-record page boundary, so + // without this an expired record BELOW `from_offset` would + // hand back a resume point behind where the caller already + // was - and an idle stream would replay those events on + // every poll. + if offset >= params.from_offset { + update_resource_cursor(&mut cursor, offset, &page_record); + } continue; } resolve_blob_payload(&txn, &mut page_record.body, &mut page_record.metadata)?; @@ -147,7 +156,14 @@ impl StreamStore { ), ) }, - resource_page_record_bytes, + |page_record, matches_filter| { + stream_read_item_wire_bytes( + matches_filter, + route.as_str().len(), + page_record.body.len(), + page_record.metadata.as_ref().map_or(0, Bytes::len), + ) + }, update_resource_cursor, |offset, _page_record| StreamReadItem::Filtered { route: route.clone(), @@ -250,7 +266,7 @@ impl StreamStore { last_realm_offset: None, last_global_offset: None, }; - let max_bytes_limit = params.max_bytes.unwrap_or(usize::MAX); + let max_bytes_limit = bounded_max_bytes(params.max_bytes); let mut has_more = false; let mut previous_fragment_end = None; let mut global_cache = GlobalFragmentCache::new(); @@ -271,7 +287,16 @@ impl StreamStore { for (slot, mut page_record) in page.records.into_iter().enumerate() { let offset = page_slot_offset(page_start, slot); if record_is_expired(page_record.expires_at, now_epoch_ms) { - update_area_cursor(&mut cursor, offset, &page_record); + // Only records the caller has not already paged past may + // move the cursor. This pre-pass walks the whole fragment, + // which starts at the enclosing 64-record page boundary, so + // without this an expired record BELOW `from_offset` would + // hand back a resume point behind where the caller already + // was - and an idle stream would replay those events on + // every poll. + if offset >= params.from_offset { + update_area_cursor(&mut cursor, offset, &page_record); + } continue; } hydrate_area_locator( @@ -308,7 +333,14 @@ impl StreamStore { ), ) }, - area_page_record_bytes, + |page_record, matches_filter| { + stream_read_item_wire_bytes( + matches_filter, + stream_route_len(params.realm, params.area, &page_record.resource), + page_record.body.len(), + page_record.metadata.as_ref().map_or(0, Bytes::len), + ) + }, update_area_cursor, |offset, page_record| StreamReadItem::Filtered { route: stream_route(params.realm, params.area, &page_record.resource), @@ -409,7 +441,7 @@ impl StreamStore { last_realm_offset: Some(from_offset), last_global_offset: None, }; - let max_bytes_limit = max_bytes.unwrap_or(usize::MAX); + let max_bytes_limit = bounded_max_bytes(max_bytes); let mut has_more = false; let mut previous_fragment_end = None; let mut global_cache = GlobalFragmentCache::new(); @@ -430,7 +462,16 @@ impl StreamStore { for (slot, mut page_record) in page.records.into_iter().enumerate() { let offset = page_slot_offset(page_start, slot); if record_is_expired(page_record.expires_at, now_epoch_ms) { - update_realm_cursor(&mut cursor, offset, &page_record); + // Only records the caller has not already paged past may + // move the cursor. This pre-pass walks the whole fragment, + // which starts at the enclosing 64-record page boundary, so + // without this an expired record BELOW `from_offset` would + // hand back a resume point behind where the caller already + // was - and an idle stream would replay those events on + // every poll. + if offset >= from_offset { + update_realm_cursor(&mut cursor, offset, &page_record); + } continue; } hydrate_realm_locator(&txn, realm, &mut page_record, &mut global_cache)?; @@ -460,7 +501,14 @@ impl StreamStore { ), ) }, - realm_page_record_bytes, + |page_record, matches_filter| { + stream_read_item_wire_bytes( + matches_filter, + stream_route_len(realm, &page_record.area, &page_record.resource), + page_record.body.len(), + page_record.metadata.as_ref().map_or(0, Bytes::len), + ) + }, update_realm_cursor, |offset, page_record| StreamReadItem::Filtered { route: crate::runtime::routing::Route::new(format!( diff --git a/src/domains/stream/store/read_support.rs b/src/domains/stream/store/read_support.rs index 7e41e6b9..0f6ec3c0 100644 --- a/src/domains/stream/store/read_support.rs +++ b/src/domains/stream/store/read_support.rs @@ -11,10 +11,6 @@ pub(super) fn page_slot_offset(page_start: u64, slot: usize) -> u64 { page_start.saturating_add(usize_to_u64_saturating(slot)) } -pub(super) fn record_payload_bytes(body: &Bytes, metadata: Option<&Bytes>) -> usize { - body.len().saturating_add(metadata.map_or(0, Bytes::len)) -} - pub(super) fn begin_read_tx( store: &StreamStore, family: u64, diff --git a/src/domains/stream/store/reads.rs b/src/domains/stream/store/reads.rs index 8b510881..62f58b3f 100644 --- a/src/domains/stream/store/reads.rs +++ b/src/domains/stream/store/reads.rs @@ -1,16 +1,17 @@ use super::read_support::{ begin_read_tx, bounded_fragment_rows, bounded_posting_rows, broad_scope_fragment_rows, - hydrate_realm_locator, load_global_locator_record, record_payload_bytes, resolve_blob_payload, + hydrate_realm_locator, load_global_locator_record, resolve_blob_payload, validate_fragment_range, GlobalFragmentCache, }; use super::{ - decode_realm_offset_from_key, encode_compact_global_page_key, - encode_compressed_compact_realm_page_key, encode_global_area_posting_key, - encode_global_area_resource_posting_key, encode_global_resource_posting_key, - encode_realm_resource_posting_key, read_limit_to_usize, record_is_expired, Bytes, + bounded_max_bytes, charge_wire_budget, decode_realm_offset_from_key, + encode_compact_global_page_key, encode_compressed_compact_realm_page_key, + encode_global_area_posting_key, encode_global_area_resource_posting_key, + encode_global_resource_posting_key, encode_realm_resource_posting_key, read_limit_to_usize, + record_is_expired, stream_read_item_wire_bytes, stream_route_len, Bytes, CompactGlobalPageValue, CompressedCompactRealmPageValue, PostingPageValue, ReadGlobalPostingParams, ReadRealmPostingParams, StreamFilterSet, StreamFilteredReason, - StreamReadItem, StreamRecord, StreamStore, GLOBAL_PAGE_RECORD_LIMIT, + StreamReadItem, StreamRecord, StreamStore, WireBudgetDecision, GLOBAL_PAGE_RECORD_LIMIT, }; use crate::domains::stream::protocol::ReadCursor; @@ -32,18 +33,50 @@ enum PostingScope { Global, } -fn posting_cursor(scope: PostingScope, offset: u64, watermark: u64, has_more: bool) -> ReadCursor { +/// `covered_through` is `None` when the page covered no offset at all, which +/// is distinct from covering offset 0 - the caller must resume where it asked +/// rather than one past it. +fn posting_cursor( + scope: PostingScope, + covered_through: Option, + watermark: u64, + has_more: bool, +) -> ReadCursor { ReadCursor { last_resource_offset: 0, last_area_offset: None, - last_realm_offset: (scope == PostingScope::Realm).then_some(offset), - last_global_offset: (scope == PostingScope::Global).then_some(offset), + last_realm_offset: (scope == PostingScope::Realm) + .then_some(covered_through) + .flatten(), + last_global_offset: (scope == PostingScope::Global) + .then_some(covered_through) + .flatten(), has_more, cursor_fingerprint: None, captured_watermark: Some(watermark), } } +fn global_posting_cursor( + last_examined: u64, + watermark: u64, + has_more: bool, + fragments_exhausted: bool, +) -> ReadCursor { + let has_more = has_more || fragments_exhausted; + let covered_through = if has_more { + last_examined + } else { + last_examined.max(watermark.saturating_sub(1)) + }; + posting_cursor( + PostingScope::Global, + Some(covered_through), + watermark, + has_more, + ) +} + fn realm_posting_record( txn: &cntryl_midge::Transaction, realm: &str, @@ -156,7 +189,7 @@ impl StreamStore { max_bytes, } = *params; self.ensure_layout_activation_for_family(family)?; - let watermark = self.get_realm_watermark(family, realm)?; + let visible_end = self.realm_visible_end(family, realm)?; let posting_page_start = from_offset / super::REALM_PAGE_RECORD_LIMIT as u64 * super::REALM_PAGE_RECORD_LIMIT as u64; let start_key = encode_realm_resource_posting_key(realm, resource, posting_page_start); @@ -166,10 +199,10 @@ impl StreamStore { let (rows, fragments_exhausted) = bounded_posting_rows(&txn, start_key, prefix, "realm-resource")?; let item_limit = read_limit_to_usize(limit); - let byte_limit = max_bytes.unwrap_or(usize::MAX); + let byte_limit = bounded_max_bytes(max_bytes); let mut items = Vec::with_capacity(item_limit.min(1_000)); let mut bytes_read = 0usize; - let mut last_examined = from_offset; + let mut covered_through: Option = None; let mut has_more = false; let mut examined = 0usize; let mut cached_parent = None; @@ -178,7 +211,7 @@ impl StreamStore { 'pages: for (_, value) in rows { for entry in PostingPageValue::try_decode(&value)?.entries { let offset = entry.offset; - if offset < from_offset || offset > watermark { + if offset < from_offset || offset >= visible_end { continue; } if items.len() >= item_limit { @@ -191,7 +224,7 @@ impl StreamStore { } examined += 1; if record_is_expired(entry.expires_at, now_epoch_ms) { - last_examined = offset; + covered_through = Some(offset); continue; } let Some(record) = realm_posting_record( @@ -202,7 +235,7 @@ impl StreamStore { &mut global_cache, )? else { - last_examined = offset; + covered_through = Some(offset); continue; }; let route = realm_posting_route(realm, &record); @@ -210,8 +243,26 @@ impl StreamStore { &txn, &crate::domains::stream::storage::encode_realm_discriminator_key(realm, offset), )?; - if !Self::record_matches_filter(filter, discriminator.as_deref()) { - last_examined = offset; + // Charge the cost of the item this record actually becomes: + // a filter-excluded record is only ever a cheap `Filtered` + // marker, never its full Event encoding. + let matches_filter = Self::record_matches_filter(filter, discriminator.as_deref()); + let record_bytes = stream_read_item_wire_bytes( + matches_filter, + route.as_str().len(), + record.body.len(), + record.metadata.as_ref().map_or(0, Bytes::len), + ); + match charge_wire_budget(offset, record_bytes, bytes_read, byte_limit)? { + WireBudgetDecision::Stop => { + has_more = true; + break 'pages; + } + WireBudgetDecision::Include => {} + } + covered_through = Some(offset); + bytes_read = bytes_read.saturating_add(record_bytes); + if !matches_filter { items.push(StreamReadItem::Filtered { route, offset, @@ -219,13 +270,6 @@ impl StreamStore { }); continue; } - let record_bytes = record_payload_bytes(&record.body, record.metadata.as_ref()); - if bytes_read.saturating_add(record_bytes) > byte_limit && !items.is_empty() { - has_more = true; - break 'pages; - } - last_examined = offset; - bytes_read = bytes_read.saturating_add(record_bytes); items.push(StreamReadItem::Event(StreamRecord { route, resource_offset: record.resource_offset, @@ -241,16 +285,22 @@ impl StreamStore { if fragments_exhausted { has_more = true; } else if !has_more { - last_examined = last_examined.max(watermark); + // Caught up: claim the whole visible frontier so the caller skips + // the realm offsets that belong to other resources instead of + // re-requesting them. `checked_sub` is what keeps that claim + // honest on an empty realm - there is no offset 0 to have covered + // yet, and naming one would make the caller resume at 1 and miss + // the realm's first event for good. + let frontier = if from_offset < visible_end { + visible_end.checked_sub(1) + } else { + from_offset.checked_sub(1) + }; + covered_through = covered_through.max(frontier); } Ok(( items, - posting_cursor( - PostingScope::Realm, - last_examined, - watermark.saturating_add(1), - has_more, - ), + posting_cursor(PostingScope::Realm, covered_through, visible_end, has_more), )) } @@ -281,7 +331,7 @@ impl StreamStore { let txn = begin_read_tx(self, family, "global posting")?; let (rows, fragments_exhausted) = bounded_posting_rows(&txn, start_key, prefix, "global")?; let item_limit = read_limit_to_usize(limit); - let byte_limit = max_bytes.unwrap_or(usize::MAX); + let byte_limit = bounded_max_bytes(max_bytes); let mut items = Vec::with_capacity(item_limit.min(1_000)); let mut bytes_read = 0usize; let mut last_examined = from_offset; @@ -320,8 +370,23 @@ impl StreamStore { &txn, &super::encode_global_discriminator_key(offset), )?; - if !Self::record_matches_filter(filter, discriminator.as_deref()) { - last_examined = offset; + let matches_filter = Self::record_matches_filter(filter, discriminator.as_deref()); + let record_bytes = stream_read_item_wire_bytes( + matches_filter, + route.as_str().len(), + record.body.len(), + record.metadata.as_ref().map_or(0, Bytes::len), + ); + match charge_wire_budget(offset, record_bytes, bytes_read, byte_limit)? { + WireBudgetDecision::Stop => { + has_more = true; + break 'pages; + } + WireBudgetDecision::Include => {} + } + last_examined = offset; + bytes_read = bytes_read.saturating_add(record_bytes); + if !matches_filter { items.push(StreamReadItem::Filtered { route, offset, @@ -329,13 +394,6 @@ impl StreamStore { }); continue; } - let record_bytes = record_payload_bytes(&record.body, record.metadata.as_ref()); - if bytes_read.saturating_add(record_bytes) > byte_limit && !items.is_empty() { - has_more = true; - break 'pages; - } - last_examined = offset; - bytes_read = bytes_read.saturating_add(record_bytes); items.push(StreamReadItem::Event(StreamRecord { route, resource_offset: record.resource_offset, @@ -348,15 +406,8 @@ impl StreamStore { })); } } - if fragments_exhausted { - has_more = true; - } else if !has_more { - last_examined = last_examined.max(watermark.saturating_sub(1)); - } - Ok(( - items, - posting_cursor(PostingScope::Global, last_examined, watermark, has_more), - )) + let cursor = global_posting_cursor(last_examined, watermark, has_more, fragments_exhausted); + Ok((items, cursor)) } /// Reads a family-global snapshot in global-offset order. @@ -388,7 +439,7 @@ impl StreamStore { broad_scope_fragment_rows(from_offset, limit), )?; let item_limit = read_limit_to_usize(limit); - let byte_limit = max_bytes.unwrap_or(usize::MAX); + let byte_limit = bounded_max_bytes(max_bytes); let mut items = Vec::with_capacity(item_limit.min(1_000)); let mut bytes_read = 0usize; let mut last_examined = from_offset; @@ -417,10 +468,6 @@ impl StreamStore { continue; } resolve_blob_payload(&txn, &mut record.body, &mut record.metadata)?; - let record_bytes = record - .body - .len() - .saturating_add(record.metadata.as_ref().map_or(0, Bytes::len)); if items.len() >= item_limit { has_more = true; break 'pages; @@ -433,8 +480,23 @@ impl StreamStore { &txn, &super::encode_global_discriminator_key(offset), )?; - if !Self::record_matches_filter(filter, discriminator.as_deref()) { - last_examined = offset; + let matches_filter = Self::record_matches_filter(filter, discriminator.as_deref()); + let record_bytes = stream_read_item_wire_bytes( + matches_filter, + stream_route_len(&record.realm, &record.area, &record.resource), + record.body.len(), + record.metadata.as_ref().map_or(0, Bytes::len), + ); + match charge_wire_budget(offset, record_bytes, bytes_read, byte_limit)? { + WireBudgetDecision::Stop => { + has_more = true; + break 'pages; + } + WireBudgetDecision::Include => {} + } + last_examined = offset; + bytes_read = bytes_read.saturating_add(record_bytes); + if !matches_filter { items.push(StreamReadItem::Filtered { route, offset, @@ -442,12 +504,6 @@ impl StreamStore { }); continue; } - if bytes_read.saturating_add(record_bytes) > byte_limit && !items.is_empty() { - has_more = true; - break 'pages; - } - last_examined = offset; - bytes_read = bytes_read.saturating_add(record_bytes); items.push(StreamReadItem::Event(StreamRecord { route, resource_offset: record.resource_offset, diff --git a/src/domains/stream/store/tests.rs b/src/domains/stream/store/tests.rs index 3db9c379..46d20dfb 100644 --- a/src/domains/stream/store/tests.rs +++ b/src/domains/stream/store/tests.rs @@ -1,4 +1,31 @@ use super::*; +use std::sync::atomic::{AtomicU64, Ordering}; + +struct TestStreamClock { + epoch_ms: AtomicU64, +} + +impl TestStreamClock { + fn new(epoch_ms: u64) -> Self { + Self { + epoch_ms: AtomicU64::new(epoch_ms), + } + } + + fn set(&self, epoch_ms: u64) { + self.epoch_ms.store(epoch_ms, Ordering::Release); + } +} + +impl crate::runtime::clock::Clock for TestStreamClock { + fn now_instant(&self) -> std::time::Instant { + std::time::Instant::now() + } + + fn now_epoch_ms(&self) -> u64 { + self.epoch_ms.load(Ordering::Acquire) + } +} mod sessions_layout_and_watermarks; use sessions_layout_and_watermarks::*; @@ -9,3 +36,4 @@ mod maintenance_and_payloads; mod model_based; mod offsets_and_reads; mod overflow_and_recovery; +mod ttl_cursor_regressions; diff --git a/src/domains/stream/store/tests/filters_ttl_and_metadata.rs b/src/domains/stream/store/tests/filters_ttl_and_metadata.rs index 089b7821..8e663c49 100644 --- a/src/domains/stream/store/tests/filters_ttl_and_metadata.rs +++ b/src/domains/stream/store/tests/filters_ttl_and_metadata.rs @@ -1,34 +1,4 @@ use super::*; -use std::sync::{ - atomic::{AtomicU64, Ordering}, - Arc, -}; - -struct TestStreamClock { - epoch_ms: AtomicU64, -} - -impl TestStreamClock { - fn new(epoch_ms: u64) -> Self { - Self { - epoch_ms: AtomicU64::new(epoch_ms), - } - } - - fn set(&self, epoch_ms: u64) { - self.epoch_ms.store(epoch_ms, Ordering::Release); - } -} - -impl crate::runtime::clock::Clock for TestStreamClock { - fn now_instant(&self) -> std::time::Instant { - std::time::Instant::now() - } - - fn now_epoch_ms(&self) -> u64 { - self.epoch_ms.load(Ordering::Acquire) - } -} #[test] fn should_compact_zero_ttl_fragments_without_positional_gaps() { @@ -924,3 +894,62 @@ fn should_return_error_given_malformed_compact_realm_page_when_reading_realm() { let error = result.expect_err("malformed compact realm page should fail read"); assert!(error.contains("ERR_INVALID_COMPACT_REALM_PAGE")); } + +#[test] +fn should_emit_filtered_marker_for_oversized_record_excluded_by_filter() { + // Arrange + // An event too large to encode as an Event item, but whose + // discriminator excludes it from the filter - so it would only ever be + // sent as a cheap `Filtered` marker, never as an Event. + let store = StreamStore::new(create_test_engine_with_cfs(vec![1])); + let events = vec![ + EventPayload { + body: Bytes::from(vec![b'a'; MAX_STREAM_RESPONSE_PAYLOAD_BYTES + 1_000]), + metadata: None, + discriminator: Some(StreamDiscriminator::from("beta.created")), + }, + EventPayload { + body: Bytes::from_static(b"alpha"), + metadata: None, + discriminator: Some(StreamDiscriminator::from("alpha.created")), + }, + ]; + store + .commit_records(CommitRecordsParams { + family: 1, + realm: "test", + area: "events", + resource: "oversized-filtered", + expected_resource_next_offset: 0, + events: &events, + ingest_metadata: None, + mode: StreamWriteMode::Buffered, + }) + .expect("commit oversized filtered record"); + let filter = StreamFilterSet { + clauses: vec![StreamFilterClause::StartsWith("alpha".to_string())], + }; + + // Act + let (items, _cursor) = store + .read_resource_with_filter( + &ReadResourceParams { + family: 1, + realm: "test", + area: "events", + resource: "oversized-filtered", + from_offset: 0, + limit: 10, + max_bytes: None, + }, + Some(&filter), + ) + .expect("filter-excluded oversized record must not fail the read"); + + // Assert + // The oversized record costs only a Filtered marker, so the read + // succeeds and the matching record after it is still delivered. + let records = event_records(items); + assert_eq!(records.len(), 1); + assert_eq!(records[0].body, Bytes::from_static(b"alpha")); +} diff --git a/src/domains/stream/store/tests/global_ordering.rs b/src/domains/stream/store/tests/global_ordering.rs index f543e1f6..88754071 100644 --- a/src/domains/stream/store/tests/global_ordering.rs +++ b/src/domains/stream/store/tests/global_ordering.rs @@ -801,6 +801,11 @@ fn should_not_regress_global_posting_cursor_when_read_starts_past_watermark() { #[test] fn should_not_regress_realm_posting_cursor_when_read_starts_past_watermark() { // Arrange + // This path is sparse - it deliberately steps over the realm offsets + // owned by other resources - so an empty page cannot mean "stay put" and + // the caller always resumes at `last_realm_offset + 1`. "Covered nothing" + // therefore has to encode as one BEHIND the requested offset; naming the + // requested offset itself would resume at 31 and skip 30 once it commits. let store = StreamStore::new(create_test_engine_with_cfs(vec![1])); store .commit_records(CommitRecordsParams { @@ -832,7 +837,11 @@ fn should_not_regress_realm_posting_cursor_when_read_starts_past_watermark() { // Assert assert!(items.is_empty()); - assert_eq!(cursor.last_realm_offset, Some(30)); + assert_eq!( + cursor.last_realm_offset, + Some(29), + "an uncovered read must resume exactly where it asked, not past it" + ); assert!(!cursor.has_more); } diff --git a/src/domains/stream/store/tests/maintenance_and_payloads.rs b/src/domains/stream/store/tests/maintenance_and_payloads.rs index 852ed8c2..66324938 100644 --- a/src/domains/stream/store/tests/maintenance_and_payloads.rs +++ b/src/domains/stream/store/tests/maintenance_and_payloads.rs @@ -257,3 +257,61 @@ fn should_roundtrip_every_reserved_blob_marker_prefix_length_with_metadata() { ); } } + +fn commit_single_event(store: &StreamStore, realm: &str, area: &str, resource: &str, next: u64) { + store + .commit_records(CommitRecordsParams { + family: 1, + realm, + area, + resource, + expected_resource_next_offset: next, + events: &[EventPayload { + body: Bytes::from_static(b"x"), + metadata: None, + discriminator: None, + }], + ingest_metadata: None, + mode: StreamWriteMode::Buffered, + }) + .expect("commit single event"); +} + +#[test] +fn should_recompact_a_posting_bucket_whose_first_entry_is_not_the_bucket_start() { + // Arrange + // A posting fragment is keyed by its FIRST ENTRY's offset, not by the + // 64-offset bucket start, and a bucket's first posting only lands on a + // bucket boundary by coincidence. Writing the compacted replacement under + // `bucket_start` therefore makes the replacement disagree with its own + // key, and `validate_merged_posting` rejects it the next time the bucket + // is merged - wedging maintenance for the whole family, since + // `run_maintenance` aborts the slice and requeues the bucket forever. + // + // The leading commit to a different area pushes the `events` area posting + // to start at global offset 1 rather than 0. + let store = StreamStore::new(create_test_engine_with_cfs(vec![1])); + commit_single_event(&store, "north", "other", "misc", 0); + for offset in 0..9 { + commit_single_event(&store, "north", "events", "orders", offset); + } + store + .run_maintenance(1) + .expect("compact the posting bucket a first time"); + for offset in 9..17 { + commit_single_event(&store, "north", "events", "orders", offset); + } + + // Act + let second = store.run_maintenance(1); + + // Assert + assert!( + second.is_ok(), + "re-compacting a posting bucket must not wedge maintenance: {second:?}" + ); + assert!( + store.run_maintenance(1).is_ok(), + "maintenance must stay healthy across repeated slices" + ); +} diff --git a/src/domains/stream/store/tests/offsets_and_reads.rs b/src/domains/stream/store/tests/offsets_and_reads.rs index f41652b2..216e5fb9 100644 --- a/src/domains/stream/store/tests/offsets_and_reads.rs +++ b/src/domains/stream/store/tests/offsets_and_reads.rs @@ -883,3 +883,199 @@ fn should_page_filtered_realm_read_through_filtered_items() { assert_eq!(second_records[0].realm_offset, Some(2)); assert!(!second_cursor.has_more); } + +#[test] +fn should_bound_resource_read_response_to_wire_frame_limit_when_max_bytes_omitted() { + // Arrange: every response is framed as a single u16-length-prefixed TLV + // value on the wire (see `encode_single_tlv_frame`), so a read response + // built past `MAX_STREAM_RESPONSE_PAYLOAD_BYTES` can never actually be + // sent. A client omitting `max_bytes` (legal per the wire spec) must + // still get a response the broker can encode, not an unbounded one. + let store = StreamStore::new(create_test_engine_with_cfs(vec![1])); + let record_body_len = 2_000usize; + let record_count = 60usize; // 60 * 2_000 = 120_000 bytes, well over u16::MAX (65_535) + let events: Vec = (0..record_count) + .map(|_| EventPayload { + body: Bytes::from(vec![b'a'; record_body_len]), + metadata: None, + discriminator: None, + }) + .collect(); + store + .commit_records(CommitRecordsParams { + family: 1, + realm: "test", + area: "events", + resource: "oversized-batch", + expected_resource_next_offset: 0, + events: &events, + ingest_metadata: None, + mode: StreamWriteMode::Buffered, + }) + .expect("seed oversized resource batch"); + + // Act: request every record in one page, with no client-supplied max_bytes. + let (items, cursor) = store + .read_resource(&ReadResourceParams { + family: 1, + realm: "test", + area: "events", + resource: "oversized-batch", + from_offset: 0, + limit: record_count as u64, + max_bytes: None, + }) + .expect("read oversized resource batch"); + + // Assert: the response must be bounded well under the batch's true size + // (120_000 bytes) and under the wire ceiling, and must report has_more + // instead of silently truncating. + let returned_bytes: usize = event_records(items.clone()) + .iter() + .map(|record| record.body.len()) + .sum(); + assert!( + returned_bytes <= MAX_STREAM_RESPONSE_PAYLOAD_BYTES, + "response body bytes {returned_bytes} exceeded the wire frame ceiling \ + {MAX_STREAM_RESPONSE_PAYLOAD_BYTES}" + ); + assert!( + items.len() < record_count, + "expected the response to stop before including every record" + ); + assert!(cursor.has_more, "cursor should signal more records remain"); +} + +#[test] +fn should_reject_read_when_lone_record_alone_exceeds_wire_frame_limit() { + // Arrange: recovery, migration, or a direct low-level store write may + // expose a legacy record larger than today's append limit. The read + // accumulator must reject it explicitly instead of building an + // unencodable response. + // + // Stream guarantees exact replay of committed history, so this must stay + // a loud, classifiable failure naming the offending offset. Emitting a + // filtered marker and advancing instead would silently drop a committed + // event from any aggregate the client rebuilds from this stream. + let store = StreamStore::new(create_test_engine_with_cfs(vec![1])); + let oversized_body_len = MAX_STREAM_RESPONSE_PAYLOAD_BYTES + 1_000; + let events = vec![EventPayload { + body: Bytes::from(vec![b'a'; oversized_body_len]), + metadata: None, + discriminator: None, + }]; + store + .commit_records(CommitRecordsParams { + family: 1, + realm: "test", + area: "events", + resource: "lone-oversized", + expected_resource_next_offset: 0, + events: &events, + ingest_metadata: None, + mode: StreamWriteMode::Buffered, + }) + .expect("seed lone oversized record"); + + // Act + let result = store.read_resource(&ReadResourceParams { + family: 1, + realm: "test", + area: "events", + resource: "lone-oversized", + from_offset: 0, + limit: 10, + max_bytes: None, + }); + + // Assert: an explicit, classifiable error - never a response that would + // panic the TLV encoder, and never a silent skip. + let error = result.expect_err("read of an unencodable lone record must fail explicitly"); + assert!( + error.contains("ERR_READ_RESPONSE_TOO_LARGE"), + "unexpected error: {error}" + ); +} + +#[test] +fn should_not_claim_coverage_of_an_empty_realm_in_the_posting_cursor() { + // Arrange + // A caught-up realm-resource posting read reports the visible frontier as + // covered so a client can skip realm offsets belonging to other + // resources. The realm watermark is inclusive and floors at zero, so an + // EMPTY realm is indistinguishable from one holding a single record at + // offset 0 - and claiming coverage there makes the client resume at 1 and + // miss the realm's very first event forever. + let store = StreamStore::new(create_test_engine_with_cfs(vec![1])); + let params = ReadRealmPostingParams { + family: 1, + realm: "north", + resource: "created", + from_offset: 0, + limit: 64, + max_bytes: None, + }; + + // Act + let (items, cursor) = store + .read_realm_resource_posting(¶ms, None) + .expect("read the posting of an empty realm"); + + // Assert + assert!(items.is_empty()); + assert_eq!( + cursor.last_realm_offset, None, + "an empty realm covers no offset, so the cursor must not name one" + ); + assert_eq!(cursor.captured_watermark, Some(0)); +} + +#[test] +fn should_report_the_visible_frontier_once_a_realm_posting_is_caught_up() { + // Arrange + // The companion to the empty-realm case: with records present, a + // caught-up read must still advance the cursor to the visible frontier, + // including across realm offsets owned by other resources. + let store = StreamStore::new(create_test_engine_with_cfs(vec![1])); + for (area, resource, offset) in [ + ("orders", "created", 0), + ("orders", "shipped", 0), + ("orders", "created", 1), + ] { + store + .commit_records(CommitRecordsParams { + family: 1, + realm: "north", + area, + resource, + expected_resource_next_offset: offset, + events: &single_event(b"seed"), + ingest_metadata: None, + mode: StreamWriteMode::Sync, + }) + .expect("seed realm record"); + } + let params = ReadRealmPostingParams { + family: 1, + realm: "north", + resource: "created", + from_offset: 0, + limit: 64, + max_bytes: None, + }; + + // Act + let (items, cursor) = store + .read_realm_resource_posting(¶ms, None) + .expect("read the caught-up realm posting"); + + // Assert + assert_eq!(event_records(items).len(), 2); + assert_eq!( + cursor.last_realm_offset, + Some(2), + "a caught-up read covers the whole visible frontier" + ); + assert_eq!(cursor.captured_watermark, Some(3)); + assert!(!cursor.has_more); +} diff --git a/src/domains/stream/store/tests/ttl_cursor_regressions.rs b/src/domains/stream/store/tests/ttl_cursor_regressions.rs new file mode 100644 index 00000000..133262af --- /dev/null +++ b/src/domains/stream/store/tests/ttl_cursor_regressions.rs @@ -0,0 +1,138 @@ +use super::*; + +fn commit_ttl_event(store: &StreamStore, expected_next_offset: u64) { + store + .commit_records(CommitRecordsParams { + family: 1, + realm: "north", + area: "orders", + resource: "created", + expected_resource_next_offset: expected_next_offset, + events: &single_event(b"ttl"), + ingest_metadata: None, + mode: StreamWriteMode::Sync, + }) + .expect("commit TTL event"); +} + +#[test] +fn should_not_regress_area_read_cursor_past_expired_records_before_from_offset() { + // Arrange + // The per-fragment expiry pre-pass walks the whole 64-record page, + // including records BELOW `from_offset`. Advancing the cursor for those + // hands a tailing client a resume point behind where it already was, so + // it re-reads events it has consumed - forever, once the stream is idle. + let db = create_test_engine_with_cfs(vec![1]); + let clock = Arc::new(TestStreamClock::new(1_000)); + let store = StreamStore::with_config(db, BatchLimits::default(), StreamTTL::with_seconds(10)) + .with_clock_for_tests(clock.clone()); + for offset in 0..3 { + commit_ttl_event(&store, offset); + } + clock.set(5_000); + for offset in 3..5 { + commit_ttl_event(&store, offset); + } + // Offsets 0..2 have expired; 3 and 4 are still live. + clock.set(12_000); + let (first_items, first_cursor) = store + .read_area(1, "north", "orders", 0, 64, None) + .expect("read the live area page"); + let resume_from = first_cursor + .last_area_offset + .expect("area cursor") + .saturating_add(1); + + // Act + let (tail_items, tail_cursor) = store + .read_area(1, "north", "orders", resume_from, 64, None) + .expect("tail the area from the resume point"); + + // Assert + assert_eq!(event_records(first_items).len(), 2); + assert!(tail_items.is_empty()); + assert_eq!( + tail_cursor.last_area_offset, + Some(resume_from), + "an empty page must leave the cursor at the requested offset, not behind it" + ); +} + +#[test] +fn should_not_regress_realm_read_cursor_past_expired_records_before_from_offset() { + // Arrange + // Same defect as the area plane: `read_realm_with_filter` runs the same + // expiry pre-pass over records the caller already paged past. + let db = create_test_engine_with_cfs(vec![1]); + let clock = Arc::new(TestStreamClock::new(1_000)); + let store = StreamStore::with_config(db, BatchLimits::default(), StreamTTL::with_seconds(10)) + .with_clock_for_tests(clock.clone()); + for offset in 0..3 { + commit_ttl_event(&store, offset); + } + clock.set(5_000); + for offset in 3..5 { + commit_ttl_event(&store, offset); + } + clock.set(12_000); + let (_, first_cursor) = store + .read_realm(1, "north", 0, 64, None) + .expect("read the live realm page"); + let resume_from = first_cursor + .last_realm_offset + .expect("realm cursor") + .saturating_add(1); + + // Act + let (tail_items, tail_cursor) = store + .read_realm(1, "north", resume_from, 64, None) + .expect("tail the realm from the resume point"); + + // Assert + assert!(tail_items.is_empty()); + assert_eq!( + tail_cursor.last_realm_offset, + Some(resume_from), + "an empty page must leave the cursor at the requested offset, not behind it" + ); +} + +#[test] +fn should_not_regress_resource_read_cursor_past_expired_records_before_from_offset() { + // Arrange + // `StreamActor::read_with_filter` short-circuits reads at or past its live + // next-offset, but the admin surface calls `read_resource_with_filter` + // straight through with a caller-supplied offset. + let db = create_test_engine_with_cfs(vec![1]); + let clock = Arc::new(TestStreamClock::new(1_000)); + let store = StreamStore::with_config(db, BatchLimits::default(), StreamTTL::with_seconds(10)) + .with_clock_for_tests(clock.clone()); + for offset in 0..3 { + commit_ttl_event(&store, offset); + } + clock.set(5_000); + for offset in 3..5 { + commit_ttl_event(&store, offset); + } + clock.set(12_000); + + // Act + let (items, cursor) = store + .read_resource(&ReadResourceParams { + family: 1, + realm: "north", + area: "orders", + resource: "created", + from_offset: 5, + limit: 64, + max_bytes: None, + }) + .expect("read the resource past its last committed offset"); + + // Assert + assert!(items.is_empty()); + assert_eq!( + cursor.last_resource_offset, 5, + "an empty page must not move the cursor behind the requested offset" + ); +} diff --git a/src/domains/stream/store/watermarks_and_metadata.rs b/src/domains/stream/store/watermarks_and_metadata.rs index e3372694..0bba9dd8 100644 --- a/src/domains/stream/store/watermarks_and_metadata.rs +++ b/src/domains/stream/store/watermarks_and_metadata.rs @@ -277,13 +277,31 @@ impl StreamStore { .map_err(|e| format!("midge commit error: {e:?}")) } - /// Get the current realm watermark. + /// Get the current realm watermark, as the highest visible realm offset. /// /// # Errors /// /// Returns an error if layout activation, storage reads, counter decoding, /// or fallback offset scanning fails. pub fn get_realm_watermark(&self, family: u64, realm: &str) -> Result { + self.realm_visible_end(family, realm) + .map(|visible_end| visible_end.saturating_sub(1)) + } + + /// Returns the *exclusive* realm visibility frontier: the first realm + /// offset that is not yet visible, and so `0` for a realm holding nothing. + /// + /// [`Self::get_realm_watermark`] is the same frontier stated inclusively, + /// which cannot express "empty" - it floors at zero, and zero is also a + /// real committed offset. Anything that must tell those apart, such as a + /// caught-up read cursor deciding how much history it may claim to have + /// covered, has to ask in this form. + /// + /// # Errors + /// + /// Returns an error if layout activation, storage reads, counter decoding, + /// or fallback offset scanning fails. + pub(crate) fn realm_visible_end(&self, family: u64, realm: &str) -> Result { self.ensure_layout_activation_for_family(family)?; let key = crate::domains::stream::storage::encode_realm_watermark_key(realm); @@ -296,23 +314,25 @@ impl StreamStore { cntryl_midge::TransactionMode::ReadOnly, ) .map_err(|e| format!("failed to begin tx: {e:?}"))?; - let persisted = txn + // The advisory row stores an inclusive offset; the counter already + // stores the exclusive next offset, and commits it atomically with the + // records, which is why it stays the authoritative floor. + let persisted_end = txn .get(&key) .map_err(|e| format!("midge get error: {e:?}"))? - .map(|bytes| WatermarkValue::decode(&bytes).map(|value| value.watermark)) - .transpose()?; - let committed = match txn + .map(|bytes| { + WatermarkValue::decode(&bytes).map(|value| value.watermark.saturating_add(1)) + }) + .transpose()? + .unwrap_or(0); + let committed_end = match txn .get(&counter_key) .map_err(|e| format!("midge get error: {e:?}"))? { - Some(bytes) => RealmCounterValue::decode(&bytes)? - .next_offset - .saturating_sub(1), - None => self - .scan_next_realm_offset(family, realm)? - .saturating_sub(1), + Some(bytes) => RealmCounterValue::decode(&bytes)?.next_offset, + None => self.scan_next_realm_offset(family, realm)?, }; - Ok(persisted.map_or(committed, |watermark| watermark.max(committed))) + Ok(persisted_end.max(committed_end)) } /// Advance the current realm watermark. diff --git a/src/observability/global.rs b/src/observability/global.rs index 7c29df23..30ba0f6f 100644 --- a/src/observability/global.rs +++ b/src/observability/global.rs @@ -152,6 +152,19 @@ pub(crate) fn try_init_observability_with_defaults( init_observability_with_defaults(default_log_level, default_otel_enabled) } +/// Like [`try_init_observability`], but quiets known-noisy dependency logs +/// that are routine on every ephemeral test engine (see +/// [`TEST_QUIET_DEPENDENCY_DIRECTIVES`]). Used by the test harness only -- +/// the production boot path keeps seeing every dependency `WARN`. +pub(crate) fn try_init_test_observability( +) -> Result, Box> { + if let Some(existing) = METRICS_COLLECTOR.get() { + return Ok(existing.clone()); + } + + init_observability_with_options(None, None, false, true) +} + pub(crate) fn try_init_bench_observability( ) -> Result, Box> { // Benchmarks should emit only stress output unless explicitly opted into logs. @@ -159,26 +172,44 @@ pub(crate) fn try_init_bench_observability( return Ok(existing.clone()); } - init_observability_with_options(Some("off"), Some(false), true) + init_observability_with_options(Some("off"), Some(false), true, false) } -fn default_env_filter(log_level: &str) -> EnvFilter { +/// Dependency directives layered on top of the default `warn` catch-all when +/// `quiet_test_dependencies` is set. `cntryl_midge` logs its routine +/// "primary lease acquired" storage-engine startup at `WARN`, which is +/// expected on every single ephemeral test engine and floods test/CI output +/// with nothing actionable -- quiet it to `error` for tests only. This never +/// applies to the production boot path (`init_observability`), so an actual +/// production storage warning still surfaces. +const TEST_QUIET_DEPENDENCY_DIRECTIVES: &str = "cntryl_midge=error"; + +fn default_env_filter(log_level: &str, quiet_test_dependencies: bool) -> EnvFilter { if log_level == "off" { EnvFilter::new("off") + } else if quiet_test_dependencies { + EnvFilter::new(format!( + "fitz={log_level},{TEST_QUIET_DEPENDENCY_DIRECTIVES},warn" + )) } else { EnvFilter::new(format!("fitz={log_level},warn")) } } -fn resolve_env_filter(ignore_env_overrides: bool, log_level: &str) -> EnvFilter { +fn resolve_env_filter( + ignore_env_overrides: bool, + log_level: &str, + quiet_test_dependencies: bool, +) -> EnvFilter { if ignore_env_overrides { - return default_env_filter(log_level); + return default_env_filter(log_level, quiet_test_dependencies); } if std::env::var("RUST_LOG").is_ok() { - EnvFilter::try_from_default_env().unwrap_or_else(|_| default_env_filter(log_level)) + EnvFilter::try_from_default_env() + .unwrap_or_else(|_| default_env_filter(log_level, quiet_test_dependencies)) } else { - default_env_filter(log_level) + default_env_filter(log_level, quiet_test_dependencies) } } @@ -217,13 +248,14 @@ fn init_observability_with_defaults( default_log_level: Option<&str>, default_otel_enabled: Option, ) -> Result, Box> { - init_observability_with_options(default_log_level, default_otel_enabled, false) + init_observability_with_options(default_log_level, default_otel_enabled, false, false) } fn init_observability_with_options( default_log_level: Option<&str>, default_otel_enabled: Option, ignore_env_overrides: bool, + quiet_test_dependencies: bool, ) -> Result, Box> { // Detect logging format let log_format = std::env::var("FITZ_LOG_FORMAT") @@ -241,7 +273,7 @@ fn init_observability_with_options( .to_lowercase(); // Build env filter (RUST_LOG takes precedence) - let env_filter = resolve_env_filter(ignore_env_overrides, &log_level); + let env_filter = resolve_env_filter(ignore_env_overrides, &log_level, quiet_test_dependencies); // Derive service identity and environment metadata let (service_instance_id, deployment_environment) = service_identity(); diff --git a/src/observability/mod.rs b/src/observability/mod.rs index 532b9250..36532b90 100644 --- a/src/observability/mod.rs +++ b/src/observability/mod.rs @@ -82,6 +82,12 @@ pub const METRIC_SESSIONS_CLOSED: &str = "fitz_sessions_closed_total"; pub const METRIC_SESSION_CLEANUP_FAILURES: &str = "fitz_session_cleanup_failures_total"; pub const METRIC_SESSION_CLEANUP_RETRIES: &str = "fitz_session_cleanup_retries_total"; pub const METRIC_SESSION_CLEANUP_SUCCESSES: &str = "fitz_session_cleanup_successes_total"; +/// A cleanup ticket exhausted its retry budget and was dropped instead of +/// being retried forever. Distinct from `METRIC_SESSION_CLEANUP_FAILURES`, +/// which counts every individual failed attempt (including ones that go on +/// to succeed on retry). +pub const METRIC_SESSION_CLEANUP_PERMANENT_FAILURES: &str = + "fitz_session_cleanup_permanent_failures_total"; pub const METRIC_FRAMES_RECEIVED: &str = "fitz_frames_received_total"; pub const METRIC_FRAMES_SENT: &str = "fitz_frames_sent_total"; @@ -103,6 +109,11 @@ pub const METRIC_INGRESS_DOMAIN_BACKPRESSURE_ACCEPTED: &str = "fitz_ingress_domain_backpressure_accepted_total"; pub const METRIC_INGRESS_DOMAIN_BACKPRESSURE_EXHAUSTED: &str = "fitz_ingress_domain_backpressure_exhausted_total"; +/// Domain commands answered with a retryable error because the actor did not +/// reply in time. Previously these closed the whole session, so they were +/// only visible as disconnects. +pub const METRIC_INGRESS_DOMAIN_DISPATCH_TIMEOUTS: &str = + "fitz_ingress_domain_dispatch_timeouts_total"; pub const METRIC_AUTH_FAILURES: &str = "fitz_auth_failures_total"; pub const METRIC_PERMISSION_DENIALS: &str = "fitz_permission_denials_total"; diff --git a/src/protocol/error_codes.rs b/src/protocol/error_codes.rs index b4885920..db7bad94 100644 --- a/src/protocol/error_codes.rs +++ b/src/protocol/error_codes.rs @@ -68,6 +68,12 @@ pub mod kv { pub const ERR_UNAUTHORIZED: u16 = 1011; // AC-KV-010: Permission denied for KV operation pub const ERR_INVALID_SUBSCRIPTION_PATTERN: u16 = 1012; pub const ERR_SUBSCRIPTION_LIMIT: u16 = 1013; + /// The domain mailbox was full, so the request was never accepted. + /// + /// Retryable: nothing was enqueued and nothing applied, so re-sending after + /// a backoff is safe. Distinct from `ERR_BACKEND_ERROR`, which is fatal and + /// says nothing about whether the request took effect. + pub const ERR_BUSY: u16 = 1014; } /// Stream domain error codes (per `CLIENT_SPEC` Stream Domain section) @@ -83,6 +89,15 @@ pub mod stream { pub const ERR_INVALID_SUBSCRIPTION_PATTERN: u16 = 2010; pub const ERR_SUBSCRIPTION_LIMIT: u16 = 2011; pub const ERR_BACKEND_ERROR: u16 = 2012; + /// A single record's wire-encoded size alone exceeds the maximum size of + /// one broker response frame, so it cannot be returned by any read call. + pub const ERR_READ_RESPONSE_TOO_LARGE: u16 = 2013; + /// The domain mailbox was full, so the request was never accepted. + /// + /// Retryable: nothing was enqueued and nothing applied, so re-sending after + /// a backoff is safe. Distinct from `ERR_BACKEND_ERROR`, which is fatal and + /// says nothing about whether the request took effect. + pub const ERR_BUSY: u16 = 2014; } /// Notice domain error codes (per `CLIENT_SPEC` Notice Domain section) @@ -93,6 +108,12 @@ pub mod notice { pub const ERR_TRANSPORT_CLOSED: u16 = 3004; pub const ERR_BACKEND_ERROR: u16 = 3005; pub const ERR_UNAUTHORIZED: u16 = 3009; // AC-NOTICE-009: Permission denied for notice operation + /// The domain mailbox was full, so the request was never accepted. + /// + /// Retryable: nothing was enqueued and nothing applied, so re-sending after + /// a backoff is safe. Distinct from `ERR_BACKEND_ERROR`, which is fatal and + /// says nothing about whether the request took effect. + pub const ERR_BUSY: u16 = 3006; } /// Queue domain error codes (per `CLIENT_SPEC` Queue Domain section) @@ -151,4 +172,13 @@ pub mod schedule { pub const ERR_SUBSCRIPTION_LIMIT: u16 = 7007; pub const ERR_INVALID_DELIVERY_MODE: u16 = 7008; pub const ERR_UNAUTHORIZED: u16 = 7009; // AC-SCHEDULE-008: Permission denied for schedule operation + // Schedule had no generic backend/saturation code; every other domain has + // one. Without it a busy schedule actor had to borrow a code that means + // something else (e.g. "invalid cron"), which misdirects the client. + pub const ERR_BACKEND_ERROR: u16 = 7010; + // Distinct from 7010 on purpose. 7010 is classified retryable: it reports a + // request the broker declined to service, so re-sending it is safe. A + // deadline expiry is not that - the command was already accepted and may + // still apply - so it needs a code clients will not auto-retry. + pub const ERR_TIMEOUT: u16 = 7011; } diff --git a/src/protocol/kv_codec/mutation_parsers.rs b/src/protocol/kv_codec/mutation_parsers.rs index 00436b9d..5f1609ca 100644 --- a/src/protocol/kv_codec/mutation_parsers.rs +++ b/src/protocol/kv_codec/mutation_parsers.rs @@ -132,6 +132,13 @@ pub(super) fn parse_scan(route_family: RouteFamily, payload: &[u8]) -> Result Result usize { usize::try_from(value).unwrap_or(usize::MAX) @@ -217,6 +218,9 @@ fn stream_error_code_for_message(message: &str) -> u16 { if message.contains(ERR_STREAM_FILTER_INVALID_PAYLOAD) { return stream::ERR_STREAM_FILTER_INVALID_PAYLOAD; } + if message.contains(ERR_READ_RESPONSE_TOO_LARGE) { + return stream::ERR_READ_RESPONSE_TOO_LARGE; + } match message { "session already active" => stream::ERR_SESSION_ALREADY_ACTIVE, @@ -544,6 +548,27 @@ mod tests { ); } + #[test] + fn should_encode_read_response_too_large_error_with_its_own_code() { + // Arrange: store-layer read paths raise this when a single record's + // wire-encoded size alone exceeds the maximum response frame size. + let response = StreamClientResponseBody::Error( + "ERR_READ_RESPONSE_TOO_LARGE: record at offset 0 is 70000 bytes, exceeding the \ + 65535-byte read response limit" + .to_string(), + ); + + // Act + let read = encode_response(604, &response); + + // Assert + assert_eq!( + &read[1..5], + &u32::from(crate::protocol::error_codes::stream::ERR_READ_RESPONSE_TOO_LARGE) + .to_be_bytes() + ); + } + #[test] fn should_parse_frozen_stream_request_golden_vectors() { // Arrange diff --git a/src/protocol/test_support.rs b/src/protocol/test_support.rs new file mode 100644 index 00000000..3a1cc4a0 --- /dev/null +++ b/src/protocol/test_support.rs @@ -0,0 +1,18 @@ +//! Shared protocol test helpers used across domain sink unit tests. + +use super::frame::ChannelId; +use crate::runtime::ClientChannel; + +/// Map a runtime `ClientChannel` to its wire `ChannelId`, for constructing +/// test frames/envelopes that exercise a specific channel. +#[cfg(test)] +pub(crate) fn channel_id_from_client(channel: ClientChannel) -> ChannelId { + match channel { + ClientChannel::Control => ChannelId::Control, + ClientChannel::Pub => ChannelId::Pub, + ClientChannel::Sub => ChannelId::Sub, + ClientChannel::Rpc => ChannelId::Rpc, + ClientChannel::Lease => ChannelId::Lease, + ClientChannel::Internal => ChannelId::Internal, + } +} diff --git a/src/runtime/actor.rs b/src/runtime/actor.rs index c87cdaef..fbce6876 100644 --- a/src/runtime/actor.rs +++ b/src/runtime/actor.rs @@ -28,7 +28,9 @@ fn delivery_error_to_send_error(target: RouteAddress, error: &DeliveryError) -> occupancy: usize_to_f64_saturating(*current_len) / usize_to_f64_saturating(*capacity), }, DeliveryError::ActorStopped | DeliveryError::Timeout => SendError::ActorStopped { target }, - DeliveryError::SinkPanicked => SendError::SinkPanicked { target }, + DeliveryError::SinkPanicked | DeliveryError::InvalidPayload { .. } => { + SendError::SinkPanicked { target } + } } } diff --git a/src/runtime/cleanup_guard.rs b/src/runtime/cleanup_guard.rs new file mode 100644 index 00000000..34e08e22 --- /dev/null +++ b/src/runtime/cleanup_guard.rs @@ -0,0 +1,43 @@ +//! Shared bounded LRU-style guard for disconnect-cleanup session tracking. +//! +//! `SessionCleanup` is delivered on the high-priority/control-plane mailbox +//! lane, so it can pass an older, already-queued normal-lane request from +//! the same session. Remembering the cleaned-up session lets that stale +//! request fail instead of silently recreating actor/session state for a +//! session that is already gone and will never be cleaned up again. + +use std::collections::{HashSet, VecDeque}; + +/// Bounded record of sessions that disconnect cleanup has already run for. +pub struct CleanedUpSessions { + order: VecDeque, + seen: HashSet, + capacity: usize, +} + +impl CleanedUpSessions { + #[must_use] + pub fn new(capacity: usize) -> Self { + Self { + order: VecDeque::new(), + seen: HashSet::new(), + capacity: capacity.max(1), + } + } + + pub fn mark(&mut self, session_id: u64) { + if self.seen.insert(session_id) { + self.order.push_back(session_id); + if self.order.len() > self.capacity { + if let Some(oldest) = self.order.pop_front() { + self.seen.remove(&oldest); + } + } + } + } + + #[must_use] + pub fn contains(&self, session_id: u64) -> bool { + self.seen.contains(&session_id) + } +} diff --git a/src/runtime/family_actor_pool.rs b/src/runtime/family_actor_pool.rs index a5a71dc9..7ec9d3a2 100644 --- a/src/runtime/family_actor_pool.rs +++ b/src/runtime/family_actor_pool.rs @@ -270,14 +270,30 @@ pub struct FamilyActorShard { /// /// The pool itself only owns bounded channels. This wrapper owns one worker /// thread per shard and creates one state value per provisioned family on that -/// worker. A handler panic fails the pool closed and drops the message that -/// triggered it; callers observe `ActorStopped` through the bounded edge. +/// worker. A handler panic fails *that family* closed and drops the message +/// that triggered it; callers observe `ActorStopped` through the bounded edge +/// for that family only. Route families are a hard isolation boundary +/// (`RouteFamily`, `docs/development/domain-boundaries-spec.md`), so a panic +/// scoped to one family must never make the pool unusable for the others it +/// multiplexes (see `should_keep_sibling_family_running_after_a_family_actor_panics`). +/// The one exception is a shard receiving work for a family it was never +/// constructed with, which cannot happen under correct routing; that remains +/// a pool-fatal condition since it indicates a routing/config bug rather than +/// a per-family runtime fault. +/// +/// A single family's failure never flips pool-wide health (`is_running`), +/// but once *every* provisioned family has failed closed the pool has zero +/// remaining capacity -- that is an honest aggregate fact about the pool, +/// not a blast-radius cascade from any one family, so it does flip +/// `is_running` to `false` (see +/// `should_fail_pool_closed_after_every_family_panics`). pub struct FamilyActorPoolRuntime { ingress: FamilyActorIngress, active: Arc, running: Arc, failed: Arc, panic_count: Arc, + family_failed: Arc>, join_handles: parking_lot::Mutex>>, } @@ -298,10 +314,58 @@ impl FamilyActorPoolRuntime { /// retaining a small, synchronous dispatch surface. #[must_use] pub fn spawn( + pool: FamilyActorPool, + active: Arc, + state_factory: F, + handler: H, + ) -> Self + where + S: Send + 'static, + F: Fn(RouteFamily) -> S + Send + Sync + 'static, + H: Fn(&mut S, RouteFamily, FamilyActorLane, M) + Send + Sync + 'static, + { + Self::spawn_inner(pool, active, state_factory, handler, None) + } + + /// Like [`Self::spawn`], but increments `family_failed_metric` once per + /// route family whose handler panics and fails closed. + /// + /// A single family's failure deliberately does not flip domain-wide + /// health/liveness (that would reintroduce the very blast-radius bug this + /// isolation exists to prevent) -- until every provisioned family has + /// failed, in which case the pool legitimately has no remaining capacity + /// and `is_running` does flip. Short of full exhaustion, this counter is + /// the only operator-visible signal for a permanently degraded + /// family/realm — without it, such a failure is observable only via a + /// log line. + #[must_use] + pub fn spawn_with_family_failed_metric( + pool: FamilyActorPool, + active: Arc, + state_factory: F, + handler: H, + family_failed_metric: &'static str, + ) -> Self + where + S: Send + 'static, + F: Fn(RouteFamily) -> S + Send + Sync + 'static, + H: Fn(&mut S, RouteFamily, FamilyActorLane, M) + Send + Sync + 'static, + { + Self::spawn_inner( + pool, + active, + state_factory, + handler, + Some(family_failed_metric), + ) + } + + fn spawn_inner( mut pool: FamilyActorPool, active: Arc, state_factory: F, handler: H, + family_failed_metric: Option<&'static str>, ) -> Self where S: Send + 'static, @@ -312,6 +376,13 @@ impl FamilyActorPoolRuntime { let running = Arc::new(AtomicBool::new(true)); let failed = Arc::new(AtomicBool::new(false)); let panic_count = Arc::new(AtomicU64::new(0)); + let family_failed: Arc> = Arc::new( + ingress + .families() + .into_iter() + .map(|family| (family.id(), AtomicBool::new(false))) + .collect(), + ); let state_factory = Arc::new(state_factory); let handler = Arc::new(handler); let mut join_handles = Vec::with_capacity(pool.shard_count()); @@ -329,6 +400,8 @@ impl FamilyActorPoolRuntime { let worker_running = running.clone(); let worker_failed = failed.clone(); let worker_panic_count = panic_count.clone(); + let worker_family_failed = family_failed.clone(); + let worker_family_failed_metric = family_failed_metric; let worker_handler = handler.clone(); let worker_ingress = ingress.clone(); join_handles.push(thread::spawn(move || { @@ -343,6 +416,10 @@ impl FamilyActorPoolRuntime { }; let Some(state) = family_states.get_mut(&work.family.id()) else { + // Work for a family this shard was never constructed + // with is a routing/config bug, not a per-family + // runtime fault -- it should never happen under + // correct routing, so it stays pool-fatal. worker_panic_count.fetch_add(1, Ordering::Relaxed); worker_failed.store(true, Ordering::Release); worker_active.store(false, Ordering::Release); @@ -354,20 +431,53 @@ impl FamilyActorPoolRuntime { break; }; + if worker_family_failed + .get(&work.family.id()) + .is_some_and(|flag| flag.load(Ordering::Acquire)) + { + // This family already failed closed after an earlier + // handler panic. Drop the message without invoking + // the handler again -- the reply sender embedded in + // `work.message` is dropped here, which callers + // observe as `ActorStopped` via `reply_wait`, not a + // hang. Sibling families keep being drained below. + tracing::warn!( + family = work.family.id(), + "dropping work for a family that already failed closed" + ); + continue; + } + if std::panic::catch_unwind(AssertUnwindSafe(|| { worker_handler(state, work.family, work.lane, work.message); })) .is_err() { worker_panic_count.fetch_add(1, Ordering::Relaxed); - worker_failed.store(true, Ordering::Release); - worker_active.store(false, Ordering::Release); - worker_ingress.wake_all(); + if let Some(flag) = worker_family_failed.get(&work.family.id()) { + flag.store(true, Ordering::Release); + } + if let Some(metric) = worker_family_failed_metric { + crate::observability::counter_inc(metric); + } tracing::error!( family = work.family.id(), - "family actor failed closed after handler panic" + "family actor failed closed for this family after handler panic" ); - break; + // Do not break and do not touch `running`/`active` here: + // sibling families on this shard, and every other + // shard, must keep making progress. `failed` is the + // one exception -- if every provisioned family is now + // failed closed, the pool has zero remaining capacity, + // which is an honest pool-wide health fact (not a + // blast-radius cascade from this one family), so + // flip it to surface that aggregate exhaustion. + if worker_family_failed + .values() + .all(|flag| flag.load(Ordering::Acquire)) + { + worker_failed.store(true, Ordering::Release); + } } } worker_running.store(false, Ordering::Release); @@ -380,6 +490,7 @@ impl FamilyActorPoolRuntime { running, failed, panic_count, + family_failed, join_handles: parking_lot::Mutex::new(join_handles), } } @@ -401,12 +512,16 @@ impl FamilyActorPoolRuntime { lane: FamilyActorLane, message: M, ) -> Result<(), FamilyActorEnqueueError> { - if !self.is_running() { + if !self.is_family_running(family) { return Err(FamilyActorEnqueueError::ActorStopped); } self.ingress.try_enqueue(family, lane, message) } + /// Pool-wide liveness. A single family's handler panic never flips this + /// (see the type-level docs) -- only an explicit [`Self::fail_closed`] + /// call, [`Self::stop`], or every provisioned family having failed + /// closed does. #[must_use] pub fn is_running(&self) -> bool { self.active.load(Ordering::Acquire) @@ -414,6 +529,32 @@ impl FamilyActorPoolRuntime { && !self.failed.load(Ordering::Acquire) } + /// Whether `family` is still accepting work. + /// + /// This is `false` for a family whose handler has panicked (fail-closed + /// for that family only) or when the whole pool has stopped/failed. An + /// unprovisioned family is not tracked here; `try_enqueue` rejects it + /// separately as `UnknownFamily` via the ingress family lookup. + #[must_use] + pub fn is_family_running(&self, family: RouteFamily) -> bool { + self.is_running() + && !self + .family_failed + .get(&family.id()) + .is_some_and(|flag| flag.load(Ordering::Acquire)) + } + + /// Count of provisioned families whose handler has panicked and failed + /// closed. Mirrors `keyed_family_executor.rs`'s method of the same name. + #[cfg(test)] + #[must_use] + pub(crate) fn failed_family_count(&self) -> usize { + self.family_failed + .values() + .filter(|flag| flag.load(Ordering::Acquire)) + .count() + } + #[must_use] pub fn health_snapshot(&self) -> FamilyActorPoolHealthSnapshot { FamilyActorPoolHealthSnapshot { @@ -864,6 +1005,86 @@ mod tests { assert!(!runtime.is_running()); } + #[test] + fn should_keep_sibling_family_running_after_a_family_actor_panics() { + // Arrange: enough families that at least two are forced onto the + // *same* shard thread by pigeonhole (shard count is capped at + // available parallelism, `should_cap_shards_at_provisioned_family_count`). + // This proves in-shard isolation -- that the shard's drain loop + // `continue`s past a panicking family rather than starving or + // breaking for its thread-mates -- not just cross-shard isolation, + // which two families alone could pass trivially on any multi-core + // host. This mirrors production, where one + // `RpcDomainSink`/`StreamDomainSink` multiplexes every provisioned + // realm/route family onto one `FamilyActorPoolRuntime` + // (see `src/boot/domains.rs`). + let shard_count = shard_count_for_family_count(usize::MAX); + let family_count = shard_count + 1; + let families = (1..=family_count) + .map(|id| family(u32::try_from(id).expect("test family fits"))) + .collect::>(); + let (panicking, sibling) = families + .iter() + .copied() + .enumerate() + .find_map(|(index, candidate)| { + families[index + 1..] + .iter() + .copied() + .find(|&other| { + family_shard_affinity(candidate, shard_count) + == family_shard_affinity(other, shard_count) + }) + .map(|other| (candidate, other)) + }) + .expect("pigeonhole guarantees a same-shard pair"); + + let pool = FamilyActorPool::::new(&families).expect("pool"); + let active = Arc::new(AtomicBool::new(true)); + let (observed_tx, observed_rx) = bounded::(4); + let runtime = FamilyActorPoolRuntime::spawn( + pool, + active, + |_| (), + move |(), target_family, _lane, message: u64| { + assert!( + target_family != panicking, + "injected handler panic for a same-shard family" + ); + if target_family == sibling { + observed_tx.send(message).expect("sibling observer"); + } + }, + ); + + // Act: trigger the panic and wait for that family to fail closed. + runtime + .try_enqueue(panicking, FamilyActorLane::Normal, 1) + .expect("panicking family enqueue"); + let deadline = std::time::Instant::now() + Duration::from_secs(1); + while runtime + .try_enqueue(panicking, FamilyActorLane::Normal, 99) + .is_ok() + && std::time::Instant::now() < deadline + { + thread::yield_now(); + } + + // Assert: a family sharing the same shard thread as the panicking + // one must keep accepting and processing work. A fault confined to + // one route family/realm must not make the domain unusable for + // every other family multiplexed onto the same worker thread. + runtime + .try_enqueue(sibling, FamilyActorLane::Normal, 42) + .expect("a same-shard sibling family must keep accepting work after a panic"); + assert_eq!( + observed_rx + .recv_timeout(Duration::from_secs(1)) + .expect("a same-shard sibling family must keep making progress after a panic"), + 42 + ); + } + #[test] fn should_reject_new_work_given_failed_family_actor() { // Arrange @@ -892,7 +1113,13 @@ mod tests { thread::yield_now(); } - // Assert + // Assert: this pool has exactly one provisioned family, so that + // family failing closed *is* full exhaustion -- the pool-wide + // `is_running()` correctly follows suit here, same as it would for + // any non-sharded domain's single actor. (A multi-family pool keeps + // `is_running()` true after one sibling's panic -- + // see `should_keep_pool_running_given_one_of_several_families_panics`.) + assert!(!runtime.is_family_running(family(1))); assert!(!runtime.is_running()); assert_eq!( runtime.try_enqueue(family(1), FamilyActorLane::Normal, 2), @@ -900,4 +1127,80 @@ mod tests { ); assert_eq!(runtime.health_snapshot().panic_count, 1); } + + #[test] + fn should_keep_pool_running_given_one_of_several_families_panics() { + // Arrange: multiple provisioned families so a single panic must be + // isolation, not exhaustion. + let families = [family(1), family(2), family(3)]; + let pool = FamilyActorPool::::new(&families).expect("pool"); + let active = Arc::new(AtomicBool::new(true)); + let (started_tx, started_rx) = bounded(1); + let runtime = FamilyActorPoolRuntime::spawn( + pool, + active, + |_| (), + move |(), target_family, _lane, _message: u64| { + if target_family == family(1) { + started_tx.send(()).expect("panic observer"); + panic!("injected handler panic"); + } + }, + ); + + // Act + runtime + .try_enqueue(family(1), FamilyActorLane::Normal, 1) + .expect("panic command enqueue"); + started_rx + .recv_timeout(Duration::from_secs(1)) + .expect("handler started"); + let deadline = std::time::Instant::now() + Duration::from_secs(1); + while runtime.is_family_running(family(1)) && std::time::Instant::now() < deadline { + thread::yield_now(); + } + + // Assert: a single family's panic isolates that family only -- pool + // health/liveness and restart-exhaustion must both stay unaffected. + assert!(!runtime.is_family_running(family(1))); + assert!(runtime.is_running()); + assert_eq!(runtime.failed_family_count(), 1); + let health = runtime.managed_actor_health_snapshot(); + assert!(health.running); + assert!(!health.restart_exhausted); + } + + #[test] + fn should_fail_pool_closed_after_every_family_panics() { + // Arrange + let families = [family(1), family(2), family(3)]; + let pool = FamilyActorPool::::new(&families).expect("pool"); + let active = Arc::new(AtomicBool::new(true)); + let runtime = FamilyActorPoolRuntime::spawn( + pool, + active, + |_| (), + |(), _family, _lane, _message: u64| { + panic!("injected handler panic"); + }, + ); + + // Act: panic every provisioned family. + for target in families { + let _ = runtime.try_enqueue(target, FamilyActorLane::Normal, 1); + } + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while runtime.is_running() && std::time::Instant::now() < deadline { + thread::yield_now(); + } + + // Assert: full exhaustion is an honest pool-wide fact, unlike a + // single family's failure, so `running`/`restart_exhausted` must + // both flip. + assert!(!runtime.is_running()); + assert_eq!(runtime.failed_family_count(), families.len()); + let health = runtime.managed_actor_health_snapshot(); + assert!(!health.running); + assert!(health.restart_exhausted); + } } diff --git a/src/runtime/ingress_support.rs b/src/runtime/ingress_support.rs new file mode 100644 index 00000000..99eb7890 --- /dev/null +++ b/src/runtime/ingress_support.rs @@ -0,0 +1,31 @@ +//! Shared ingress helpers for domain sinks. + +use super::{DeliveryError, Envelope}; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Reject delivery if the domain sink has been marked inactive. +pub(crate) fn ensure_actor_active(active: &AtomicBool) -> Result<(), DeliveryError> { + if !active.load(Ordering::Relaxed) { + return Err(DeliveryError::ActorStopped); + } + + Ok(()) +} + +/// Trace-log an inbound envelope at debug level, tagged with the domain name. +/// +/// `message` carries the domain-specific log text (e.g. `"Lease domain +/// sink: received envelope"`) so each caller keeps its own capitalization. +pub(crate) fn log_envelope_received( + domain: &'static str, + message: &'static str, + envelope: &Envelope, +) { + tracing::debug!( + domain = domain, + destination = %envelope.destination(), + source = ?envelope.source(), + "{}", + message + ); +} diff --git a/src/runtime/keyed_actor_pool.rs b/src/runtime/keyed_actor_pool.rs index df0e22ec..236fa7ed 100644 --- a/src/runtime/keyed_actor_pool.rs +++ b/src/runtime/keyed_actor_pool.rs @@ -45,7 +45,13 @@ where /// A no-op if an actor for `key` already exists, even if that actor has /// since failed closed after a panic — callers observe delivery failure /// through the normal `Router::route` error path, matching every other - /// fail-closed actor in the runtime. + /// fail-closed actor in the runtime. This key's own dead entry is never + /// replaced or restarted. + /// + /// `max_actors` bounds *live* actors, not ever-created ones: a dead entry + /// left behind by some other key's past panic does not count against the + /// cap, since a transient fault confined to one key must not permanently + /// consume shared capacity that unrelated keys need to ever get a slot. pub fn ensure_spawned(&self, key: K, address: RouteAddress, actor_factory: F) -> bool where A: Actor, @@ -59,7 +65,8 @@ where if actors.contains_key(&key) { return true; } - if self.max_actors == 0 || actors.len() >= self.max_actors { + let live_count = actors.values().filter(|actor| actor.is_running()).count(); + if self.max_actors == 0 || live_count >= self.max_actors { return false; } actors.entry(key).or_insert_with(|| { @@ -132,6 +139,61 @@ mod tests { assert_eq!(count.load(Ordering::SeqCst), 12); } + #[test] + fn should_reclaim_capacity_from_dead_actors_for_an_unrelated_key() { + // Arrange: a pool capped at 2 live actors. Two distinct keys each + // get an actor that panics immediately on any message, mirroring a + // production key (e.g. a Stream realm/area watermark coordinator, + // see `src/domains/stream/sink/domain_sink_impl.rs`) whose handler + // fails once and is left fail-closed forever. + struct PanicActor; + impl Actor for PanicActor { + type Message = u64; + fn receive(&mut self, _msg: u64, _ctx: &mut Context) { + panic!("injected handler panic"); + } + } + + let router = Arc::new(Router::new()); + let pool: KeyedActorPool = KeyedActorPool::new(router.clone(), 16, 2); + + for key in 0..2u64 { + let address = RouteAddress::new( + RouteFamily::new(1), + Route::new(format!("stream://bench/area/{key}")), + ); + assert!(pool.ensure_spawned(key, address.clone(), || PanicActor)); + router + .route(Envelope::new(address, 1_u64)) + .expect("route to spawned actor"); + } + + // Wait for both to actually fail closed (panic is async). + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(1); + while pool.actors.read().values().any(ManagedActor::is_running) + && std::time::Instant::now() < deadline + { + std::thread::yield_now(); + } + assert!( + pool.actors.read().values().all(|actor| !actor.is_running()), + "both keys should have failed closed" + ); + + // Act: a brand-new, unrelated key has never failed and has nothing + // to do with the two dead entries above. + let fresh_address = + RouteAddress::new(RouteFamily::new(1), Route::new("stream://bench/area/2")); + let spawned = pool.ensure_spawned(2u64, fresh_address, || PanicActor); + + // Assert: capacity exhausted purely by historical, unrelated panics + // must not permanently block a key that never failed. + assert!( + spawned, + "an unrelated key must still get a slot after other keys' actors failed closed" + ); + } + #[test] fn should_preserve_existing_actors_when_capacity_is_reached() { // Arrange diff --git a/src/runtime/keyed_family_executor.rs b/src/runtime/keyed_family_executor.rs new file mode 100644 index 00000000..e147a867 --- /dev/null +++ b/src/runtime/keyed_family_executor.rs @@ -0,0 +1,698 @@ +//! Bounded synchronous execution with family isolation and per-key ordering. + +use crate::runtime::family_actor_pool::{ + FamilyActorEnqueueError, FamilyActorLane, FAMILY_ACTOR_CONTROL_LANE_CAPACITY, + FAMILY_ACTOR_NORMAL_LANE_CAPACITY, +}; +use crate::runtime::routing::RouteFamily; +use parking_lot::{Condvar, Mutex}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::hash::Hash; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::Arc; +use std::thread::{self, JoinHandle}; + +struct FamilyState { + state: Arc, + normal: HashMap>, + ready: VecDeque, + active_keys: HashSet, + normal_len: usize, + control: VecDeque, + control_active: bool, + failed: bool, +} + +impl FamilyState { + fn new(state: S) -> Self { + Self { + state: Arc::new(state), + normal: HashMap::new(), + ready: VecDeque::new(), + active_keys: HashSet::new(), + normal_len: 0, + control: VecDeque::new(), + control_active: false, + failed: false, + } + } +} + +struct Scheduler { + families: BTreeMap>, + stopped: bool, + next_family: usize, +} + +enum Work { + Normal { + family: RouteFamily, + key: K, + message: M, + state: Arc, + }, + Control { + family: RouteFamily, + message: M, + state: Arc, + }, +} + +struct Shared { + scheduler: Mutex>, + ready: Condvar, +} + +/// Fixed-thread executor that serializes one key while allowing sibling keys +/// in the same route family to overlap. +pub(crate) struct KeyedFamilyExecutor { + shared: Arc>, + workers: Mutex>>, +} + +impl KeyedFamilyExecutor +where + K: Clone + Eq + Hash + Send + 'static, + M: Send + 'static, + S: Send + Sync + 'static, +{ + pub(crate) fn new( + families: &[RouteFamily], + worker_count: usize, + state_factory: StateFactory, + handler: Handler, + family_failed: Failure, + ) -> Result + where + StateFactory: Fn(RouteFamily) -> S, + Handler: Fn(&S, RouteFamily, FamilyActorLane, Option<&K>, M) + Send + Sync + 'static, + Failure: Fn(RouteFamily) + Send + Sync + 'static, + { + if families.is_empty() { + return Err("no route families were provisioned".to_owned()); + } + if worker_count == 0 { + return Err("worker count must be greater than zero".to_owned()); + } + let mut provisioned = BTreeMap::new(); + for family in families { + if family.id() == 0 { + return Err("route family zero cannot be provisioned".to_owned()); + } + if provisioned + .insert(family.id(), FamilyState::new(state_factory(*family))) + .is_some() + { + return Err(format!("duplicate route family {}", family.id())); + } + } + let shared = Arc::new(Shared { + scheduler: Mutex::new(Scheduler { + families: provisioned, + stopped: false, + next_family: 0, + }), + ready: Condvar::new(), + }); + let handler = Arc::new(handler); + let family_failed = Arc::new(family_failed); + let workers = (0..worker_count.min(32)) + .map(|index| { + let shared = shared.clone(); + let handler = handler.clone(); + let family_failed = family_failed.clone(); + thread::Builder::new() + .name(format!("fitz-keyed-family-{index}")) + .spawn(move || { + worker_loop(&shared, handler.as_ref(), family_failed.as_ref()); + }) + .map_err(|error| format!("spawn keyed family worker: {error}")) + }) + .collect::, _>>()?; + Ok(Self { + shared, + workers: Mutex::new(workers), + }) + } + + pub(crate) fn production_worker_count() -> usize { + thread::available_parallelism() + .map_or(1, std::num::NonZeroUsize::get) + .min(32) + } + + pub(crate) fn try_enqueue( + &self, + family: RouteFamily, + key: K, + message: M, + ) -> Result<(), FamilyActorEnqueueError> { + let mut scheduler = self.shared.scheduler.lock(); + if scheduler.stopped { + return Err(FamilyActorEnqueueError::ActorStopped); + } + let Some(state) = scheduler.families.get_mut(&family.id()) else { + return Err(FamilyActorEnqueueError::UnknownFamily); + }; + if state.failed { + return Err(FamilyActorEnqueueError::ActorStopped); + } + if state.normal_len == FAMILY_ACTOR_NORMAL_LANE_CAPACITY { + return Err(FamilyActorEnqueueError::NormalLaneFull); + } + let queue = state.normal.entry(key.clone()).or_default(); + let was_empty = queue.is_empty(); + queue.push_back(message); + state.normal_len += 1; + if was_empty && !state.active_keys.contains(&key) { + state.ready.push_back(key); + } + drop(scheduler); + self.shared.ready.notify_one(); + Ok(()) + } + + pub(crate) fn try_enqueue_control( + &self, + family: RouteFamily, + message: M, + ) -> Result<(), FamilyActorEnqueueError> { + let mut scheduler = self.shared.scheduler.lock(); + if scheduler.stopped { + return Err(FamilyActorEnqueueError::ActorStopped); + } + let Some(state) = scheduler.families.get_mut(&family.id()) else { + return Err(FamilyActorEnqueueError::UnknownFamily); + }; + if state.failed { + return Err(FamilyActorEnqueueError::ActorStopped); + } + if state.control.len() == FAMILY_ACTOR_CONTROL_LANE_CAPACITY { + return Err(FamilyActorEnqueueError::ControlLaneFull); + } + state.control.push_back(message); + drop(scheduler); + self.shared.ready.notify_all(); + Ok(()) + } + + pub(crate) fn is_family_running(&self, family: RouteFamily) -> bool { + let scheduler = self.shared.scheduler.lock(); + !scheduler.stopped + && scheduler + .families + .get(&family.id()) + .is_some_and(|state| !state.failed) + } + + pub(crate) fn is_running(&self) -> bool { + let scheduler = self.shared.scheduler.lock(); + !scheduler.stopped && scheduler.families.values().any(|state| !state.failed) + } + + pub(crate) fn failed_family_count(&self) -> usize { + self.shared + .scheduler + .lock() + .families + .values() + .filter(|state| state.failed) + .count() + } + + pub(crate) fn stop(&self) { + let mut scheduler = self.shared.scheduler.lock(); + scheduler.stopped = true; + for state in scheduler.families.values_mut() { + state.normal.clear(); + state.ready.clear(); + state.normal_len = 0; + state.control.clear(); + } + drop(scheduler); + self.shared.ready.notify_all(); + } + + pub(crate) fn join(&self) { + self.stop(); + for worker in self.workers.lock().drain(..) { + let _ = worker.join(); + } + } +} + +impl Drop for KeyedFamilyExecutor { + fn drop(&mut self) { + let mut scheduler = self.shared.scheduler.lock(); + scheduler.stopped = true; + drop(scheduler); + self.shared.ready.notify_all(); + for worker in self.workers.get_mut().drain(..) { + let _ = worker.join(); + } + } +} + +fn worker_loop( + shared: &Shared, + handler: &Handler, + family_failed: &Failure, +) where + K: Clone + Eq + Hash, + Handler: Fn(&S, RouteFamily, FamilyActorLane, Option<&K>, M), + Failure: Fn(RouteFamily), +{ + loop { + let work = { + let mut scheduler = shared.scheduler.lock(); + loop { + if scheduler.stopped { + return; + } + if let Some(work) = take_work(&mut scheduler) { + break work; + } + shared.ready.wait(&mut scheduler); + } + }; + let family = match &work { + Work::Normal { family, .. } | Work::Control { family, .. } => *family, + }; + let completed_key = match &work { + Work::Normal { key, .. } => Some(key.clone()), + Work::Control { .. } => None, + }; + let result = catch_unwind(AssertUnwindSafe(|| match work { + Work::Normal { + family, + ref key, + message, + ref state, + } => handler(state, family, FamilyActorLane::Normal, Some(key), message), + Work::Control { + family, + message, + ref state, + } => handler(state, family, FamilyActorLane::Control, None, message), + })); + let mut scheduler = shared.scheduler.lock(); + let Some(state) = scheduler.families.get_mut(&family.id()) else { + continue; + }; + if result.is_ok() { + finish_work(state, completed_key); + } else { + let first_failure = !state.failed; + state.failed = true; + state.normal.clear(); + state.ready.clear(); + state.normal_len = 0; + state.control.clear(); + state.active_keys.clear(); + state.control_active = false; + drop(scheduler); + if first_failure { + family_failed(family); + } + shared.ready.notify_all(); + continue; + } + drop(scheduler); + shared.ready.notify_all(); + } +} + +fn take_work(scheduler: &mut Scheduler) -> Option> +where + K: Clone + Eq + Hash, +{ + let ids = scheduler.families.keys().copied().collect::>(); + for delta in 0..ids.len() { + let index = (scheduler.next_family + delta) % ids.len(); + let id = ids[index]; + let state = scheduler.families.get_mut(&id)?; + if state.failed || state.control_active { + continue; + } + let family = RouteFamily::new(id); + if !state.control.is_empty() { + if state.active_keys.is_empty() { + state.control_active = true; + scheduler.next_family = (index + 1) % ids.len(); + return Some(Work::Control { + family, + message: state.control.pop_front()?, + state: state.state.clone(), + }); + } + continue; + } + while let Some(key) = state.ready.pop_front() { + if state.active_keys.contains(&key) { + continue; + } + let Some(message) = state.normal.get_mut(&key).and_then(VecDeque::pop_front) else { + state.normal.remove(&key); + continue; + }; + state.normal_len -= 1; + state.active_keys.insert(key.clone()); + scheduler.next_family = (index + 1) % ids.len(); + return Some(Work::Normal { + family, + key, + message, + state: state.state.clone(), + }); + } + } + None +} + +fn finish_work(state: &mut FamilyState, completed_key: Option) +where + K: Clone + Eq + Hash, +{ + match completed_key { + Some(key) => { + state.active_keys.remove(&key); + if state + .normal + .get(&key) + .is_some_and(|queue| !queue.is_empty()) + { + state.ready.push_back(key); + } else { + state.normal.remove(&key); + } + } + None => state.control_active = false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::time::Duration; + + #[test] + fn should_overlap_different_keys_without_overlapping_one_key() { + // Arrange + let active = Arc::new(Mutex::new(HashSet::new())); + let overlapped = Arc::new(AtomicBool::new(false)); + let same_key_overlap = Arc::new(AtomicBool::new(false)); + let (release_tx, release_rx) = crossbeam_channel::bounded::<()>(0); + let (entered_tx, entered_rx) = crossbeam_channel::bounded(2); + let executor = KeyedFamilyExecutor::new( + &[RouteFamily::new(1)], + 2, + |_| (), + { + let active = active.clone(); + let overlapped = overlapped.clone(); + let same_key_overlap = same_key_overlap.clone(); + move |(), _, _, key: Option<&u64>, ()| { + let key = *key.expect("normal key"); + let mut keys = active.lock(); + if !keys.insert(key) { + same_key_overlap.store(true, Ordering::SeqCst); + } + if keys.len() > 1 { + overlapped.store(true, Ordering::SeqCst); + } + drop(keys); + entered_tx.send(()).expect("record entry"); + release_rx.recv().expect("release handler"); + active.lock().remove(&key); + } + }, + |_| {}, + ) + .expect("create executor"); + + // Act + executor.try_enqueue(RouteFamily::new(1), 1, ()).unwrap(); + executor.try_enqueue(RouteFamily::new(1), 1, ()).unwrap(); + executor.try_enqueue(RouteFamily::new(1), 2, ()).unwrap(); + entered_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + entered_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + + // Assert + assert!(overlapped.load(Ordering::SeqCst)); + assert!(!same_key_overlap.load(Ordering::SeqCst)); + release_tx.send(()).unwrap(); + release_tx.send(()).unwrap(); + entered_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + release_tx.send(()).unwrap(); + executor.join(); + } + + #[test] + fn should_preserve_same_key_fifo() { + // Arrange + let seen = Arc::new(Mutex::new(Vec::new())); + let (done_tx, done_rx) = crossbeam_channel::bounded(4); + let executor = KeyedFamilyExecutor::new( + &[RouteFamily::new(1)], + 4, + |_| (), + { + let seen = seen.clone(); + move |(), _, _, _: Option<&u64>, message| { + seen.lock().push(message); + done_tx.send(()).unwrap(); + } + }, + |_| {}, + ) + .unwrap(); + + // Act + for message in 0..4 { + executor + .try_enqueue(RouteFamily::new(1), 7, message) + .unwrap(); + } + for _ in 0..4 { + done_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + } + + // Assert + assert_eq!(*seen.lock(), vec![0, 1, 2, 3]); + executor.join(); + } + + #[test] + fn should_prioritize_exclusive_control_over_queued_normal_work() { + // Arrange + let seen = Arc::new(Mutex::new(Vec::new())); + let (first_entered_tx, first_entered_rx) = crossbeam_channel::bounded(1); + let (release_tx, release_rx) = crossbeam_channel::bounded(1); + let (done_tx, done_rx) = crossbeam_channel::bounded(3); + let executor = KeyedFamilyExecutor::new( + &[RouteFamily::new(1)], + 2, + |_| (), + { + let seen = seen.clone(); + move |(), _, lane, _, message| { + if message == 1 { + first_entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + } + seen.lock().push((lane, message)); + done_tx.send(()).unwrap(); + } + }, + |_| {}, + ) + .unwrap(); + executor.try_enqueue(RouteFamily::new(1), 1, 1).unwrap(); + first_entered_rx + .recv_timeout(Duration::from_secs(1)) + .unwrap(); + + // Act: enqueue the control message *before* the sibling-key normal + // message. With 2 shards and only key 1 active, a free shard is + // otherwise entitled to grab a ready sibling key immediately -- + // dispatch only skips a family once control is non-empty and a key + // is still active (see `take_work`). Enqueuing normal-then-control + // leaves a real window where control is still empty when the free + // shard looks for work, so it can race ahead: an actual production + // race, not a bug, but not what this test means to exercise. + executor + .try_enqueue_control(RouteFamily::new(1), 3) + .unwrap(); + executor.try_enqueue(RouteFamily::new(1), 2, 2).unwrap(); + release_tx.send(()).unwrap(); + for _ in 0..3 { + done_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + } + + // Assert + assert_eq!( + *seen.lock(), + vec![ + (FamilyActorLane::Normal, 1), + (FamilyActorLane::Control, 3), + (FamilyActorLane::Normal, 2) + ] + ); + executor.join(); + } + + #[test] + fn should_fail_only_panicking_family_and_keep_sibling_progressing() { + // Arrange + let failures = Arc::new(AtomicUsize::new(0)); + let (done_tx, done_rx) = crossbeam_channel::bounded(1); + let executor = KeyedFamilyExecutor::new( + &[RouteFamily::new(1), RouteFamily::new(2)], + 2, + |_| (), + move |(), family, _, _, message| { + assert_ne!(family, RouteFamily::new(1), "injected panic"); + done_tx.send(message).unwrap(); + }, + { + let failures = failures.clone(); + move |_| { + failures.fetch_add(1, Ordering::SeqCst); + } + }, + ) + .unwrap(); + + // Act + executor.try_enqueue(RouteFamily::new(1), 1, 1).unwrap(); + executor.try_enqueue(RouteFamily::new(2), 1, 2).unwrap(); + + // Assert + assert_eq!(done_rx.recv_timeout(Duration::from_secs(1)).unwrap(), 2); + let deadline = std::time::Instant::now() + Duration::from_secs(1); + while executor.is_family_running(RouteFamily::new(1)) + && std::time::Instant::now() < deadline + { + std::thread::yield_now(); + } + assert!(!executor.is_family_running(RouteFamily::new(1))); + assert!(executor.is_family_running(RouteFamily::new(2))); + assert_eq!(failures.load(Ordering::SeqCst), 1); + assert_eq!( + executor.try_enqueue(RouteFamily::new(1), 1, 3), + Err(FamilyActorEnqueueError::ActorStopped) + ); + executor.join(); + } + + #[test] + fn should_rotate_ready_keys_fairly() { + // Arrange + let seen = Arc::new(Mutex::new(Vec::new())); + let (done_tx, done_rx) = crossbeam_channel::bounded(4); + let executor = KeyedFamilyExecutor::new( + &[RouteFamily::new(1)], + 1, + |_| (), + { + let seen = seen.clone(); + move |(), _, _, key: Option<&u64>, ()| { + seen.lock().push(*key.unwrap()); + done_tx.send(()).unwrap(); + } + }, + |_| {}, + ) + .unwrap(); + + // Act + executor.try_enqueue(RouteFamily::new(1), 1, ()).unwrap(); + executor.try_enqueue(RouteFamily::new(1), 1, ()).unwrap(); + executor.try_enqueue(RouteFamily::new(1), 1, ()).unwrap(); + executor.try_enqueue(RouteFamily::new(1), 2, ()).unwrap(); + for _ in 0..4 { + done_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + } + + // Assert + assert_eq!(*seen.lock(), vec![1, 2, 1, 1]); + executor.join(); + } + + #[test] + fn should_enforce_both_lane_capacities() { + // Arrange + let (entered_tx, entered_rx) = crossbeam_channel::bounded(1); + let (release_tx, release_rx) = crossbeam_channel::bounded(1); + let executor = KeyedFamilyExecutor::new( + &[RouteFamily::new(1)], + 1, + |_| (), + move |(), _, _, _, message: usize| { + if message == usize::MAX { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + } + }, + |_| {}, + ) + .unwrap(); + executor + .try_enqueue(RouteFamily::new(1), 0, usize::MAX) + .unwrap(); + entered_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + + // Act + for message in 0..FAMILY_ACTOR_NORMAL_LANE_CAPACITY { + executor + .try_enqueue(RouteFamily::new(1), message + 1, message) + .unwrap(); + } + for message in 0..FAMILY_ACTOR_CONTROL_LANE_CAPACITY { + executor + .try_enqueue_control(RouteFamily::new(1), message) + .unwrap(); + } + + // Assert + assert_eq!( + executor.try_enqueue(RouteFamily::new(1), 99_999, 1), + Err(FamilyActorEnqueueError::NormalLaneFull) + ); + assert_eq!( + executor.try_enqueue_control(RouteFamily::new(1), 1), + Err(FamilyActorEnqueueError::ControlLaneFull) + ); + release_tx.send(()).unwrap(); + executor.join(); + } + + #[test] + fn should_reject_work_after_stop_and_join_workers() { + // Arrange + let executor = KeyedFamilyExecutor::new( + &[RouteFamily::new(1)], + 2, + |_| (), + |(), _, _, _: Option<&u64>, ()| {}, + |_| {}, + ) + .unwrap(); + + // Act + executor.stop(); + executor.join(); + + // Assert + assert!(!executor.is_running()); + assert_eq!( + executor.try_enqueue(RouteFamily::new(1), 1, ()), + Err(FamilyActorEnqueueError::ActorStopped) + ); + assert_eq!( + executor.try_enqueue_control(RouteFamily::new(1), ()), + Err(FamilyActorEnqueueError::ActorStopped) + ); + } +} diff --git a/src/runtime/mod.rs b/src/runtime/mod.rs index 59518aff..22c2e51a 100644 --- a/src/runtime/mod.rs +++ b/src/runtime/mod.rs @@ -22,6 +22,7 @@ pub mod actor; pub mod cf_validation; +pub mod cleanup_guard; pub mod client_frame; pub mod clock; pub mod context; @@ -29,10 +30,13 @@ pub mod domain_event; pub mod domain_manifest; pub mod envelope; pub mod family_actor_pool; +pub(crate) mod ingress_support; pub mod keyed_actor_pool; +pub(crate) mod keyed_family_executor; pub mod mailbox; pub mod managed_actor; pub mod matcher; +pub mod reply_wait; pub mod router; pub mod routing; /// Actor-spawning fixture used only by unit tests; production uses family @@ -44,6 +48,7 @@ pub mod supervision; // Re-export commonly used types pub use actor::{Actor, ActorError, ActorId, ActorRef, ActorState, Context, SendError}; +pub use cleanup_guard::CleanedUpSessions; pub use client_frame::{ClientChannel, ClientFrameMeta, EncodedClientFrame}; pub use clock::{ epoch_ms_to_instant_with_reference, instant_to_epoch_ms_with_reference, Clock, SystemClock, diff --git a/src/runtime/reply_wait.rs b/src/runtime/reply_wait.rs new file mode 100644 index 00000000..58656586 --- /dev/null +++ b/src/runtime/reply_wait.rs @@ -0,0 +1,60 @@ +use crate::runtime::DeliveryError; + +/// Map a reply-channel wait failure to the `DeliveryError` that actually +/// describes it, instead of collapsing every failure into `ActorStopped`. +/// +/// Shared by every domain that waits synchronously on an actor reply (queue, +/// stream, rpc) so the distinction cannot drift back apart: collapsing a +/// timeout into `ActorStopped` reports a busy actor as a dead one, and the +/// ingress layer treats those very differently. +/// +/// The message was already accepted into the actor's mailbox by this point +/// (enqueue succeeded), so a wait failure here means one of two distinct +/// things: +/// - `Timeout`: the actor is still alive but did not reply before the +/// deadline (e.g. busy with other work) - busy, not "dead". +/// +/// This says nothing about whether the request may be retried. The command +/// was already accepted into the mailbox and may still execute, so callers +/// must treat the outcome as unknown. Only enqueue-time failures +/// (`MailboxFull`/`HighLaneFull`) mean the command was never accepted and +/// are therefore safe to retry. +/// - `Disconnected`: the reply sender was dropped without ever sending, +/// which only happens if the actor stopped (e.g. panicked) while holding +/// this message - genuinely stopped. +#[must_use] +pub fn map_reply_wait_error(error: crossbeam_channel::RecvTimeoutError) -> DeliveryError { + match error { + crossbeam_channel::RecvTimeoutError::Timeout => DeliveryError::Timeout, + crossbeam_channel::RecvTimeoutError::Disconnected => DeliveryError::ActorStopped, + } +} + +#[cfg(test)] +mod reply_wait_error_tests { + use super::{map_reply_wait_error, DeliveryError}; + + #[test] + fn should_map_reply_wait_timeout_to_timeout_not_actor_stopped() { + // Arrange + let error = crossbeam_channel::RecvTimeoutError::Timeout; + + // Act + let delivery_error = map_reply_wait_error(error); + + // Assert + assert!(matches!(delivery_error, DeliveryError::Timeout)); + } + + #[test] + fn should_map_reply_wait_disconnect_to_actor_stopped() { + // Arrange + let error = crossbeam_channel::RecvTimeoutError::Disconnected; + + // Act + let delivery_error = map_reply_wait_error(error); + + // Assert + assert!(matches!(delivery_error, DeliveryError::ActorStopped)); + } +} diff --git a/src/runtime/router.rs b/src/runtime/router.rs index 92ab9e47..f0193cc4 100644 --- a/src/runtime/router.rs +++ b/src/runtime/router.rs @@ -105,6 +105,13 @@ pub enum DeliveryError { Timeout, /// A sink panicked while accepting an envelope. SinkPanicked, + /// The response could not be framed for the wire. + /// + /// Permanent for this payload, and deliberately not saturation: a TLV + /// value carries a `u16` length, so an oversized response can never be + /// sent no matter how long the transport is given. Retrying it wastes + /// work; the fix is always to paginate at the source. + InvalidPayload { len: usize, max: usize }, } impl DeliveryError { @@ -123,6 +130,10 @@ impl DeliveryError { DeliveryError::ActorStopped | DeliveryError::Timeout | DeliveryError::SinkPanicked => { 1.0 } + // Not a saturation signal - the destination has room; the payload + // is simply unframable. Reporting 1.0 here would drive backoff + // against a condition that waiting cannot fix. + DeliveryError::InvalidPayload { .. } => 0.0, } } } @@ -148,6 +159,9 @@ impl std::fmt::Display for DeliveryError { DeliveryError::ActorStopped => write!(f, "Actor has stopped"), DeliveryError::Timeout => write!(f, "Delivery timed out"), DeliveryError::SinkPanicked => write!(f, "Sink panicked during delivery"), + DeliveryError::InvalidPayload { len, max } => { + write!(f, "Response payload {len} bytes exceeds wire limit {max}") + } } } } @@ -306,7 +320,8 @@ impl Router { } DeliveryError::ActorStopped | DeliveryError::Timeout - | DeliveryError::SinkPanicked => {} + | DeliveryError::SinkPanicked + | DeliveryError::InvalidPayload { .. } => {} } } } @@ -553,10 +568,22 @@ impl Router { pub(crate) fn route_high_priority(&self, envelope: Envelope) -> Result<(), RouteError> { let dest = envelope.destination().clone(); + // Mirror `route()`'s exact-then-domain-pattern fallback. Every + // production domain sink registers via `register_domain_pattern` + // (see `domain_manifest.rs`), never an exact address, so an + // exact-only lookup here would make `route_high_priority` unusable + // for reaching a real domain sink - which is exactly the class of + // control-plane traffic (e.g. session cleanup dispatch) this method + // exists for. + let route_str = dest.route().as_str(); + let extracted_domain = extract_domain(route_str); + let fallback_domain = extracted_domain.unwrap_or(""); + let domain = extracted_domain.unwrap_or("unknown"); let sink = self - .registry - .get(&dest) - .ok_or_else(|| RouteError::RouteNotFound(dest.clone()))?; + .resolve_sink_for_route(&dest, fallback_domain) + .ok_or_else(|| { + Self::route_not_found(&dest, domain, MissingRouteKind::ExactOrDomainPattern) + })?; match Self::catch_sink_panic(|| sink.deliver_high_priority(envelope)) { Ok(()) => Ok(()), diff --git a/src/runtime/router/tests.rs b/src/runtime/router/tests.rs index d39f4708..8d4ed7f3 100644 --- a/src/runtime/router/tests.rs +++ b/src/runtime/router/tests.rs @@ -116,6 +116,30 @@ fn should_route_to_domain_pattern_across_families() { assert_eq!(sink.delivered.lock().len(), 2); } +#[test] +fn should_route_high_priority_to_domain_pattern_across_families() { + // Arrange + // Every domain sink in production registers via `register_domain_pattern` + // (see `domain_manifest.rs`), never an exact address. Control-plane + // callers such as session cleanup dispatch use `route_high_priority` to + // reach that sink and must not require an exact registration that + // production never creates. + let router = Router::new(); + let sink = Arc::new(MockSink::new()); + router.register_domain_pattern("queue", sink.clone()); + + // Act + let address = test_address(7, "queue://cleanup"); + let result = router.route_high_priority(Envelope::new(address, "cleanup")); + + // Assert + assert!( + result.is_ok(), + "expected domain-pattern fallback, got {result:?}" + ); + assert_eq!(sink.delivered.lock().len(), 1); +} + #[test] fn should_prefer_exact_match_over_domain_pattern() { // Arrange diff --git a/src/testkit/transport/server.rs b/src/testkit/transport/server.rs index 3afc7466..eaae7353 100644 --- a/src/testkit/transport/server.rs +++ b/src/testkit/transport/server.rs @@ -299,8 +299,11 @@ impl TestServer { .map_err(|e| Box::new(e) as Box)?; // Initialize observability (metrics + tracing) once for tests - // Safe to call multiple times - will only initialize once - let _ = crate::observability::try_init_observability(); + // Safe to call multiple times - will only initialize once. Quiets + // known-noisy dependency logs (e.g. cntryl_midge's routine + // per-engine "primary lease acquired" WARN) that are expected on + // every ephemeral test engine and otherwise flood test/CI output. + let _ = crate::observability::global::try_init_test_observability(); if auth_required { init_test_runtime_jwks_cache(); diff --git a/src/testkit/transport/tests.rs b/src/testkit/transport/tests.rs index 7302e0b1..71012437 100644 --- a/src/testkit/transport/tests.rs +++ b/src/testkit/transport/tests.rs @@ -124,6 +124,47 @@ async fn should_cleanup_all_session_state_given_abrupt_websocket_disconnect() { } } +#[tokio::test(start_paused = true)] +async fn should_keep_authenticated_websocket_open_beyond_connect_deadline() { + // Arrange + let server = TestServer::start().await.expect("start test server"); + let mut websocket = server.connect_ws().await.expect("connect websocket client"); + let connect_frame = build_connect_frame("test-realm", &generate_test_jwt("test-realm")); + websocket + .send_frame(&connect_frame) + .await + .expect("send CONNECT frame"); + server + .wait_for_authenticated_sessions(1) + .await + .expect("wait for authenticated session"); + tokio::time::advance( + crate::api::ingress::CONNECT_DEADLINE + std::time::Duration::from_millis(1), + ) + .await; + let route = "kv://test-realm/app/connect-deadline"; + let mut payload = Vec::new(); + payload.extend_from_slice( + &u32::try_from(route.len()) + .expect("route length fits u32") + .to_be_bytes(), + ); + payload.extend_from_slice(route.as_bytes()); + payload.push(1); + payload.push(0); + let mut builder = TlvFrameBuilder::new(); + builder.encode_field(100, &payload); + + // Act + let response = websocket.request(&builder.build(), 2_000).await; + + // Assert + assert!( + response.is_ok(), + "authenticated websocket closed after CONNECT: {response:?}" + ); +} + #[tokio::test] async fn should_accept_websocket_upgrade_given_allowed_origin() { // Arrange diff --git a/src/utils/storage_key.rs b/src/utils/storage_key.rs index 2ca81470..0d6a9eb3 100644 --- a/src/utils/storage_key.rs +++ b/src/utils/storage_key.rs @@ -47,9 +47,23 @@ pub fn realm_domain_prefix(realm: &str, domain: &str) -> Vec { encoder.into_vec() } +/// Exclusive upper bound covering every key that begins with `prefix`. +/// +/// Uses lexkey's `prefix_successor` rather than `encode_range_upper`. The +/// latter yields `prefix || 0xff`, which lexkey documents as correct only when +/// the bytes following the prefix are themselves lexkey-encoded - UTF-8 strings +/// and fixed-width numbers can never reach `0xff`. Several callers append raw, +/// unencoded client bytes instead, and a key beginning with `0xff` then sorts +/// past that bound: the write succeeds and the key becomes invisible to every +/// scan of its own resource. +/// +/// `prefix_successor` returns `None` only for an empty or all-`0xff` prefix. +/// Every prefix built here ends with a separator, so that cannot occur; the +/// fallback keeps the old bound rather than silently scanning unbounded. #[must_use] pub fn prefix_range_end(prefix: &[u8]) -> Vec { - LexKey::encode_range_upper(prefix, None).as_bytes().to_vec() + LexKey::prefix_successor(prefix) + .unwrap_or_else(|| LexKey::encode_range_upper(prefix, None).as_bytes().to_vec()) } #[must_use] @@ -192,6 +206,24 @@ mod tests { // Assert assert!(range_end.as_slice() > prefix.as_slice()); assert!(b"acme\0kv\0users\0x".as_slice() < range_end.as_slice()); + // The bound must cover EVERY suffix, not just printable ones. Callers + // append raw client bytes, so a suffix may begin with 0xff - the byte + // lexkey uses as its range end marker. Only testing ordinary suffixes + // is how a whole class of keys became invisible to scans. + for suffix in [ + [0x00u8].as_slice(), + [0x7f].as_slice(), + [0xfe, 0xfe].as_slice(), + [0xff].as_slice(), + [0xff, 0xff, 0xff].as_slice(), + ] { + let mut key = prefix.to_vec(); + key.extend_from_slice(suffix); + assert!( + key.as_slice() < range_end.as_slice(), + "suffix {suffix:?} sorts outside its own prefix range" + ); + } } #[test] diff --git a/tests/benchmark_contract.rs b/tests/benchmark_contract.rs deleted file mode 100644 index 29c86686..00000000 --- a/tests/benchmark_contract.rs +++ /dev/null @@ -1,264 +0,0 @@ -use std::collections::{BTreeMap, BTreeSet}; -use std::fs; -use std::path::{Path, PathBuf}; - -use serde_json::Value; - -#[test] -fn should_keep_perf_targets_resolvable_to_benchmark_baseline_rows() { - // Arrange - let repo_root = repo_root(); - let baseline_ids = baseline_record_ids(&repo_root); - let target_entries = perf_target_entries(&repo_root); - - // Act - let missing_targets = target_entries - .iter() - .filter(|(_, benchmark_id)| !baseline_ids.contains(*benchmark_id)) - .map(|(key, benchmark_id)| format!("{key}: {benchmark_id}")) - .collect::>(); - - // Assert - assert!( - missing_targets.is_empty(), - "perf target benchmark_id entries missing from config/bench_baseline.json:\n{}", - missing_targets.join("\n") - ); -} - -#[test] -fn should_keep_release_benchmark_ids_resolvable_to_baseline_rows() { - // Arrange - let repo_root = repo_root(); - let baseline_ids = baseline_record_ids(&repo_root); - let release_ids = release_benchmark_ids(&repo_root); - - // Act - let missing_release_ids = release_ids - .iter() - .filter(|benchmark_id| !baseline_ids.contains(*benchmark_id)) - .cloned() - .collect::>(); - - // Assert - assert!( - missing_release_ids.is_empty(), - "config/bench_release_ids.txt entries missing from config/bench_baseline.json:\n{}", - missing_release_ids.join("\n") - ); -} - -#[test] -fn should_keep_exactly_fourteen_primary_tier4_rows_in_the_release_manifest() { - // Arrange - let repo_root = repo_root(); - let release_ids = release_benchmark_ids(&repo_root); - - // Act - let invalid_rows = release_ids - .iter() - .filter(|id| !id.contains("|throughput_ops_per_s|") || id.contains("_latency")) - .collect::>(); - - // Assert - assert_eq!(release_ids.len(), 14, "release manifest row count"); - assert!( - invalid_rows.is_empty(), - "release manifest must contain primary throughput rows only:\n{}", - invalid_rows - .iter() - .map(|id| id.as_str()) - .collect::>() - .join("\n") - ); -} - -#[test] -fn should_keep_latency_records_out_of_perf_targets() { - // Arrange - let repo_root = repo_root(); - let targets = perf_target_entries(&repo_root); - - // Act - let latency_targets = targets - .into_iter() - .filter(|(_, benchmark_id)| benchmark_id.contains("_latency")) - .map(|(key, benchmark_id)| format!("{key}: {benchmark_id}")) - .collect::>(); - - // Assert - assert!( - latency_targets.is_empty(), - "latency records are report-only and cannot become perf targets:\n{}", - latency_targets.join("\n") - ); -} - -#[test] -fn should_omit_unsupported_direct_lease_rows_from_the_transport_gate() { - // Arrange - let repo_root = repo_root(); - let targets = perf_targets(&repo_root); - - // Act - let direct_gate_rows = targets - .iter() - .filter(|(key, _)| key.starts_with("stress:tier4-lease-gate|")) - .filter(|(_, target)| target_str(target, "layer") == "direct") - .collect::>(); - - // Assert - assert!( - direct_gate_rows.is_empty(), - "Lease transport gates must not claim an unsupported direct transport row" - ); -} - -#[test] -fn should_label_tier4_rpc_targets_with_completion_semantics() { - // Arrange - let repo_root = repo_root(); - let targets = perf_targets(&repo_root); - - // Act - let missing_labels = targets - .iter() - .filter(|(key, _)| key.starts_with("stress:tier4-rpc-")) - .filter_map(|(key, target)| { - let missing = missing_fields( - target, - &[ - "completion_mode", - "completed_unit", - "inflight_per_client", - "worker_count", - ], - ); - (!missing.is_empty()).then(|| format!("{key}: {}", missing.join(", "))) - }) - .collect::>(); - - // Assert - assert!( - missing_labels.is_empty(), - "tier RPC perf targets must label completion semantics:\n{}", - missing_labels.join("\n") - ); -} - -#[test] -fn should_label_tier4_notice_targets_with_completion_semantics() { - // Arrange - let repo_root = repo_root(); - let targets = perf_targets(&repo_root); - - // Act - let missing_labels = targets - .iter() - .filter(|(key, _)| key.starts_with("stress:tier4-notice-")) - .filter_map(|(key, target)| { - let missing = missing_fields( - target, - &[ - "completion_mode", - "completed_unit", - "publisher_count", - "subscriber_count", - ], - ); - (!missing.is_empty()).then(|| format!("{key}: {}", missing.join(", "))) - }) - .collect::>(); - - // Assert - assert!( - missing_labels.is_empty(), - "tier Notice perf targets must label completion semantics:\n{}", - missing_labels.join("\n") - ); -} - -#[test] -fn should_keep_unstable_rpc_pipelined_tcp_row_out_of_release_ids() { - // Arrange - let repo_root = repo_root(); - let release_ids = release_benchmark_ids(&repo_root); - - // Act - let promoted = release_ids.iter().any(|id| { - id.contains("tier4-rpc-pipeline") - || id.contains("should_characterize_tcp_32_inflight_pipeline") - }); - - // Assert - assert!( - !promoted, - "tcp multiclient pipelined RPC row is variance-gated and must not be release-gated" - ); -} - -fn repo_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) -} - -fn baseline_record_ids(repo_root: &Path) -> BTreeSet { - let baseline = read_json(repo_root, "config/bench_baseline.json"); - baseline - .get("records") - .and_then(Value::as_array) - .expect("bench baseline records array") - .iter() - .map(|record| target_str(record, "id").to_owned()) - .collect() -} - -fn perf_target_entries(repo_root: &Path) -> BTreeMap { - perf_targets(repo_root) - .iter() - .map(|(key, target)| (key.clone(), target_str(target, "benchmark_id").to_owned())) - .collect() -} - -fn perf_targets(repo_root: &Path) -> BTreeMap { - let targets = read_json(repo_root, "config/perf_targets.json"); - targets - .get("targets") - .and_then(Value::as_object) - .expect("perf targets object") - .iter() - .map(|(key, value)| (key.clone(), value.clone())) - .collect() -} - -fn release_benchmark_ids(repo_root: &Path) -> Vec { - fs::read_to_string(repo_root.join("config/bench_release_ids.txt")) - .expect("read config/bench_release_ids.txt") - .lines() - .map(str::trim) - .filter(|line| !line.is_empty() && !line.starts_with('#')) - .map(ToOwned::to_owned) - .collect() -} - -fn read_json(repo_root: &Path, relative_path: &str) -> Value { - let path = repo_root.join(relative_path); - let content = fs::read_to_string(&path) - .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())); - serde_json::from_str(&content) - .unwrap_or_else(|error| panic!("failed to parse {}: {error}", path.display())) -} - -fn target_str<'a>(value: &'a Value, field: &str) -> &'a str { - value - .get(field) - .and_then(Value::as_str) - .unwrap_or_else(|| panic!("missing string field {field} in {value}")) -} - -fn missing_fields(value: &Value, fields: &[&str]) -> Vec { - fields - .iter() - .filter(|field| value.get(**field).is_none()) - .map(|field| (*field).to_string()) - .collect() -} diff --git a/tests/dependency_drift_workflow.rs b/tests/dependency_drift_workflow.rs deleted file mode 100644 index 6c74f645..00000000 --- a/tests/dependency_drift_workflow.rs +++ /dev/null @@ -1,58 +0,0 @@ -const WORKFLOW: &str = include_str!("../.github/workflows/dependency-drift.yml"); -const CARGO_MANIFEST: &str = include_str!("../Cargo.toml"); - -#[test] -fn should_check_each_git_main_dependency_on_a_weekly_schedule() { - // Arrange - let dependency_names = ["cntryl-lexkey", "cntryl-midge", "cntryl-stress"]; - - // Act - let missing = dependency_names - .into_iter() - .filter(|name| !WORKFLOW.contains(name)) - .collect::>(); - - // Assert - assert!(WORKFLOW.contains("cron:")); - assert!(missing.is_empty(), "missing dependency checks: {missing:?}"); -} - -#[test] -fn should_track_internal_git_dependencies_on_main_until_published_releases_exist() { - // Arrange - let expected_dependencies = [ - r#"cntryl-lexkey = { git = "https://github.com/cntryl/lexkey-rs", branch = "main" }"#, - r#"cntryl-midge = { git = "https://github.com/cntryl/midge", branch = "main" }"#, - r#"cntryl-stress = { git = "https://github.com/cntryl/stress", branch = "main" }"#, - ]; - - // Act - let missing = expected_dependencies - .into_iter() - .filter(|dependency| !CARGO_MANIFEST.contains(dependency)) - .collect::>(); - - // Assert - assert!( - missing.is_empty(), - "dependencies not tracking main: {missing:?}" - ); -} - -#[test] -fn should_have_permission_plus_logic_to_report_drift_once() { - // Arrange - let required_contract = ["issues: write", "ahead_by", "total_count === 0"]; - - // Act - let missing = required_contract - .into_iter() - .filter(|fragment| !WORKFLOW.contains(fragment)) - .collect::>(); - - // Assert - assert!( - missing.is_empty(), - "missing drift-reporting contract: {missing:?}" - ); -} diff --git a/tests/kv_basics.rs b/tests/kv_basics.rs deleted file mode 100644 index 78dffac0..00000000 --- a/tests/kv_basics.rs +++ /dev/null @@ -1,529 +0,0 @@ -//! KV basics consolidation — authorization, session, realm isolation, permission pipeline -//! -//! Consolidated from: `kv_auth.rs`, `kv_realm_isolation.rs`, `kv_session_permissions.rs`, -//! and `permission_check_pipeline.rs`. - -use bytes::Bytes; -use fitz::auth::{Access, Claims, Permission}; -use fitz::domains::kv::{ - KvActor, KvMessage, KvResourceScope, KvResponse, SessionActor as KvSessionActor, TxMode, -}; -use fitz::runtime::routing::{Route, RouteFamily}; -use fitz::session::actor::SessionActor as SessionActorLayer2; -use fitz::session::permissions::SessionPermissions; -use fitz::session::session::SessionId; -use fitz::testkit::create_test_engine_with_cfs; - -fn create_kv_actor() -> KvActor { - let store = create_test_engine_with_cfs(vec![1, 2, 3, 4, 5]); - KvActor::new(store) -} - -// --------------------------------------------------------------------------- -// KV domain + session-level authorization tests (migrated) -// --------------------------------------------------------------------------- - -#[test] -fn should_reject_unauthorized_realm_access() { - // Arrange - let permissions = vec![ - Permission::parse("kv://realm1/**#read").unwrap(), - Permission::parse("kv://realm1/**#write").unwrap(), - ]; - let session_perms = SessionPermissions::from_permissions(permissions); - let session_actor = KvSessionActor::new(SessionId(1), session_perms); - - let mut kv_actor = create_kv_actor(); - - // Act - let msg = KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "realm2".to_string(), - "kv".to_string(), - "users".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - let result = session_actor.begin(msg, &mut kv_actor); - - // Assert - assert!(result.is_err(), "Should reject unauthorized realm"); - assert!( - result.unwrap_err().contains("unauthorized"), - "Error should mention unauthorized" - ); -} - -#[test] -fn should_allow_authorized_realm_access() { - // Arrange - let permissions = vec![ - Permission::parse("kv://mycompany/**#read").unwrap(), - Permission::parse("kv://mycompany/**#write").unwrap(), - ]; - let session_perms = SessionPermissions::from_permissions(permissions); - let session_actor = KvSessionActor::new(SessionId(1), session_perms); - - let mut kv_actor = create_kv_actor(); - - // Act - let msg = KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "mycompany".to_string(), - "kv".to_string(), - "users".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - let result = session_actor.begin(msg, &mut kv_actor); - - // Assert - assert!(result.is_ok(), "Should allow authorized realm"); -} - -#[test] -fn should_enforce_realm_equality_strictly_in_session() { - // Arrange - let permissions = vec![Permission::parse("kv://acme/**#write").unwrap()]; - let session_perms = SessionPermissions::from_permissions(permissions); - let session_actor = KvSessionActor::new(SessionId(1), session_perms); - - let mut kv_actor = create_kv_actor(); - - // Act - let invalid_realms = vec![ - "ACME", // Case-sensitive: different case - "acme-2", // Different realm (similar name) - "xacme", // Prefix doesn't match - ]; - - for invalid_realm in invalid_realms { - let msg = KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - invalid_realm.to_string(), - "kv".to_string(), - "data".to_string(), - ), - mode: TxMode::ReadOnly, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - let result = session_actor.begin(msg, &mut kv_actor); - - // Assert - assert!( - result.is_err(), - "Realm '{invalid_realm}' should not match 'acme'" - ); - } -} - -#[test] -fn should_accept_valid_message_type() { - // Arrange - let permissions = vec![Permission::parse("kv://tenant/**#write").unwrap()]; - let session_perms = SessionPermissions::from_permissions(permissions); - let session_actor = KvSessionActor::new(SessionId(1), session_perms); - - let mut kv_actor = create_kv_actor(); - - // Act - let msg = KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "tenant".to_string(), - "kv".to_string(), - "table".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - let result = session_actor.begin(msg, &mut kv_actor); - - // Assert - assert!(result.is_ok()); -} - -#[test] -fn should_reject_invalid_message_type() { - // Arrange - let permissions = vec![Permission::parse("kv://tenant/**#write").unwrap()]; - let session_perms = SessionPermissions::from_permissions(permissions); - let session_actor = KvSessionActor::new(SessionId(1), session_perms); - - let mut kv_actor = create_kv_actor(); - - // Act - // Get a tx_id first (we need one to create a Commit message) - let begin_msg = KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "test".to_string(), - "kv".to_string(), - "data".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - let begin_result = kv_actor.handle(begin_msg.clone()); - let KvResponse::BeginOk { tx_id } = begin_result else { - panic!("Expected BeginOk"); - }; - - let msg = KvMessage::Commit { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "test", "kv", "data"), - }; // Not a Begin message - - let result = session_actor.begin(msg, &mut kv_actor); - - // Assert - assert!(result.is_err()); - assert!(result.unwrap_err().contains("invalid message type")); -} - -#[test] -fn should_allow_subsequent_operations_after_begin() { - // Arrange - let permissions = vec![Permission::parse("kv://authed/**#write").unwrap()]; - let session_perms = SessionPermissions::from_permissions(permissions); - let session_actor = KvSessionActor::new(SessionId(1), session_perms); - - let mut kv_actor = create_kv_actor(); - - // Act - let begin_msg = KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "authed".to_string(), - "kv".to_string(), - "data".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - let result = session_actor.begin(begin_msg, &mut kv_actor); - assert!(result.is_ok()); - - // Extract tx_id from successful Begin - let fitz::domains::kv::KvResponse::BeginOk { tx_id } = kv_actor.handle(KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "authed".to_string(), - "kv".to_string(), - "data".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }) else { - panic!("Expected BeginOk"); - }; - - // Continue: Subsequent Put operation (realm already validated) - let put_msg = KvMessage::Put { - tx_id, - scope: KvResourceScope::new(RouteFamily::new(1), "authed", "kv", "data"), - key: Bytes::from_static(b"key1"), - value: Bytes::from_static(b"value1"), - }; - - let result = session_actor.operation(&mut kv_actor, put_msg); - - // Assert - assert!(matches!(result, KvResponse::PutOk)); -} - -#[test] -fn should_allow_read_permission_for_transactions() { - // Arrange - let permissions = vec![Permission::parse("kv://analytics/**#read").unwrap()]; - let session_perms = SessionPermissions::from_permissions(permissions); - let session_actor = KvSessionActor::new(SessionId(1), session_perms); - - let mut kv_actor = create_kv_actor(); - - // Act - let msg = KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "analytics".to_string(), - "kv".to_string(), - "reports".to_string(), - ), - mode: TxMode::ReadOnly, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - let result = session_actor.begin(msg, &mut kv_actor); - - // Assert - assert!(result.is_ok()); -} - -#[test] -fn should_enforce_realm_isolation_across_sessions() { - // Arrange - let perms1 = vec![Permission::parse("kv://acme/**#write").unwrap()]; - let perms2 = vec![Permission::parse("kv://evil/**#write").unwrap()]; - - let session1 = KvSessionActor::new(SessionId(1), SessionPermissions::from_permissions(perms1)); - let session2 = KvSessionActor::new(SessionId(2), SessionPermissions::from_permissions(perms2)); - - let mut actor1 = create_kv_actor(); - let mut actor2 = create_kv_actor(); - - // Act - let msg1 = KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "acme".to_string(), - "kv".to_string(), - "secrets".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - let result1 = session1.begin(msg1, &mut actor1); - assert!(result1.is_ok(), "Session1 should access acme"); - - // Continue: Session2 tries to access acme realm (not authorized) - let msg2 = KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "acme".to_string(), - "kv".to_string(), - "secrets".to_string(), - ), - mode: TxMode::ReadOnly, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - let result2 = session2.begin(msg2, &mut actor2); - - // Assert - assert!( - result2.is_err(), - "Session2 should not access realm outside its permissions" - ); -} - -// --------------------------------------------------------------------------- -// Session-permissions & pipeline tests (migrated) -// --------------------------------------------------------------------------- - -#[test] -fn should_reject_read_only_session_begin_read_write_permission() { - // Arrange - let p = Permission::parse("kv://acme#read").unwrap(); - let perms = SessionPermissions::from_permissions(vec![p]); - let actor = KvSessionActor::new(SessionId(1), perms.clone()); - let mut kv = KvActor::new(create_test_engine_with_cfs(vec![1])); - let msg = KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "acme".to_string(), - "kv".to_string(), - "table1".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - // Act - let res = actor.begin(msg, &mut kv); - - // Assert - assert!(res.is_err()); - assert!(res.unwrap_err().contains("unauthorized")); -} - -#[test] -fn should_allow_read_only_session_begin_read_only_permission() { - // Arrange - let p = Permission::parse("kv://acme#read").unwrap(); - let perms = SessionPermissions::from_permissions(vec![p]); - let actor = KvSessionActor::new(SessionId(1), perms.clone()); - let mut kv = KvActor::new(create_test_engine_with_cfs(vec![1])); - let msg = KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "acme".to_string(), - "kv".to_string(), - "table1".to_string(), - ), - mode: TxMode::ReadOnly, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - // Act - let res = actor.begin(msg, &mut kv); - - // Assert - assert!(res.is_ok()); -} - -#[test] -fn should_allow_write_session_begin_read_write_permission() { - // Arrange - let p = Permission::parse("kv://acme#write").unwrap(); - let perms = SessionPermissions::from_permissions(vec![p]); - let actor = KvSessionActor::new(SessionId(1), perms.clone()); - let mut kv = KvActor::new(create_test_engine_with_cfs(vec![1])); - let msg = KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "acme".to_string(), - "kv".to_string(), - "table1".to_string(), - ), - mode: TxMode::ReadWrite, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - // Act - let res = actor.begin(msg, &mut kv); - - // Assert - assert!(res.is_ok()); -} - -#[test] -fn should_allow_write_session_begin_read_only_permission() { - // Arrange - let p = Permission::parse("kv://acme#write").unwrap(); - let perms = SessionPermissions::from_permissions(vec![p]); - let actor = KvSessionActor::new(SessionId(1), perms.clone()); - let mut kv = KvActor::new(create_test_engine_with_cfs(vec![1])); - let msg = KvMessage::Begin { - scope: KvResourceScope::new( - RouteFamily::new(1), - "acme".to_string(), - "kv".to_string(), - "table1".to_string(), - ), - mode: TxMode::ReadOnly, - write_options: cntryl_midge::WriteOptions::buffered(), - }; - - // Act - let res = actor.begin(msg, &mut kv); - - // Assert - assert!(res.is_ok()); -} - -// Permission pipeline tests (layer-2 behavior) -#[test] -fn should_check_realm_match_first_in_pipeline() { - // Arrange - let perm = Permission::parse("kv://prod/**#write").unwrap(); - let perms = SessionPermissions::from_permissions(vec![perm.clone()]); - - let claims = Claims { - sub: "user:order1".to_string(), - identity_claim: Some("tid".to_string()), - identity_value: Some("prod".to_string()), - permissions: vec![perm], - exp: 9_999_999_999, - }; - - let mut actor = SessionActorLayer2::new(SessionId(1), perms.clone()); - actor.authenticate(claims, perms); - - // Act - let authorized = actor.authorize(&Route::new("kv://staging/users/put"), Access::Write); - - // Assert - assert!( - !authorized, - "realm mismatch should fail permission check pipeline" - ); -} - -#[test] -fn should_check_area_match_after_realm_in_pipeline() { - // Arrange - let perm = Permission::parse("kv://acme/app/**#write").unwrap(); - let perms = SessionPermissions::from_permissions(vec![perm.clone()]); - - let claims = Claims { - sub: "user:order2".to_string(), - identity_claim: Some("tid".to_string()), - identity_value: Some("acme".to_string()), - permissions: vec![perm], - exp: 9_999_999_999, - }; - - let mut actor = SessionActorLayer2::new(SessionId(1), perms.clone()); - actor.authenticate(claims, perms); - - // Act - let authorized = actor.authorize(&Route::new("kv://acme/system/config/put"), Access::Write); - - // Assert - assert!( - !authorized, - "area mismatch should fail even with matching realm" - ); -} - -#[test] -fn should_check_scope_match_after_area_in_pipeline() { - // Arrange - let perm = Permission::parse("kv://acme/app/**#read").unwrap(); - let perms = SessionPermissions::from_permissions(vec![perm.clone()]); - - let claims = Claims { - sub: "user:order3".to_string(), - identity_claim: Some("tid".to_string()), - identity_value: Some("acme".to_string()), - permissions: vec![perm], - exp: 9_999_999_999, - }; - - let mut actor = SessionActorLayer2::new(SessionId(1), perms.clone()); - actor.authenticate(claims, perms); - - // Act - let authorized = actor.authorize(&Route::new("kv://acme/app/users/put"), Access::Write); - - // Assert - assert!( - !authorized, - "scope mismatch should fail even with matching realm and area" - ); -} - -#[test] -fn should_allow_when_all_permission_checks_pass() { - // Arrange - let perm = Permission::parse("kv://acme/app/**#write").unwrap(); - let perms = SessionPermissions::from_permissions(vec![perm.clone()]); - - let claims = Claims { - sub: "user:order4".to_string(), - identity_claim: Some("tid".to_string()), - identity_value: Some("acme".to_string()), - permissions: vec![perm], - exp: 9_999_999_999, - }; - - let mut actor = SessionActorLayer2::new(SessionId(1), perms.clone()); - actor.authenticate(claims, perms); - - // Act - let authorized = actor.authorize(&Route::new("kv://acme/app/users/put"), Access::Write); - - // Assert - assert!(authorized, "all permission checks pass"); -} diff --git a/tests/semantic_boundaries.rs b/tests/semantic_boundaries.rs deleted file mode 100644 index fe8486d0..00000000 --- a/tests/semantic_boundaries.rs +++ /dev/null @@ -1,1480 +0,0 @@ -use std::collections::BTreeSet; -use std::fs; -use std::path::{Path, PathBuf}; - -const DOMAINS: &[&str] = &[ - "kv", "lease", "notice", "queue", "rpc", "schedule", "stream", -]; -const SYNC_CORE_DIRS: &[&str] = &["session", "runtime", "protocol", "domains"]; -const SYNC_CORE_ASYNC_FORBIDDEN: &[&str] = &[ - "async fn", - "async move", - "async {", - ".await", - "tokio::", - "async_trait", - "futures::", - "futures_util", -]; -const SYNC_CORE_TRANSPORT_FORBIDDEN: &[&str] = &[ - "hyper::", - "hyper_util::", - "http_body_util::", - "axum::", - "warp::", - "reqwest::", - "tungstenite::", - "tokio_tungstenite", - "hyper_tungstenite", - "crate::api::admin::", - "crate::api::handlers::", - "crate::api::http::", - "crate::api::tcp::", - "crate::api::transport::", - "crate::api::ws::", -]; -const SYNC_CORE_API_FORBIDDEN: &[&str] = &["crate::api::"]; -const ADMIN_BOUNDARY_FORBIDDEN: &[&str] = &[ - "crate::boot::domains::", - "crate::domains::kv::sink", - "crate::domains::queue::sink", - "crate::domains::notice::sink", - "crate::domains::stream::sink", - "crate::domains::rpc::sink", - "crate::domains::lease::sink", - "crate::domains::schedule::sink", - "crate::domains::kv::actor", - "crate::domains::queue::actor", - "crate::domains::notice::actor", - "crate::domains::stream::actor", - "crate::domains::rpc::actor", - "crate::domains::lease::actor", - "crate::domains::schedule::actor", -]; -const ADMIN_BOUNDARY_ALLOWED: &[&str] = &[ - "crate::domains::kv::sink::AdminKvRowsRequest", - "crate::domains::stream::sink::AdminStreamReadRequest", -]; -const PRODUCTION_RUST_LINE_LIMIT: usize = 1_000; - -#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] -struct OwnedSourceFile { - owner: &'static str, - path: PathBuf, -} - -#[test] -fn should_keep_sync_core_synchronous() { - // Arrange - let repo_root = repo_root(); - let files = sync_core_source_files(&repo_root); - - // Act - let report = report_for_patterns(&repo_root, &files, SYNC_CORE_ASYNC_FORBIDDEN); - - // Assert - assert!( - report.is_empty(), - "found async constructs outside src/api:\n{report}" - ); -} - -#[test] -fn should_keep_transport_plus_admin_frameworks_out_of_sync_core() { - // Arrange - let repo_root = repo_root(); - let files = sync_core_source_files(&repo_root); - - // Act - let report = report_for_patterns(&repo_root, &files, SYNC_CORE_TRANSPORT_FORBIDDEN); - - // Assert - assert!( - report.is_empty(), - "found transport or admin framework dependencies in sync core:\n{report}" - ); -} - -#[test] -fn should_keep_sync_core_independent_from_api_modules() { - // Arrange - let repo_root = repo_root(); - let files = sync_core_source_files(&repo_root); - - // Act - let report = report_for_patterns(&repo_root, &files, SYNC_CORE_API_FORBIDDEN); - - // Assert - assert!( - report.is_empty(), - "sync core must not depend on src/api modules:\n{report}" - ); -} - -#[test] -fn should_keep_protocol_plus_domains_on_dispatch_boundary() { - // Arrange - let repo_root = repo_root(); - let protocol_files = source_files_under(&repo_root.join("src").join("protocol")); - let domain_files = source_files_under(&repo_root.join("src").join("domains")); - - // Act - let protocol_report = report_for_patterns(&repo_root, &protocol_files, &["crate::domains::"]); - let domain_report = report_for_patterns(&repo_root, &domain_files, &["crate::protocol::"]); - let report = [ - (!protocol_report.is_empty()).then(|| { - format!("protocol imports domain DTOs outside dispatch::wire:\n{protocol_report}") - }), - (!domain_report.is_empty()).then(|| { - format!( - "domains import protocol contracts outside dispatch::protocol:\n{domain_report}" - ) - }), - ] - .into_iter() - .flatten() - .collect::>() - .join("\n"); - - // Assert - assert!( - report.is_empty(), - "protocol/domain dependencies must cross through src/dispatch:\n{report}" - ); -} - -#[test] -fn should_disallow_direct_cross_domain_references() { - // Arrange - let repo_root = repo_root(); - let files = domain_owned_source_files(&repo_root); - - // Act - let report = format_violation_report(&collect_foreign_domain_reference_violations( - &repo_root, &files, - )); - - // Assert - assert!( - report.is_empty(), - "found direct cross-domain references:\n{report}" - ); -} - -#[test] -fn should_disallow_foreign_route_scheme_literals() { - // Arrange - let repo_root = repo_root(); - let files = domain_owned_source_files(&repo_root); - - // Act - let report = - format_violation_report(&collect_foreign_route_scheme_violations(&repo_root, &files)); - - // Assert - assert!( - report.is_empty(), - "found foreign route scheme literals:\n{report}" - ); -} - -#[test] -fn should_keep_admin_api_on_runtime_facades() { - // Arrange - let repo_root = repo_root(); - let files = admin_api_source_files(&repo_root); - - // Act - let report = report_for_patterns_with_allowed( - &repo_root, - &files, - ADMIN_BOUNDARY_FORBIDDEN, - ADMIN_BOUNDARY_ALLOWED, - ); - - // Assert - assert!( - report.is_empty(), - "admin API bypasses runtime/admin facades:\n{report}" - ); -} - -#[test] -fn should_keep_rpc_route_actor_removed_from_default_surface() { - // Arrange - let repo_root = repo_root(); - let rpc_mod = repo_root - .join("src") - .join("domains") - .join("rpc") - .join("mod.rs"); - let content = read_source_file(&rpc_mod); - let forbidden_exports = [ - "\npub mod actor;", - "\npub mod session;", - "\npub(crate) mod actor;", - "\npub(crate) mod session;", - "\npub use actor::RpcRouteActor;", - "\npub use session::SessionActor;", - "legacy_actor_tests", - ]; - - // Act - let mut violations = forbidden_exports - .iter() - .filter(|forbidden| content.contains(**forbidden)) - .map(|forbidden| format!("src/domains/rpc/mod.rs exposes `{}`", forbidden.trim())) - .collect::>(); - for file_name in ["actor.rs", "session.rs", "legacy_actor_tests.rs"] { - let path = repo_root - .join("src") - .join("domains") - .join("rpc") - .join(file_name); - if path.exists() { - violations.push(format!( - "{} still exists", - relative_display_path(&repo_root, &path) - )); - } - } - let report = format_violation_report(&violations); - - // Assert - assert!( - report.is_empty(), - "RPC route actor and legacy session helper must stay pruned from the default surface:\n{report}" - ); -} - -#[test] -fn should_keep_shadow_notice_surface_removed() { - // Arrange - let repo_root = repo_root(); - let notice_dir = repo_root.join("src").join("domains").join("notice"); - let notice_mod = read_source_file(¬ice_dir.join("mod.rs")); - let forbidden_exports = [ - "\npub mod actor;", - "\npub mod events;", - "\npub mod session;", - "\npub use actor::NoticeRouteActor;", - "\npub use session::SessionActor;", - ]; - - // Act - let mut violations = forbidden_exports - .iter() - .filter(|forbidden| notice_mod.contains(**forbidden)) - .map(|forbidden| format!("src/domains/notice/mod.rs exposes `{}`", forbidden.trim())) - .collect::>(); - for relative in [ - "src/domains/notice/actor.rs", - "src/domains/notice/events.rs", - "src/domains/notice/session.rs", - "tests/notice_basics.rs", - "tests/notice_advanced.rs", - ] { - if repo_root.join(relative).exists() { - violations.push(format!("{relative} retains the shadow Notice surface")); - } - } - let report = format_violation_report(&violations); - - // Assert - assert!( - report.is_empty(), - "shadow Notice actors and events must stay absent:\n{report}" - ); -} - -#[test] -fn should_keep_notice_family_state_key_type_safe() { - // Arrange - let repo_root = repo_root(); - let sink = read_source_file( - &repo_root - .join("src") - .join("domains") - .join("notice") - .join("sink.rs"), - ); - - // Act - let has_typed_key = sink.contains( - "HashMap>", - ); - let retains_round_trip = sink.contains("RouteFamily::try_from(*family_id)"); - - // Assert - assert!( - has_typed_key, - "Notice family state must use RouteFamily keys" - ); - assert!( - !retains_round_trip, - "Notice cleanup must not reconstruct RouteFamily from an integer key" - ); -} - -#[test] -fn should_keep_notice_backpressure_plus_duplicate_paths_bounded() { - // Arrange - let repo_root = repo_root(); - let notice_sink_dir = repo_root - .join("src") - .join("domains") - .join("notice") - .join("sink"); - let domain_sink = read_source_file(¬ice_sink_dir.join("domain_sink_impl.rs")); - let delivery_worker = read_source_file(¬ice_sink_dir.join("delivery_worker.rs")); - - // Act - let duplicate_check = domain_sink.find("self.try_reuse_existing(sub_msg)"); - let pattern_compile = domain_sink.find("Self::compile_pattern(sub_msg)"); - let has_deadline_retry = delivery_worker.contains("NOTICE_MAILBOX_RETRY_TIMEOUT") - && delivery_worker.contains("Instant::now() < deadline"); - let has_fixed_retry_loop = delivery_worker.contains("MAX_RETRIES"); - - // Assert - assert!( - duplicate_check - .zip(pattern_compile) - .is_some_and(|(check, compile)| check < compile), - "Notice duplicate lookup must precede pattern compilation" - ); - assert!( - has_deadline_retry, - "Notice backpressure retry must use a deadline" - ); - assert!( - !has_fixed_retry_loop, - "Notice retry must not restore a fixed spin count" - ); -} - -#[test] -fn should_compile_lease_bench_commands_only_for_tests_or_benchkit() { - // Arrange - let repo_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let model = read_source_file(&repo_root.join("src/domains/lease/sink/model.rs")); - let lifecycle = read_source_file( - &repo_root.join("src/domains/lease/sink/lifecycle_and_admin/lifecycle.rs"), - ); - - // Act - let gate = "#[cfg(any(test, feature = \"benchkit\"))]"; - - // Assert - assert!(model.matches(gate).count() >= 2); - assert!(lifecycle.matches(gate).count() >= 2); -} - -#[test] -fn should_keep_shadow_lease_actor_removed_from_default_surface() { - // Arrange - let repo_root = repo_root(); - let lease_dir = repo_root.join("src").join("domains").join("lease"); - let lease_mod = read_source_file(&lease_dir.join("mod.rs")); - let removed_files = [ - "actor.rs", - "guard.rs", - "session.rs", - "events.rs", - "projection.rs", - ]; - - // Act - let exposed_shadow_modules = ["actor", "guard", "session", "events", "projection"] - .into_iter() - .filter(|module| lease_mod.contains(&format!("pub mod {module};"))) - .collect::>(); - let retained_shadow_files = removed_files - .into_iter() - .filter(|file| lease_dir.join(file).exists()) - .collect::>(); - - // Assert - assert!( - exposed_shadow_modules.is_empty() && retained_shadow_files.is_empty(), - "shadow Lease surface remains: modules={exposed_shadow_modules:?}, files={retained_shadow_files:?}" - ); -} - -#[test] -fn should_keep_panicking_stream_storage_decoders_test_only() { - // Arrange - let repo_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let storage = repo_root.join("src/domains/stream/storage"); - let compact = read_source_file(&storage.join("compact_page_values.rs")); - let hierarchy = read_source_file(&storage.join("resource_area_realm_values.rs")); - - // Act - let test_gate = "#[cfg(test)]"; - - // Assert - assert!(compact.matches(test_gate).count() >= 3); - assert!(hierarchy.matches(test_gate).count() >= 3); -} - -#[test] -fn should_keep_lease_benchmark_mutation_actor_serialized() { - // Arrange - let repo_root = repo_root(); - let lifecycle = read_source_file( - &repo_root.join("src/domains/lease/sink/lifecycle_and_admin/lifecycle.rs"), - ); - - // Act - let forbidden = [ - "acquire_direct_for_bench", - "release_direct_for_bench", - ".runtime().handle_acquire", - ".runtime().handle_release", - ]; - let violations = forbidden - .into_iter() - .filter(|pattern| lifecycle.contains(pattern)) - .collect::>(); - - // Assert - assert!( - violations.is_empty(), - "Lease benchmark helpers bypass actor serialization: {violations:?}" - ); -} - -#[test] -fn should_keep_scheduler_plus_duplicate_transport_surfaces_private() { - // Arrange - let repo_root = repo_root(); - let runtime_module = read_source_file(&repo_root.join("src/runtime/mod.rs")); - let api_module = read_source_file(&repo_root.join("src/api/mod.rs")); - - // Act - let violations = [ - runtime_module - .contains("pub use scheduler::Scheduler") - .then_some("runtime::Scheduler is publicly re-exported"), - api_module - .contains("pub mod ws;") - .then_some("duplicate api::ws transport module is exported"), - api_module - .contains("pub mod transport;") - .then_some("duplicate api::transport module is exported"), - repo_root - .join("src/api/ws.rs") - .exists() - .then_some("duplicate src/api/ws.rs transport file remains"), - repo_root - .join("src/api/transport.rs") - .exists() - .then_some("duplicate src/api/transport.rs file remains"), - ] - .into_iter() - .flatten() - .map(str::to_string) - .collect::>(); - let report = format_violation_report(&violations); - - // Assert - assert!( - report.is_empty(), - "legacy scheduler and duplicate transport surfaces must stay absent:\n{report}" - ); -} - -#[test] -fn should_keep_production_rust_files_below_line_budget() { - // Arrange - let repo_root = repo_root(); - let files = production_rust_source_files(&repo_root); - - // Act - let violations = files - .iter() - .filter_map(|path| { - let line_count = read_source_file(path).lines().count(); - (line_count > PRODUCTION_RUST_LINE_LIMIT).then(|| { - format!( - "{} has {line_count} lines", - relative_display_path(&repo_root, path) - ) - }) - }) - .collect::>(); - let report = format_violation_report(&violations); - - // Assert - assert!( - report.is_empty(), - "production Rust files must stay at or below {PRODUCTION_RUST_LINE_LIMIT} lines:\n{report}" - ); -} - -#[test] -fn should_keep_domain_actor_mailbox_capacity_centralized() { - // Arrange - let repo_root = repo_root(); - let files = domain_production_source_files(&repo_root); - - // Act - let report = format_violation_report(&collect_domain_mailbox_capacity_violations( - &repo_root, &files, - )); - - // Assert - assert!( - report.is_empty(), - "domain managed actor mailbox capacity must use DOMAIN_ACTOR_MAILBOX_CAPACITY:\n{report}" - ); -} - -#[test] -fn should_document_all_rpc_error_codes_in_client_spec() { - // Arrange - let repo_root = repo_root(); - let error_codes = repo_root - .join("src") - .join("protocol") - .join("error_codes.rs"); - let client_spec = repo_root - .join("docs") - .join("clients") - .join("spec") - .join("queue-rpc-kv.md"); - let constants = rpc_error_constants(&error_codes); - let documented_rows = rpc_error_rows_from_markdown(&client_spec); - - // Act - let missing_rows = constants - .iter() - .filter(|row| !documented_rows.contains(*row)) - .map(|(code, name)| format!("{code} {name} missing from docs/clients/spec/queue-rpc-kv.md")) - .collect::>(); - let report = format_violation_report(&missing_rows); - - // Assert - assert!( - report.is_empty(), - "RPC client spec must list every RPC error code/name from src/protocol/error_codes.rs:\n{report}" - ); -} - -#[test] -fn should_keep_rpc_design_seams_explicit() { - // Arrange - let repo_root = repo_root(); - let rpc = repo_root.join("src/domains/rpc/sink"); - let constants = read_source_file(&rpc.join("state_model/constants.rs")); - let mailbox = read_source_file(&rpc.join("mailbox_sink_impl.rs")); - let requests = read_source_file(&rpc.join("state_model/requests.rs")); - let route_state = read_source_file(&rpc.join("state_model/route_state.rs")); - let state = read_source_file(&rpc.join("state_model/state.rs")); - let worker = read_source_file(&rpc.join("state_model/worker.rs")); - let registration_table = read_source_file(&rpc.join("state_model/registration_table.rs")); - let ready_queue = read_source_file(&rpc.join("state_model/ready_queue.rs")); - let response_forwarder = read_source_file(&rpc.join("response_forwarder.rs")); - - // Act - let violations = [ - ( - !constants.contains("RPC_MSG_TYPE_REQUEST"), - "request message constant", - ), - ( - !constants.contains("RPC_MSG_TYPE_RESPONSE"), - "response message constant", - ), - ( - !mailbox.contains("deliver_with_priority"), - "shared delivery guard", - ), - ( - !requests.contains("dispatch_info: RpcPendingDispatchInfo"), - "owned pending dispatch view", - ), - ( - !route_state.contains("struct RegistrationRotor"), - "registration rotor", - ), - ( - state.contains("clippy::too_many_lines"), - "small dispatch coordinator", - ), - ( - state.contains("fn dispatch_or_queue_request(\n"), - "test-only dispatch wrapper", - ), - ( - !registration_table.contains("struct RegistrationTable"), - "registration table", - ), - ( - !ready_queue.contains("struct RouteReadyQueue"), - "route-ready queue", - ), - ( - !state.contains("trait RpcRequestState") || !state.contains("trait RpcResponseState"), - "request and response state facades", - ), - ( - !response_forwarder.contains("struct RpcResponseForwarder"), - "response forwarder", - ), - ( - !worker.contains("struct RegistrationCredit"), - "registration credit accounting", - ), - ( - [ - "/// Selects the next available registration", - "/// Claims one registration credit", - "/// Reserves one unit of global pending capacity", - "/// Coordinates duplicate, capacity, fairness, and tracking policy", - ] - .iter() - .any(|contract| !state.contains(contract)), - "RPC state policy documentation", - ), - ] - .into_iter() - .filter_map(|(missing, label)| missing.then_some(label)) - .collect::>(); - - // Assert - assert!(violations.is_empty(), "missing RPC seams: {violations:?}"); -} - -#[test] -fn should_use_registration_vocabulary_throughout_rpc_state_model_source() { - // Arrange - let state = read_source_file(&repo_root().join("src/domains/rpc/sink/state_model/state.rs")); - let registration_table = read_source_file( - &repo_root().join("src/domains/rpc/sink/state_model/registration_table.rs"), - ); - - // Act - // Scan comments and literals too: these internal model files should use one vocabulary. - let mixed_terms = [ - ("state.rs", state), - ("registration_table.rs", registration_table), - ] - .into_iter() - .flat_map(|(file, source)| { - source - .split(|character: char| !(character.is_ascii_alphanumeric() || character == '_')) - .filter(|identifier| identifier.contains("worker")) - .map(move |identifier| format!("{file}:{identifier}")) - .collect::>() - }) - .collect::>(); - - // Assert - assert!( - mixed_terms.is_empty(), - "RPC state model source must use registration vocabulary: {mixed_terms:?}" - ); -} - -#[test] -fn should_keep_stream_design_seams_explicit() { - // Arrange - let stream = repo_root().join("src/domains/stream"); - let keys = read_source_file(&stream.join("storage/keys_and_models.rs")); - let model = read_source_file(&stream.join("sink/model.rs")); - let sink = read_source_file(&stream.join("sink/domain_sink_impl.rs")); - let core = read_source_file( - &stream.join("sink/domain_sink_impl/domain_core_impl/watermark_coordination.rs"), - ); - let codecs = read_source_file(&stream.join("storage/compact_page_values.rs")); - let sequence = read_source_file(&stream.join("store/sequence_and_filters.rs")); - let actor = read_source_file(&stream.join("actor.rs")); - let store = read_source_file(&stream.join("store/mod.rs")); - let store_sources = source_files_under(&stream.join("store")) - .iter() - .map(|path| read_source_file(path)) - .collect::>() - .join("\n"); - - // Act - let violations = [ - ( - !keys.contains("impl TryFrom for KeyPrefix"), - "key-prefix decoding", - ), - ( - !model.contains("struct SubscriptionRegistry"), - "subscription registry", - ), - ( - !model.contains("struct AdminSnapshotState"), - "admin snapshot state", - ), - ( - !model.contains("struct WatermarkCoordinators"), - "watermark coordinators", - ), - ( - !codecs.contains("trait PageRecordCodec"), - "page-record codec", - ), - ( - actor.contains("impl ActiveAppendSession {}"), - "empty append-session impl", - ), - ( - !store.contains("enum StreamStoreError"), - "stream store error", - ), - ( - [ - "commit_records_promotion_frontier(", - "commit_session_promotion_frontier(", - "read_resource_promotion_frontier(", - "read_area_promotion_frontier(", - "read_realm_promotion_frontier(", - ] - .iter() - .any(|wrapper| store_sources.contains(wrapper)), - "single-layout wrapper twins", - ), - ( - !core.contains("fn dispatch_watermark_commit"), - "shared watermark dispatch", - ), - ( - !sink.contains("fn dispatch_family_command"), - "shared family command dispatch", - ), - ( - !sequence.contains("fn load_existing_watermark_for_guard"), - "shared watermark guard read", - ), - ( - !sequence.contains("for key in keys"), - "discriminator row loop", - ), - ( - !keys.contains("LEGACY D3 PROTOTYPE PREFIXES"), - "legacy prototype prefix boundary", - ), - ] - .into_iter() - .filter_map(|(missing, label)| missing.then_some(label)) - .collect::>(); - - // Assert - assert!( - violations.is_empty(), - "missing Stream seams: {violations:?}" - ); -} - -#[test] -fn should_keep_queue_design_seams_explicit() { - // Arrange - let queue = repo_root().join("src/domains/queue"); - let actor = read_source_file(&queue.join("actor/mod.rs")); - let ack = read_source_file(&queue.join("actor/reserve_and_ack.rs")); - let storage = read_source_file(&queue.join("actor/storage.rs")); - let sink = read_source_file(&queue.join("sink/domain_sink_impl.rs")); - - // Act - let violations = [ - ( - !queue.join("actor/dlq.rs").exists(), - "DLQ transition module", - ), - ( - !queue.join("actor/dead_letter_admin.rs").exists(), - "dead-letter admin module", - ), - ( - !queue.join("actor/startup_reconciliation.rs").exists(), - "startup reconciliation module", - ), - ( - !actor.contains("fn wire_code") || !actor.contains("fn as_str"), - "DLQ reason mappings", - ), - ( - !actor.contains("trait QueueDataPlane") || !actor.contains("trait QueueAdminPlane"), - "queue interface traits", - ), - ( - !ack.contains("fn validate_ack_authorization"), - "ack authorization seam", - ), - ( - !ack.contains("stage_delayed") || !ack.contains("fast path"), - "ack staging and fast-path documentation", - ), - ( - !storage.contains("fn commit_transaction"), - "shared transaction commit", - ), - ( - !sink.contains("struct QueueCounts"), - "queue counts accessor", - ), - ( - !actor.contains("QUEUE_IDLE_HORIZON") - || !actor.contains("QUEUE_STORAGE_RETRY_BACKOFF") - || !actor.contains("QUEUE_ACTOR_REPLY_TIMEOUT"), - "queue timing constants", - ), - ] - .into_iter() - .filter_map(|(missing, label)| missing.then_some(label)) - .collect::>(); - - // Assert - assert!(violations.is_empty(), "missing Queue seams: {violations:?}"); -} - -#[test] -fn should_keep_schedule_design_seams_explicit() { - // Arrange - let schedule = repo_root().join("src/domains/schedule"); - let actor = read_source_file(&schedule.join("actor/claim_and_ack.rs")); - let actor_mod = read_source_file(&schedule.join("actor/mod.rs")); - let sink = read_source_file(&schedule.join("sink/domain_sink_impl.rs")); - let model = read_source_file(&schedule.join("sink/model.rs")); - let store = read_source_file(&schedule.join("store/model.rs")); - - // Act - let violations = [ - ( - !schedule.join("sink/delivery_strategy.rs").exists(), - "delivery strategy", - ), - ( - !sink.contains("fn claim_due") - || !sink.contains("fn deliver_claims") - || !sink.contains("fn acknowledge_delivered"), - "due scan stages", - ), - ( - !actor.contains("fn pop_due_from_heap") - || !actor.contains("fn recompute_next_fires") - || !actor.contains("fn persist_claims") - || !actor.contains("fn apply_claims_to_state"), - "claim stages", - ), - ( - !model.contains("enum PendingFireState"), - "pending-fire state", - ), - ( - !actor_mod.contains("#[cfg(test)]") || !actor_mod.contains("test_actor_harness"), - "test-only actor harness", - ), - ( - !sink.contains("trait ScheduleObservability"), - "observability interface", - ), - ( - !store.contains("trait SchedulePersistence"), - "persistence interface", - ), - (schedule.join("events.rs").exists(), "dead schedule events"), - ( - !actor_mod.contains("SCAN_DEDUP_WINDOW") || !model.contains("EXECUTIONS_WINDOW_MS"), - "schedule timing constants", - ), - ( - !model.contains("sink wrapper") || !model.contains("runtime body"), - "sink runtime naming docs", - ), - ] - .into_iter() - .filter_map(|(missing, label)| missing.then_some(label)) - .collect::>(); - - // Assert - assert!( - violations.is_empty(), - "missing Schedule seams: {violations:?}" - ); -} - -#[test] -fn should_complete_reopened_kv_plus_lease_design_criteria() { - // Arrange - let root = repo_root().join("src/domains"); - let kv_domain = read_source_file(&root.join("kv/sink/domain_sink_impl.rs")); - let kv_mailbox = read_source_file(&root.join("kv/sink/mailbox_sink_impl.rs")); - let lease_expiry = read_source_file(&root.join("lease/sink/domain_sink_impl/expiry.rs")); - let lease_mailbox = read_source_file(&root.join("lease/sink/mailbox_sink_impl.rs")); - - // Act - let violations = [ - ( - !kv_domain.contains("use crate::domains::kv::KvActor;") - || kv_domain.contains("crate::domains::kv::KvActor::"), - "KV domain actor import cleanup", - ), - ( - !kv_mailbox.contains("use crate::domains::kv::{KvActor, KvError, KvResponse};") - || ["KvActor", "KvError", "KvResponse"] - .iter() - .any(|name| kv_mailbox.contains(&format!("crate::domains::kv::{name}"))), - "KV mailbox imports cleanup", - ), - ( - !lease_expiry.contains( - "/// Removes every queued waiter owned by the session before empty queues are dropped.", - ), - "Lease session-waiter ordering docs", - ), - ( - !lease_mailbox.contains("fn scope_operation_owner") - || lease_mailbox.matches("session_scoped_owner_id(").count() != 1, - "Lease owner-scoping step", - ), - ] - .into_iter() - .filter_map(|(missing, label)| missing.then_some(label)) - .collect::>(); - - // Assert - assert!( - violations.is_empty(), - "reopened design criteria remain incomplete: {violations:?}" - ); -} - -#[test] -fn should_document_unified_wildcard_registration_plus_exact_lease_semantics() { - // Arrange - let root = repo_root().join("docs"); - let wire = read_source_file(&root.join("clients/spec/wire-routing.md")); - let boundaries = read_source_file(&root.join("development/domain-boundaries-spec.md")); - let laws = read_source_file(&root.join("development/architectural-laws.md")); - let schedule = read_source_file(&root.join("clients/spec/lease-schedule.md")); - let operations = read_source_file(&root.join("clients/spec/operations.md")); - - // Act - let combined = [ - wire.as_str(), - boundaries.as_str(), - laws.as_str(), - schedule.as_str(), - operations.as_str(), - ] - .join("\n"); - - // Assert - assert!(wire - .contains("KV, Queue, Notice, Stream, RPC, and Schedule each permit at most 128 wildcard")); - assert!( - wire.contains("Notifications carry the matching `subscription_id` and the exact concrete") - ); - assert!(wire.contains("Ready concrete routes rotate fairly")); - assert!(boundaries.contains("exact and wildcard registrations are equal candidates")); - assert!(boundaries.contains("Lease does not participate in this wildcard contract")); - assert!(laws.contains("whole-segment `*` and `**`")); - assert!(schedule.contains("Overlapping\npatterns remain distinct")); - assert!(schedule.contains("Watches are exact-route subscriptions")); - assert!(schedule.contains("5010 = ERR_INVALID_SUBSCRIPTION_ROUTE")); - assert!(operations.contains("KV, Queue, Notice, Stream, RPC, and Schedule registrations")); - assert!(operations.contains("Duplicate `(session, original registration")); - assert!(operations.contains("Matching never\ncrosses `RouteFamily`")); - assert!(operations.contains("the exact concrete route")); - assert!(operations.contains("Lease is intentionally different")); - assert!(!combined.contains("Wildcard worker registration is not part of the contract")); - assert!(!combined.contains("Workers register exact listening routes")); - assert!(!combined.contains("Wildcard schedule subscribe is invalid")); - assert!(!combined.contains("Lease subscriptions accept wildcard")); - assert!(!combined.contains("Lease watches support `*`")); -} - -#[test] -fn should_keep_boot_runtime_design_seams_explicit() { - // Arrange - let root = repo_root(); - let boot = read_source_file(&root.join("src/boot/mod.rs")); - let storage = read_source_file(&root.join("src/boot/storage.rs")); - let config = read_source_file(&root.join("src/boot/runtime/config.rs")); - let cloud = read_source_file(&root.join("src/boot/runtime/config/cloud_provider.rs")); - let env = read_source_file(&root.join("src/boot/runtime/config/env.rs")); - let domains = read_source_file(&root.join("src/boot/domains.rs")); - let pool = - read_source_file(&root.join("src/runtime/family_".to_string() + "a" + "ctor_pool.rs")); - let managed = - read_source_file(&root.join("src/runtime/managed_".to_string() + "a" + "ctor.rs")); - let shutdown = read_source_file(&root.join("src/boot/shutdown.rs")); - - // Act - let required = [ - (boot.contains("enum BootStage"), "named boot stages"), - (boot.contains("fn start_listeners"), "listener stage"), - (boot.contains("fn open_storage_stage"), "storage stage"), - (boot.contains("fn register_domains_stage"), "domain stage"), - (!boot.contains("clippy::too_many_lines"), "boot line lint"), - ( - boot.matches(&["\n ShutdownContext ", &char::from(123).to_string()].concat()) - .count() - == 1, - "shutdown context construction", - ), - ( - root.join("src/boot/storage/backoff.rs").is_file(), - "storage backoff module", - ), - ( - root.join("src/boot/storage/contention.rs").is_file(), - "storage contention module", - ), - ( - read_source_file(&root.join("src/boot/storage/contention.rs")) - .contains("enum ContentionKind"), - "typed storage contention seam", - ), - ( - config.contains("struct TransportConfig"), - "transport sub-config", - ), - ( - config.contains("struct StorageConfig"), - "storage sub-config", - ), - (config.contains("struct DrainConfig"), "drain sub-config"), - ( - storage.contains("fn open_with_retry"), - "shared storage retry loop", - ), - ( - cloud.contains("fn s3_compatible_provider"), - "shared S3-compatible provider constructor", - ), - ( - cloud.contains("PROVIDER_DESCRIPTORS"), - "provider descriptor table", - ), - ( - config.contains("fn cloud_durable_write_options"), - "cloud write options mapping", - ), - ( - env.contains("fn positive_u64_from_env"), - "positive integer environment parser", - ), - ( - managed.contains(&("Unsupervised ".to_string() + "a" + "ctors do not fire timers")), - "unsupervised timer contract", - ), - ( - domains.contains("DomainKind::ALL.len()"), - "domain handle consistency regression", - ), - ( - pool.contains(&("struct Family".to_string() + "A" + "ctorPoolHealthSnapshot")), - "family pool health type", - ), - ( - shutdown.contains("PRIORITY_FATAL"), - "named shutdown priority", - ), - ( - !boot.contains("fn warn_defaulted_fast_queue_policy"), - "queue warning ownership", - ), - ( - config.contains("fn warn_defaulted_fast_queue_policy"), - "queue warning policy", - ), - ]; - let missing = required - .into_iter() - .filter_map(|(present, label)| (!present).then_some(label)) - .collect::>(); - - // Assert - assert!(missing.is_empty(), "missing boot/runtime seams"); -} - -#[test] -fn should_document_route_bearing_schedule_notify_wire_format() { - // Arrange - let root = repo_root().join("docs"); - let schedule = read_source_file(&root.join("clients/spec/lease-schedule.md")); - let migration = read_source_file(&root.join("operations/migration-guide.md")); - - // Act - let has_route_bearing_schema = schedule.contains("[u32 BE] exact_route_len") - && schedule.contains("[bytes] exact_route") - && schedule.contains("[subscription_id][exact_route][payload]"); - - // Assert - assert!(has_route_bearing_schema); - assert!(migration - .contains("`[subscription_id][payload]` to `[subscription_id][exact_route][payload]`")); -} - -#[test] -fn should_keep_runtime_ingress_payload_dispatch_free_of_payload_unwraps() { - // Arrange - let repo_root = repo_root(); - let files = [ - repo_root - .join("src") - .join("api") - .join("runtime_ingress") - .join("trait_impls.rs"), - repo_root - .join("src") - .join("api") - .join("runtime_ingress") - .join("domain_frame_dispatcher.rs"), - ]; - - // Act - let violations = files - .iter() - .flat_map(|path| { - let relative_path = relative_display_path(&repo_root, path); - read_source_file(path) - .lines() - .enumerate() - .filter(|(_, line)| line.contains("payload") && line.contains(".unwrap()")) - .map({ - let relative_path = relative_path.clone(); - move |(line_index, _)| { - format!( - "{}:{} contains a payload unwrap invariant", - relative_path, - line_index + 1 - ) - } - }) - .collect::>() - }) - .collect::>(); - let report = format_violation_report(&violations); - - // Assert - assert!( - report.is_empty(), - "runtime ingress payload dispatch must stay free of payload unwrap invariants:\n{report}" - ); -} - -fn repo_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) -} - -fn domain_owned_source_files(repo_root: &Path) -> Vec { - let mut files = Vec::new(); - for &domain in DOMAINS { - collect_owned_rust_files( - &repo_root.join("src").join("domains").join(domain), - domain, - &mut files, - ); - } - files.sort(); - files -} - -fn collect_owned_rust_files( - directory: &Path, - owner: &'static str, - files: &mut Vec, -) { - let entries = fs::read_dir(directory) - .unwrap_or_else(|error| panic!("failed to read {}: {error}", directory.display())); - - for entry in entries { - let path = entry - .unwrap_or_else(|error| { - panic!("failed to read entry in {}: {error}", directory.display()) - }) - .path(); - - if path.is_dir() { - collect_owned_rust_files(&path, owner, files); - } else if path.extension().and_then(|ext| ext.to_str()) == Some("rs") { - files.push(OwnedSourceFile { owner, path }); - } - } -} - -fn sync_core_source_files(repo_root: &Path) -> Vec { - let mut files = Vec::new(); - for directory in SYNC_CORE_DIRS { - collect_rust_files(&repo_root.join("src").join(directory), &mut files); - } - files.sort(); - files -} - -fn source_files_under(directory: &Path) -> Vec { - let mut files = Vec::new(); - collect_rust_files(directory, &mut files); - files.sort(); - files -} - -fn production_rust_source_files(repo_root: &Path) -> Vec { - let mut files = Vec::new(); - collect_rust_files(&repo_root.join("src"), &mut files); - files.retain(|path| { - let relative = relative_display_path(repo_root, path); - !relative.ends_with("/tests.rs") - && !relative.contains("/tests/") - && !relative.contains("/test_helpers.rs") - }); - files.sort(); - files -} - -fn domain_production_source_files(repo_root: &Path) -> Vec { - let mut files = Vec::new(); - collect_rust_files(&repo_root.join("src").join("domains"), &mut files); - files.retain(|path| { - let relative = relative_display_path(repo_root, path); - !relative.ends_with("/tests.rs") - && !relative.contains("/tests/") - && !relative.contains("/test_helpers.rs") - }); - files.sort(); - files -} - -fn admin_api_source_files(repo_root: &Path) -> Vec { - let mut files = Vec::new(); - collect_rust_files(&repo_root.join("src").join("api").join("admin"), &mut files); - files.retain(|path| { - let relative = relative_display_path(repo_root, path); - !relative.ends_with("/tests.rs") && !relative.contains("/tests/") - }); - files.sort(); - files -} - -fn collect_rust_files(directory: &Path, files: &mut Vec) { - let entries = fs::read_dir(directory) - .unwrap_or_else(|error| panic!("failed to read {}: {error}", directory.display())); - - for entry in entries { - let path = entry - .unwrap_or_else(|error| { - panic!("failed to read entry in {}: {error}", directory.display()) - }) - .path(); - - if path.is_dir() { - collect_rust_files(&path, files); - } else if path.extension().and_then(|ext| ext.to_str()) == Some("rs") { - files.push(path); - } - } -} - -fn collect_domain_mailbox_capacity_violations(repo_root: &Path, files: &[PathBuf]) -> Vec { - let mut violations = Vec::new(); - - for path in files { - let content = read_source_file(path); - let lines = content.lines().collect::>(); - for (line_index, line) in lines.iter().enumerate() { - if !line.contains("ManagedActor::spawn_supervised") { - continue; - } - - let window_end = (line_index + 8).min(lines.len()); - let window = lines[line_index..window_end].join("\n"); - if !window.contains("DOMAIN_ACTOR_MAILBOX_CAPACITY") { - violations.push(format!( - "{}:{} spawns managed actor without centralized domain mailbox capacity", - relative_display_path(repo_root, path), - line_index + 1 - )); - } - } - } - - violations -} - -fn collect_foreign_domain_reference_violations( - repo_root: &Path, - files: &[OwnedSourceFile], -) -> Vec { - let mut violations = Vec::new(); - - for file in files { - let content = read_source_file(&file.path); - for (line_index, line) in content.lines().enumerate() { - for other_domain in DOMAINS { - if *other_domain == file.owner { - continue; - } - - let needle = format!("crate::domains::{other_domain}::"); - if line.contains(&needle) { - violations.push(format!( - "{}:{} references {other_domain} domain module", - relative_display_path(repo_root, &file.path), - line_index + 1 - )); - } - } - } - } - - violations -} - -fn collect_foreign_route_scheme_violations( - repo_root: &Path, - files: &[OwnedSourceFile], -) -> Vec { - let mut violations = Vec::new(); - - for file in files { - let content = read_source_file(&file.path); - for (line_index, line) in content.lines().enumerate() { - for other_domain in DOMAINS { - if *other_domain == file.owner { - continue; - } - - let needle = format!("{other_domain}://"); - if line.contains(&needle) { - violations.push(format!( - "{}:{} hard-codes {other_domain} route scheme", - relative_display_path(repo_root, &file.path), - line_index + 1 - )); - } - } - } - } - - violations -} - -fn report_for_patterns(repo_root: &Path, files: &[PathBuf], forbidden: &[&str]) -> String { - report_for_patterns_with_allowed(repo_root, files, forbidden, &[]) -} - -fn report_for_patterns_with_allowed( - repo_root: &Path, - files: &[PathBuf], - forbidden: &[&str], - allowed: &[&str], -) -> String { - let violations = files - .iter() - .flat_map(|path| { - let content = read_source_file(path); - let relative_path = relative_display_path(repo_root, path); - - content - .lines() - .enumerate() - .flat_map(|(line_index, line)| { - forbidden - .iter() - .filter(move |needle| { - line.contains(**needle) - && !allowed.iter().any(|exception| line.contains(exception)) - }) - .map({ - let relative_path = relative_path.clone(); - move |needle| { - format!("{}:{} contains {needle}", relative_path, line_index + 1) - } - }) - }) - .collect::>() - }) - .collect::>(); - - format_violation_report(&violations) -} - -fn read_source_file(path: &Path) -> String { - fs::read_to_string(path) - .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())) -} - -fn rpc_error_constants(path: &Path) -> Vec<(u16, String)> { - let mut in_rpc_section = false; - let mut rows = Vec::new(); - - for line in read_source_file(path).lines() { - let trimmed = line.trim(); - if trimmed == "pub mod rpc {" { - in_rpc_section = true; - continue; - } - if in_rpc_section && trimmed == "}" { - break; - } - if !in_rpc_section || !trimmed.starts_with("pub const ") { - continue; - } - - let definition = &trimmed["pub const ".len()..]; - let Some((name, value_suffix)) = definition.split_once(": u16 = ") else { - continue; - }; - let digits = value_suffix - .chars() - .take_while(char::is_ascii_digit) - .collect::(); - if let Ok(code) = digits.parse::() { - rows.push((code, name.to_string())); - } - } - - rows -} - -fn rpc_error_rows_from_markdown(path: &Path) -> BTreeSet<(u16, String)> { - read_source_file(path) - .lines() - .filter_map(|line| { - let trimmed = line.trim(); - if !trimmed.starts_with('|') { - return None; - } - - let columns = trimmed - .trim_matches('|') - .split('|') - .map(str::trim) - .collect::>(); - if columns.len() < 2 { - return None; - } - - let Ok(code) = columns[0].parse::() else { - return None; - }; - if !(6001..=6013).contains(&code) { - return None; - } - - Some((code, columns[1].to_string())) - }) - .collect() -} - -fn relative_display_path(repo_root: &Path, path: &Path) -> String { - path.strip_prefix(repo_root) - .unwrap_or(path) - .to_string_lossy() - .replace('\\', "/") -} - -fn format_violation_report(violations: &[String]) -> String { - if violations.is_empty() { - String::new() - } else { - violations.join("\n") - } -}